diff --git a/.ci/latest_deps_build_failed_issue_template.md b/.ci/latest_deps_build_failed_issue_template.md
new file mode 100644
index 000000000..052540250
--- /dev/null
+++ b/.ci/latest_deps_build_failed_issue_template.md
@@ -0,0 +1,4 @@
+---
+title: CI run against latest deps is failing
+---
+See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}
diff --git a/.ci/patch_for_twisted_trunk.sh b/.ci/patch_for_twisted_trunk.sh
deleted file mode 100755
index f52458198..000000000
--- a/.ci/patch_for_twisted_trunk.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py
diff --git a/.dockerignore b/.dockerignore
index 09f073331..9f92cfec5 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -9,8 +9,4 @@
 !poetry.lock
 !requirements.txt
 
-# TODO: remove these once we have moved over to using poetry-core in pyproject.toml
-!MANIFEST.in
-!setup.py
-
 **/__pycache__
diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml
new file mode 100644
index 000000000..1a61d179d
--- /dev/null
+++ b/.github/workflows/latest_deps.yml
@@ -0,0 +1,156 @@
+# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
+# dependencies which match the broad requirements. Since most CI runs are against
+# the locked poetry environment, run specifically against the latest dependencies to
+# know if there's an upcoming breaking change.
+#
+# As an overview this workflow:
+# - checks out develop,
+# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
+# - runs mypy and test suites in that checkout.
+#
+# Based on the twisted trunk CI job.
+
+name: Latest dependencies
+
+on:
+  schedule:
+    - cron: 0 7 * * *
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # The dev dependencies aren't exposed in the wheel metadata (at least with current
+      # poetry-core versions), so we install with poetry.
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.2.0b1"
+      # Dump installed versions for debugging.
+      - run: poetry run pip list > before.txt
+      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
+      # `pip install matrix-synapse[all]` as closely as possible.
+      - run: poetry update --no-dev
+      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
+      - run: poetry run mypy
+  trial:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - database: "sqlite"
+          - database: "postgres"
+            postgres-version: "14"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install .[all,test]
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: python -m twisted.trial --jobs=2 tests
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+
+  sytest:
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:testing
+      volumes:
+        - ${{ github.workspace }}:/src
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: focal
+
+          - sytest-tag: focal
+            postgres: postgres
+            workers: workers
+            redis: redis
+    env:
+      POSTGRES: ${{ matrix.postgres && 1}}
+      WORKERS: ${{ matrix.workers && 1 }}
+      REDIS: ${{ matrix.redis && 1 }}
+      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ensure sytest runs `pip install`
+        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
+        run: rm /src/poetry.lock
+        working-directory: /src
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+
+  # TODO: run complement (as with twisted trunk, see #12473).
+
+  # open an issue if the build fails, so we know about it.
+  open-issue:
+    if: failure()
+    needs:
+      # TODO: should mypy be included here? It feels more brittle than the other two.
+      - mypy
+      - trial
+      - sytest
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          update_existing: true
+          filename: .ci/latest_deps_build_failed_issue_template.md
+
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5a98f6193..cad4cb6d7 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -15,24 +15,18 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - run: pip install -e .
+      - run: pip install .
       - run: scripts-dev/generate_sample_config.sh --check
       - run: scripts-dev/config-lint.sh
 
   lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
+    # This does a vanilla `poetry install` - no extras. I'm slightly anxious
+    # that we might skip some typechecks on code that uses extras. However,
+    # I think the right way to fix this is to mark any extras needed for
+    # typechecking as development dependencies. To detect this, we ought to
+    # turn up mypy's strictness: disallow unknown imports and be accept fewer
+    # uses of `Any`.
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
 
   lint-crlf:
     runs-on: ubuntu-latest
@@ -71,23 +65,23 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
         database: ["sqlite"]
-        toxenv: ["py"]
+        extras: ["all"]
         include:
           # Newest Python without optional deps
           - python-version: "3.10"
-            toxenv: "py-noextras"
+            extras: ""
 
           # Oldest Python with PostgreSQL
           - python-version: "3.7"
             database: "postgres"
             postgres-version: "10"
-            toxenv: "py"
+            extras: "all"
 
           # Newest Python with newest PostgreSQL
           - python-version: "3.10"
             database: "postgres"
             postgres-version: "14"
-            toxenv: "py"
+            extras: "all"
 
     steps:
       - uses: actions/checkout@v2
@@ -99,17 +93,16 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
+          extras: ${{ matrix.extras }}
       - name: Await PostgreSQL
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
+      - run: poetry run trial --jobs=2 tests
         env:
-          TRIAL_FLAGS: "--jobs=2"
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
           SYNAPSE_POSTGRES_HOST: localhost
           SYNAPSE_POSTGRES_USER: postgres
@@ -156,23 +149,24 @@ jobs:
 
   trial-pypy:
     # Very slow; only run if the branch name includes 'pypy'
+    # Note: sqlite only; no postgres. Completely untested since poetry move.
     if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["pypy-3.7"]
+        extras: ["all"]
 
     steps:
       - uses: actions/checkout@v2
+      # Install libs necessary for PyPy to build binary wheels for dependencies
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          extras: ${{ matrix.extras }}
+      - run: poetry run trial --jobs=2 tests
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml
index fb9d46b7b..8fc1affb7 100644
--- a/.github/workflows/twisted_trunk.yml
+++ b/.github/workflows/twisted_trunk.yml
@@ -6,16 +6,25 @@ on:
 
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   mypy:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          extras: "all"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run mypy
 
   trial:
     runs-on: ubuntu-latest
@@ -23,14 +32,15 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
-          python-version: 3.7
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          python-version: "3.x"
+          extras: "all test"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run trial --jobs 2 tests
 
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
@@ -55,11 +65,23 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
+        # Note: The poetry commands want to create a virtualenv in /src/.venv/,
+        #       but the sytest-synapse container expects it to be in /venv/.
+        #       We symlink it before running poetry so that poetry actually
+        #       ends up installing to `/venv`.
+        run: |
+          ln -s -T /venv /src/.venv
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
         working-directory: /src
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
+        env:
+          # Use offline mode to avoid reinstalling the pinned version of
+          # twisted.
+          OFFLINE: 1
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
diff --git a/.gitignore b/.gitignore
index c011cd27a..e58affb24 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,8 +15,7 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
-# We do want the poetry lockfile. TODO: is there a good reason for ignoring
-# '*.lock' above? If not, let's nuke it.
+# We do want the poetry lockfile.
 !poetry.lock
 
 # stuff that is likely to exist when you run a server locally
diff --git a/CHANGES.md b/CHANGES.md
index a7d2529b5..d8ecb11e4 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,75 @@
+Synapse 1.58.0rc1 (2022-04-26)
+==============================
+
+As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61.
+
+Features
+--------
+
+- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398))
+- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537))
+- Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465))
+- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427))
+- Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543))
+
+
+Bugfixes
+--------
+
+- Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213))
+- Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319))
+- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/v3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476))
+- Fix a bug introduced in Synapse 1.27.0 where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1, no matter how many extremities were deleted. ([\#12496](https://github.com/matrix-org/synapse/issues/12496))
+- Fix a long-standing bug where the image thumbnails embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510))
+- Fix a bug in the implementation of [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202) where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520))
+- Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522))
+
+
+Improved Documentation
+----------------------
+
+- Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340))
+- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527))
+- Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451))
+- Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457))
+- Strongly recommend [Poetry](https://python-poetry.org/) for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475))
+- Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492))
+- Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495))
+- Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501))
+- Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533))
+
+
+Deprecations and Removals
+-------------------------
+
+- The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))
+- Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382))
+
+
+Internal Changes
+----------------
+
+- Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394))
+- Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399))
+- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395))
+- Run CI in the locked [Poetry](https://python-poetry.org/) environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472))
+- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445))
+- Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450))
+- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497))
+- Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455))
+- Limit length of `device_id` to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454))
+- Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464))
+- Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466))
+- complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467))
+- Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474))
+- Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483))
+- Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511))
+- Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519))
+- Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528))
+- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468))
+- Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529))
+
+
 Synapse 1.57.1 (2022-04-20)
 ===========================
 
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index d744c090a..000000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,54 +0,0 @@
-include LICENSE
-include VERSION
-include *.rst
-include *.md
-include demo/README
-include demo/demo.tls.dh
-include demo/*.py
-include demo/*.sh
-
-include synapse/py.typed
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
-
-recursive-include docs *
-recursive-include scripts-dev *
-recursive-include synapse *.pyi
-recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
-
-recursive-include synapse/res *
-recursive-include synapse/static *.css
-recursive-include synapse/static *.gif
-recursive-include synapse/static *.html
-recursive-include synapse/static *.js
-
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
-exclude Dockerfile
-exclude mypy.ini
-exclude sytest-blacklist
-exclude test_postgresql.sh
-
-include book.toml
-include pyproject.toml
-recursive-include changelog.d *
-
-include .flake8
-prune .circleci
-prune .github
-prune .ci
-prune contrib
-prune debian
-prune demo/etc
-prune docker
-prune stubs
diff --git a/README.rst b/README.rst
index 595fb5ff6..d71d73367 100644
--- a/README.rst
+++ b/README.rst
@@ -293,39 +293,42 @@ directory of your choice::
     git clone https://github.com/matrix-org/synapse.git
     cd synapse
 
-Synapse has a number of external dependencies, that are easiest
-to install using pip and a virtualenv::
+Synapse has a number of external dependencies. We maintain a fixed development
+environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend
 
-    python3 -m venv ./env
-    source ./env/bin/activate
-    pip install -e ".[all,dev]"
+    pip install --user pipx
+    pipx install poetry
+
+as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`
+for other installation methods.) Then ask poetry to create a virtual environment
+from the project and install Synapse's dependencies::
+
+    poetry install --extras "all test"
 
 This will run a process of downloading and installing all the needed
-dependencies into a virtual env. If any dependencies fail to install,
-try installing the failing modules individually::
-
-    pip install -e "module-name"
+dependencies into a virtual env.
 
 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
-    ./demo/start.sh
+    poetry run ./demo/start.sh
 
-(to stop, you can use `./demo/stop.sh`)
+(to stop, you can use `poetry run ./demo/stop.sh`)
 
-See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
+See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
 for more information.
 
 If you just want to start a single instance of the app and run it directly::
 
     # Create the homeserver.yaml config once
-    python -m synapse.app.homeserver \
+    poetry run synapse_homeserver \
       --server-name my.domain.name \
       --config-path homeserver.yaml \
       --generate-config \
       --report-stats=[yes|no]
 
     # Start the app
-    python -m synapse.app.homeserver --config-path homeserver.yaml
+    poetry run synapse_homeserver --config-path homeserver.yaml
 
 
 Running the unit tests
@@ -334,7 +337,7 @@ Running the unit tests
 After getting up and running, you may wish to run Synapse's unit tests to
 check that everything is installed correctly::
 
-    trial tests
+    poetry run trial tests
 
 This should end with a 'PASSED' result (note that exact numbers will
 differ)::
diff --git a/debian/build_virtualenv b/debian/build_virtualenv
index e69116361..b06879259 100755
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -30,9 +30,19 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
         ;;
 esac
 
-# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
-# than the 2/3 compatible `virtualenv`.
+# Manually install Poetry and export a pip-compatible `requirements.txt`
+# We need a Poetry pre-release as the export command is buggy in < 1.2
+TEMP_VENV="$(mktemp -d)"
+python3 -m venv "$TEMP_VENV"
+source "$TEMP_VENV/bin/activate"
+pip install -U pip
+pip install poetry==1.2.0b1
+poetry export --extras all --extras test -o exported_requirements.txt
+deactivate
+rm -rf "$TEMP_VENV"
 
+# Use --no-deps to only install pinned versions in exported_requirements.txt,
+# and to avoid https://github.com/pypa/pip/issues/9644
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
@@ -41,9 +51,11 @@ dh_virtualenv \
     --preinstall="lxml" \
     --preinstall="mock" \
     --preinstall="wheel" \
+    --extra-pip-arg="--no-deps" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all,systemd,test" \
+    --requirements="exported_requirements.txt"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
diff --git a/debian/changelog b/debian/changelog
index 2db6ed949..20e756f6d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium
+
+  * Use poetry to manage the bundled virtualenv included with this package.
+  * New Synapse release 1.58.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 11:15:20 +0100
+
 matrix-synapse-py3 (1.57.1) stable; urgency=medium
 
   * New synapse release 1.57.1.
diff --git a/debian/clean b/debian/clean
new file mode 100644
index 000000000..d488f298d
--- /dev/null
+++ b/debian/clean
@@ -0,0 +1 @@
+exported_requirements.txt
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 6f87702cc..4523c6064 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 WORKDIR /synapse
 
 # Copy just what we need to run `poetry export`...
-COPY pyproject.toml poetry.lock README.rst /synapse/
+COPY pyproject.toml poetry.lock /synapse/
 
 RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
 
@@ -98,9 +98,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
 # ... and what we need to `pip install`.
-# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy
-# pyproject.toml here, ditching setup.py and MANIFEST.in.
-COPY setup.py MANIFEST.in README.rst /synapse/
+COPY pyproject.toml README.rst /synapse/
 
 # Install the synapse package itself.
 RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
index 6fb1cdbfb..9ccb2b22a 100644
--- a/docker/Dockerfile-workers
+++ b/docker/Dockerfile-workers
@@ -2,10 +2,19 @@
 FROM matrixdotorg/synapse
 
 # Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+     redis-server nginx-light
 
-# Remove the default nginx sites
+# Install supervisord with pip instead of apt, to avoid installing a second
+# copy of python.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install supervisor~=4.2
+
+# Disable the default nginx sites
 RUN rm /etc/nginx/sites-enabled/default
 
 # Copy Synapse worker, nginx and supervisord configuration template files
@@ -19,5 +28,7 @@ EXPOSE 8080/tcp
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
 ENTRYPOINT ["/configure_workers_and_start.py"]
 
+# Replace the healthcheck with one which checks *all* the workers. The script
+# is generated by configure_workers_and_start.py.
 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
     CMD /bin/sh /healthcheck.sh
diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile
index 982219a91..65df2d114 100644
--- a/docker/complement/SynapseWorkers.Dockerfile
+++ b/docker/complement/SynapseWorkers.Dockerfile
@@ -13,8 +13,8 @@ RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/cadd
   tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root
 
 # Install postgresql
-RUN apt-get update
-RUN apt-get install -y postgresql
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
 
 # Configure a user and create a database for Synapse
 RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
@@ -34,40 +34,14 @@ WORKDIR /data
 # Copy the caddy config
 COPY conf-workers/caddy.complement.json /root/caddy.json
 
+# Copy the entrypoint
+COPY conf-workers/start-complement-synapse-workers.sh /
+
 # Expose caddy's listener ports
 EXPOSE 8008 8448
 
-ENTRYPOINT \
-  # Replace the server name in the caddy config
-  sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \
-  # Start postgres
-  pg_ctlcluster 13 main start 2>&1 && \
-  # Start caddy
-  /root/caddy start --config /root/caddy.json 2>&1 && \
-  # Set the server name of the homeserver
-  SYNAPSE_SERVER_NAME=${SERVER_NAME} \
-  # No need to report stats here
-  SYNAPSE_REPORT_STATS=no \
-  # Set postgres authentication details which will be placed in the homeserver config file
-  POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \
-  # Specify the workers to test with
-  SYNAPSE_WORKER_TYPES="\
-    event_persister, \
-    event_persister, \
-    background_worker, \
-    frontend_proxy, \
-    event_creator, \
-    user_dir, \
-    media_repository, \
-    federation_inbound, \
-    federation_reader, \
-    federation_sender, \
-    synchrotron, \
-    appservice, \
-    pusher" \
-  # Run the script that writes the necessary config files and starts supervisord, which in turn
-  # starts everything else
-  /configure_workers_and_start.py
+ENTRYPOINT /start-complement-synapse-workers.sh
 
+# Update the healthcheck to have a shorter check interval
 HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
     CMD /bin/sh /healthcheck.sh
diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh
new file mode 100755
index 000000000..2c1e05bd6
--- /dev/null
+++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement
+
+set -e
+
+function log {
+    d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
+    echo "$d $@"
+}
+
+# Replace the server name in the caddy config
+sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json
+
+log "starting postgres"
+pg_ctlcluster 13 main start
+
+log "starting caddy"
+/root/caddy start --config /root/caddy.json
+
+# Set the server name of the homeserver
+export SYNAPSE_SERVER_NAME=${SERVER_NAME}
+
+# No need to report stats here
+export SYNAPSE_REPORT_STATS=no
+
+# Set postgres authentication details which will be placed in the homeserver config file
+export POSTGRES_PASSWORD=somesecret
+export POSTGRES_USER=postgres
+export POSTGRES_HOST=localhost
+
+# Specify the workers to test with
+export SYNAPSE_WORKER_TYPES="\
+    event_persister, \
+    event_persister, \
+    background_worker, \
+    frontend_proxy, \
+    event_creator, \
+    user_dir, \
+    media_repository, \
+    federation_inbound, \
+    federation_reader, \
+    federation_sender, \
+    synchrotron, \
+    appservice, \
+    pusher"
+
+# Run the script that writes the necessary config files and starts supervisord, which in turn
+# starts everything else
+exec /configure_workers_and_start.py
diff --git a/docker/complement/conf/homeserver.yaml b/docker/complement/conf/homeserver.yaml
index c9d6a3124..174f87f52 100644
--- a/docker/complement/conf/homeserver.yaml
+++ b/docker/complement/conf/homeserver.yaml
@@ -103,8 +103,10 @@ experimental_features:
   spaces_enabled: true
   # Enable history backfilling support
   msc2716_enabled: true
-  # server-side support for partial state in /send_join
+  # server-side support for partial state in /send_join responses
   msc3706_enabled: true
+  # client-side support for partial state in /send_join responses
+  faster_joins: true
   # Enable jump to date endpoint
   msc3030_enabled: true
 
diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2
index 0de2c6143..408ef7278 100644
--- a/docker/conf-workers/supervisord.conf.j2
+++ b/docker/conf-workers/supervisord.conf.j2
@@ -5,6 +5,9 @@
 nodaemon=true
 user=root
 
+[include]
+files = /etc/supervisor/conf.d/*.conf
+
 [program:nginx]
 command=/usr/sbin/nginx -g "daemon off;"
 priority=500
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index 3e91024e8..3bda6c300 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -29,7 +29,7 @@
 import os
 import subprocess
 import sys
-from typing import Any, Dict, Set
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set
 
 import jinja2
 import yaml
@@ -201,7 +201,7 @@ upstream {upstream_worker_type} {{
 
 
 # Utility functions
-def log(txt: str):
+def log(txt: str) -> None:
     """Log something to the stdout.
 
     Args:
@@ -210,7 +210,7 @@ def log(txt: str):
     print(txt)
 
 
-def error(txt: str):
+def error(txt: str) -> NoReturn:
     """Log something and exit with an error code.
 
     Args:
@@ -220,7 +220,7 @@ def error(txt: str):
     sys.exit(2)
 
 
-def convert(src: str, dst: str, **template_vars):
+def convert(src: str, dst: str, **template_vars: object) -> None:
     """Generate a file from a template
 
     Args:
@@ -290,7 +290,7 @@ def add_sharding_to_shared_config(
         shared_config.setdefault("media_instance_running_background_jobs", worker_name)
 
 
-def generate_base_homeserver_config():
+def generate_base_homeserver_config() -> None:
     """Starts Synapse and generates a basic homeserver config, which will later be
     modified for worker support.
 
@@ -302,12 +302,14 @@ def generate_base_homeserver_config():
     subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
 
 
-def generate_worker_files(environ, config_path: str, data_dir: str):
+def generate_worker_files(
+    environ: Mapping[str, str], config_path: str, data_dir: str
+) -> None:
     """Read the desired list of workers from environment variables and generate
     shared homeserver, nginx and supervisord configs.
 
     Args:
-        environ: _Environ[str]
+        environ: os.environ instance.
         config_path: The location of the generated Synapse main worker config file.
         data_dir: The location of the synapse data directory. Where log and
             user-facing config files live.
@@ -341,7 +343,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # base shared worker jinja2 template.
     #
     # This config file will be passed to all workers, included Synapse's main process.
-    shared_config = {"listeners": listeners}
+    shared_config: Dict[str, Any] = {"listeners": listeners}
 
     # The supervisord config. The contents of which will be inserted into the
     # base supervisord jinja2 template.
@@ -369,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     nginx_locations = {}
 
     # Read the desired worker configuration from the environment
-    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types is None:
+    worker_types_env = environ.get("SYNAPSE_WORKER_TYPES")
+    if worker_types_env is None:
         # No workers, just the main process
         worker_types = []
     else:
         # Split type names by comma
-        worker_types = worker_types.split(",")
+        worker_types = worker_types_env.split(",")
 
     # Create the worker configuration directory if it doesn't already exist
     os.makedirs("/conf/workers", exist_ok=True)
@@ -446,21 +448,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
         # Write out the worker's logging config file
 
-        # Check whether we should write worker logs to disk, in addition to the console
-        extra_log_template_args = {}
-        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-                dir=data_dir, name=worker_name
-            )
-
-        # Render and write the file
-        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
-        convert(
-            "/conf/log.config",
-            log_config_filepath,
-            worker_name=worker_name,
-            **extra_log_template_args,
-        )
+        log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)
 
         # Then a worker config file
         convert(
@@ -496,6 +484,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
     # Finally, we'll write out the config files.
 
+    # log config for the master process
+    master_log_config = generate_worker_log_config(environ, "master", data_dir)
+    shared_config["log_config"] = master_log_config
+
     # Shared homeserver config
     convert(
         "/conf/shared.yaml.j2",
@@ -512,9 +504,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     )
 
     # Supervisord config
+    os.makedirs("/etc/supervisor", exist_ok=True)
     convert(
         "/conf/supervisord.conf.j2",
-        "/etc/supervisor/conf.d/supervisord.conf",
+        "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         worker_config=supervisord_config,
     )
@@ -532,15 +525,31 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
         os.mkdir(log_dir)
 
 
-def start_supervisord():
-    """Starts up supervisord which then starts and monitors all other necessary processes
+def generate_worker_log_config(
+    environ: Mapping[str, str], worker_name: str, data_dir: str
+) -> str:
+    """Generate a log.config file for the given worker.
 
-    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    Returns: the path to the generated file
     """
-    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+    # Check whether we should write worker logs to disk, in addition to the console
+    extra_log_template_args = {}
+    if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+        extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+            dir=data_dir, name=worker_name
+        )
+    # Render and write the file
+    log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+    convert(
+        "/conf/log.config",
+        log_config_filepath,
+        worker_name=worker_name,
+        **extra_log_template_args,
+    )
+    return log_config_filepath
 
 
-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
     config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
     config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
     data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
@@ -567,7 +576,13 @@ def main(args, environ):
 
     # Start supervisord, which will start Synapse, all of the configured worker
     # processes, redis, nginx etc. according to the config we created above.
-    start_supervisord()
+    log("Starting supervisord")
+    os.execl(
+        "/usr/local/bin/supervisord",
+        "supervisord",
+        "-c",
+        "/etc/supervisor/supervisord.conf",
+    )
 
 
 if __name__ == "__main__":
diff --git a/docker/start.py b/docker/start.py
index ac62bbc8b..4ac8f0347 100755
--- a/docker/start.py
+++ b/docker/start.py
@@ -6,27 +6,28 @@ import os
 import platform
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional
 
 import jinja2
 
 
 # Utility functions
-def log(txt):
+def log(txt: str) -> None:
     print(txt, file=sys.stderr)
 
 
-def error(txt):
+def error(txt: str) -> NoReturn:
     log(txt)
     sys.exit(2)
 
 
-def convert(src, dst, environ):
+def convert(src: str, dst: str, environ: Mapping[str, object]) -> None:
     """Generate a file from a template
 
     Args:
-        src (str): path to input file
-        dst (str): path to file to write
-        environ (dict): environment dictionary, for replacement mappings.
+        src: path to input file
+        dst: path to file to write
+        environ: environment dictionary, for replacement mappings.
     """
     with open(src) as infile:
         template = infile.read()
@@ -35,25 +36,30 @@ def convert(src, dst, environ):
         outfile.write(rendered)
 
 
-def generate_config_from_template(config_dir, config_path, environ, ownership):
+def generate_config_from_template(
+    config_dir: str,
+    config_path: str,
+    os_environ: Mapping[str, str],
+    ownership: Optional[str],
+) -> None:
     """Generate a homeserver.yaml from environment variables
 
     Args:
-        config_dir (str): where to put generated config files
-        config_path (str): where to put the main config file
-        environ (dict): environment dictionary
-        ownership (str|None): "<user>:<group>" string which will be used to set
+        config_dir: where to put generated config files
+        config_path: where to put the main config file
+        os_environ: environment mapping
+        ownership: "<user>:<group>" string which will be used to set
             ownership of the generated configs. If None, ownership will not change.
     """
     for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
-        if v not in environ:
+        if v not in os_environ:
             error(
                 "Environment variable '%s' is mandatory when generating a config file."
                 % (v,)
             )
 
     # populate some params from data files (if they exist, else create new ones)
-    environ = environ.copy()
+    environ: Dict[str, Any] = dict(os_environ)
     secrets = {
         "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
         "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
@@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
     subprocess.check_output(args)
 
 
-def run_generate_config(environ, ownership):
+def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None:
     """Run synapse with a --generate-config param to generate a template config file
 
     Args:
-        environ (dict): env var dict
-        ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change.
+        environ: env vars from `os.enrivon`.
+        ownership: "userid:groupid" arg for chmod. If None, ownership will not change.
 
     Never returns.
     """
@@ -178,7 +184,7 @@ def run_generate_config(environ, ownership):
     os.execv(sys.executable, args)
 
 
-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
     mode = args[1] if len(args) > 1 else "run"
 
     # if we were given an explicit user to switch to, do so
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index 6aa48e191..65570cefb 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -17,6 +17,7 @@
 # Usage
   - [Federation](federate.md)
   - [Configuration](usage/configuration/README.md)
+    - [Configuration Manual](usage/configuration/config_documentation.md) 
     - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
     - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
     - [Structured Logging](structured_logging.md)
diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md
index 4076fcab6..c8794299e 100644
--- a/docs/admin_api/user_admin_api.md
+++ b/docs/admin_api/user_admin_api.md
@@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users/<user_id>/delete_devices
   "devices": [
     "QBUAZIFURK",
     "AUIECTSRND"
-  ],
+  ]
 }
 ```
 
diff --git a/docs/code_style.md b/docs/code_style.md
index ebda6dcc8..db7edcd76 100644
--- a/docs/code_style.md
+++ b/docs/code_style.md
@@ -6,60 +6,36 @@ The Synapse codebase uses a number of code formatting tools in order to
 quickly and automatically check for formatting (and sometimes logical)
 errors in code.
 
-The necessary tools are detailed below.
+The necessary tools are:
 
-First install them with:
+- [black](https://black.readthedocs.io/en/stable/), a source code formatter;
+- [isort](https://pycqa.github.io/isort/), which organises each file's imports;
+- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and
+- [mypy](https://mypy.readthedocs.io/en/stable/), a type checker.
+
+Install them with:
 
 ```sh
 pip install -e ".[lint,mypy]"
 ```
 
--   **black**
+The easiest way to run the lints is to invoke the linter script as follows.
 
-    The Synapse codebase uses [black](https://pypi.org/project/black/)
-    as an opinionated code formatter, ensuring all comitted code is
-    properly formatted.
-
-    Have `black` auto-format your code (it shouldn't change any
-    functionality) with:
-
-    ```sh
-    black .
-    ```
-
--   **flake8**
-
-    `flake8` is a code checking tool. We require code to pass `flake8`
-    before being merged into the codebase.
-
-    Check all application and test code with:
-
-    ```sh
-    flake8 .
-    ```
-
--   **isort**
-
-    `isort` ensures imports are nicely formatted, and can suggest and
-    auto-fix issues such as double-importing.
-
-    Auto-fix imports with:
-
-    ```sh
-    isort .
-    ```
+```sh
+scripts-dev/lint.sh
+```
 
 It's worth noting that modern IDEs and text editors can run these tools
 automatically on save. It may be worth looking into whether this
 functionality is supported in your editor for a more convenient
-development workflow. It is not, however, recommended to run `flake8` on
-save as it takes a while and is very resource intensive.
+development workflow. It is not, however, recommended to run `flake8` or `mypy`
+on save as they take a while and can be very resource intensive.
 
 ## General rules
 
 -   **Naming**:
-    -   Use camel case for class and type names
-    -   Use underscores for functions and variables.
+    -   Use `CamelCase` for class and type names
+    -   Use underscores for `function_names` and `variable_names`.
 -   **Docstrings**: should follow the [google code
     style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings).
     See the
diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md
index 0d9cf6019..3b5c77401 100644
--- a/docs/development/contributing_guide.md
+++ b/docs/development/contributing_guide.md
@@ -48,19 +48,28 @@ can find many good git tutorials on the web.
 
 # 4. Install the dependencies
 
-Once you have installed Python 3 and added the source, please open a terminal and
-setup a *virtualenv*, as follows:
+Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies
+and development environment. Once you have installed Python 3 and added the
+source, you should install `poetry`.
+Of their installation methods, we recommend
+[installing `poetry` using `pipx`](https://python-poetry.org/docs/#installing-with-pipx),
+
+```shell
+pip install --user pipx
+pipx install poetry
+```
+
+but see poetry's [installation instructions](https://python-poetry.org/docs/#installation)
+for other installation methods.
+
+Next, open a terminal and install dependencies as follows:
 
 ```sh
 cd path/where/you/have/cloned/the/repository
-python3 -m venv ./env
-source ./env/bin/activate
-pip install wheel
-pip install -e ".[all,dev]"
-pip install tox
+poetry install --extras all
 ```
 
-This will install the developer dependencies for the project.
+This will install the runtime and developer dependencies for the project.
 
 
 # 5. Get in touch.
@@ -117,11 +126,10 @@ The linters look at your code and do two things:
 - ensure that your code follows the coding style adopted by the project;
 - catch a number of errors in your code.
 
-The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies).
+The linter takes no time at all to run as soon as you've [downloaded the dependencies](#4-install-the-dependencies).
 
 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh
+poetry run ./scripts-dev/lint.sh
 ```
 
 Note that this script *will modify your files* to fix styling errors.
@@ -131,15 +139,13 @@ If you wish to restrict the linters to only the files changed since the last com
 (much faster!), you can instead run:
 
 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh -d
+poetry run ./scripts-dev/lint.sh -d
 ```
 
 Or if you know exactly which files you wish to lint, you can instead run:
 
 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
+poetry run ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
 ## Run the unit tests (Twisted trial).
@@ -148,16 +154,14 @@ The unit tests run parts of Synapse, including your changes, to see if anything
 was broken. They are slower than the linters but will typically catch more errors.
 
 ```sh
-source ./env/bin/activate
-trial tests
+poetry run trial tests
 ```
 
 If you wish to only run *some* unit tests, you may specify
 another module instead of `tests` - or a test class or a method:
 
 ```sh
-source ./env/bin/activate
-trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
+poetry run trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
 ```
 
 If your tests fail, you may wish to look at the logs (the default log level is `ERROR`):
@@ -169,7 +173,7 @@ less _trial_temp/test.log
 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
 
 ```sh
-SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
+SYNAPSE_TEST_LOG_LEVEL=DEBUG poetry run trial tests
 ```
 
 By default, tests will use an in-memory SQLite database for test data. For additional
@@ -180,7 +184,7 @@ database state to be stored in a file named `test.db` under the trial process'
 working directory. Typically, this ends up being `_trial_temp/test.db`. For example:
 
 ```sh
-SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests
+SYNAPSE_TEST_PERSIST_SQLITE_DB=1 poetry run trial tests
 ```
 
 The database file can then be inspected with:
diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md
new file mode 100644
index 000000000..8ef7d357d
--- /dev/null
+++ b/docs/development/dependencies.md
@@ -0,0 +1,239 @@
+# Managing dependencies with Poetry
+
+This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/).
+
+# Background
+
+Synapse uses a variety of third-party Python packages to function as a homeserver.
+Some of these are direct dependencies, listed in `pyproject.toml` under the
+`[tool.poetry.dependencies]` section. The rest are transitive dependencies (the
+things that our direct dependencies themselves depend on, and so on recursively.)
+
+We maintain a locked list of all our dependencies (transitive included) so that
+we can track exactly which version of each dependency appears in a given release.
+See [here](https://github.com/matrix-org/synapse/issues/11537#issue-1074469665)
+for discussion of why we wanted this for Synapse. We chose to use
+[`poetry`](https://python-poetry.org/) to manage this locked list; see
+[this comment](https://github.com/matrix-org/synapse/issues/11537#issuecomment-1015975819)
+for the reasoning.
+
+The locked dependencies get included in our "self-contained" releases: namely,
+our docker images and our debian packages. We also use the locked dependencies
+in development and our continuous integration.
+
+Separately, our "broad" dependencies—the version ranges specified in
+`pyproject.toml`—are included as metadata in our "sdists" and "wheels" [uploaded
+to PyPI](https://pypi.org/project/matrix-synapse). Installing from PyPI or from
+the Synapse source tree directly will _not_ use the locked dependencies; instead,
+they'll pull in the latest version of each package available at install time.
+
+## Example dependency
+
+An example may help. We have a broad dependency on
+[`phonenumbers`](https://pypi.org/project/phonenumbers/), as declared in
+this snippet from pyproject.toml [as of Synapse 1.57](
+https://github.com/matrix-org/synapse/blob/release-v1.57/pyproject.toml#L133
+):
+
+```toml
+[tool.poetry.dependencies]
+# ...
+phonenumbers = ">=8.2.0"
+```
+
+In our lockfile this is
+[pinned]( https://github.com/matrix-org/synapse/blob/dfc7646504cef3e4ff396c36089e1c6f1b1634de/poetry.lock#L679-L685)
+to version 8.12.44, even though
+[newer versions are available](https://pypi.org/project/phonenumbers/#history).
+
+```toml
+[[package]]
+name = "phonenumbers"
+version = "8.12.44"
+description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
+category = "main"
+optional = false
+python-versions = "*"
+```
+
+The lockfile also includes a
+[cryptographic checksum](https://github.com/matrix-org/synapse/blob/release-v1.57/poetry.lock#L2178-L2181)
+of the sdists and wheels provided for this version of `phonenumbers`.
+
+```toml
+[metadata.files]
+# ...
+phonenumbers = [
+    {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"},
+    {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"},
+]
+```
+
+We can see this pinned version inside the docker image for that release:
+
+```
+$ docker pull matrixdotorg/synapse:v1.57.0
+...
+$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers
+Name: phonenumbers
+Version: 8.12.44
+Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers.
+Home-page: https://github.com/daviddrysdale/python-phonenumbers
+Author: David Drysdale
+Author-email: dmd@lurklurk.org
+License: Apache License 2.0
+Location: /usr/local/lib/python3.9/site-packages
+Requires:
+Required-by: matrix-synapse
+```
+
+Whereas the wheel metadata just contains the broad dependencies:
+
+```
+$ cd /tmp
+$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl
+...
+$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers
+Requires-Dist: phonenumbers (>=8.2.0)
+```
+
+# Tooling recommendation: direnv
+
+[`direnv`](https://direnv.net/) is a tool for activating environments in your
+shell inside a given directory. Its support for poetry is unofficial (a
+community wiki recipe only), but works solidly in our experience. We thoroughly
+recommend it for daily use. To use it:
+
+1. [Install `direnv`](https://direnv.net/docs/installation.html) - it's likely
+   packaged for your system already.
+2. Teach direnv about poetry. The [shell config here](https://github.com/direnv/direnv/wiki/Python#poetry)
+   needs to be added to `~/.config/direnv/direnvrc` (or more generally `$XDG_CONFIG_HOME/direnv/direnvrc`).
+3. Mark the synapse checkout as a poetry project: `echo layout poetry > .envrc`.
+4. Convince yourself that you trust this `.envrc` configuration and project.
+   Then formally confirm this to `direnv` by running `direnv allow`.
+
+Then whenever you navigate to the synapse checkout, you should be able to run
+e.g. `mypy` instead of `poetry run mypy`; `python` instead of
+`poetry run python`; and your shell commands will automatically run in the
+context of poetry's venv, without having to run `poetry shell` beforehand.
+
+
+# How do I...
+
+## ...reset my venv to the locked environment?
+
+```shell
+poetry install --extras all --remove-untracked
+```
+
+## ...run a command in the `poetry` virtualenv?
+
+Use `poetry run cmd args` when you need the python virtualenv context.
+To avoid typing `poetry run` all the time, you can run  `poetry shell`
+to start a new shell in the poetry virtualenv context. Within `poetry shell`,
+`python`, `pip`, `mypy`, `trial`, etc. are all run inside the project virtualenv
+and isolated from the rest o the system.
+
+Roughly speaking, the translation from a traditional virtualenv is:
+- `env/bin/activate` -> `poetry shell`, and
+- `deactivate` -> close the terminal (Ctrl-D, `exit`, etc.)
+
+See also the direnv recommendation above, which makes `poetry run` and
+`poetry shell` unnecessary.
+
+
+## ...inspect the `poetry` virtualenv?
+
+Some suggestions:
+
+```shell
+# Current env only
+poetry env info
+# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7,
+# and another for Python 3.10.
+poetry env list --full-path
+poetry run pip list
+```
+
+Note that `poetry show` describes the abstract *lock file* rather than your
+on-disk environment. With that said, `poetry show --tree` can sometimes be
+useful.
+
+
+## ...add a new dependency?
+
+Either:
+- manually update `pyproject.toml`; then `poetry lock --no-update`; or else
+- `poetry add packagename`. See `poetry add --help`; note the `--dev`,
+  `--extras` and `--optional` flags in particular.
+  - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39).
+
+Include the updated `pyproject.toml` and `poetry.lock` files in your commit.
+
+## ...remove a dependency?
+
+This is not done often and is untested, but
+
+```shell
+poetry remove packagename
+```
+
+ought to do the trick. Alternatively, manually update `pyproject.toml` and
+`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock`
+files in your commit.
+
+## ...update the version range for an existing dependency?
+
+Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`.
+Include the updated `pyproject.toml` and `poetry.lock` in your commit.
+
+## ...update a dependency in the locked environment?
+
+Use
+
+```shell
+poetry update packagename
+```
+
+to use the latest version of `packagename` in the locked environment, without
+affecting the broad dependencies listed in the wheel.
+
+There doesn't seem to be a way to do this whilst locking a _specific_ version of
+`packagename`. We can workaround this (crudely) as follows:
+
+```shell
+poetry add packagename==1.2.3
+# This should update pyproject.lock.
+
+# Now undo the changes to pyproject.toml. For example
+# git restore pyproject.toml
+
+# Get poetry to recompute the content-hash of pyproject.toml without changing
+# the locked package versions.
+poetry lock --no-update
+```
+
+Either way, include the updated `poetry.lock` file in your commit.
+
+## ...export a `requirements.txt` file?
+
+```shell
+poetry export --extras all
+```
+
+Be wary of bugs in `poetry export` and `pip install -r requirements.txt`.
+
+Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may
+be required.
+
+## ...build a test wheel?
+
+I usually use
+
+```shell
+poetry run pip install build && poetry run python -m build
+```
+
+because [`build`](https://github.com/pypa/build) is a standardish tool which
+doesn't require poetry. (It's what we use in CI too). However, you could try
+`poetry build` too.
diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md
index 5a0c84795..69caa8a73 100644
--- a/docs/reverse_proxy.md
+++ b/docs/reverse_proxy.md
@@ -206,6 +206,28 @@ backend matrix
   server matrix 127.0.0.1:8008
 ```
 
+
+[Delegation](delegate.md) example:
+```
+frontend https
+  acl matrix-well-known-client-path path /.well-known/matrix/client
+  acl matrix-well-known-server-path path /.well-known/matrix/server
+  use_backend matrix-well-known-client if matrix-well-known-client-path
+  use_backend matrix-well-known-server if matrix-well-known-server-path
+ 
+backend matrix-well-known-client
+  http-after-response set-header Access-Control-Allow-Origin "*"
+  http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+  http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+  http-request return status 200 content-type application/json string '{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}'
+
+backend matrix-well-known-server
+  http-after-response set-header Access-Control-Allow-Origin "*"
+  http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+  http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+  http-request return status 200 content-type application/json string '{"m.server":"matrix.example.com:443"}'
+```
+
 ### Relayd
 
 ```
diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md
index b160d9352..d51650108 100644
--- a/docs/systemd-with-workers/README.md
+++ b/docs/systemd-with-workers/README.md
@@ -10,15 +10,15 @@ See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/
 for the systemd unit files.
 
 The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/)
-contains an example configuration for the `federation_reader` worker.
+contains an example configuration for the `generic_worker` worker.
 
 ## Synapse configuration files
 
 See [the worker documentation](../workers.md) for information on how to set up the
 configuration files and reverse-proxy correctly.
-Below is a sample `federation_reader` worker configuration file.
+Below is a sample `generic_worker` worker configuration file.
 ```yaml
-{{#include workers/federation_reader.yaml}}
+{{#include workers/generic_worker.yaml}}
 ```
 
 Systemd manages daemonization itself, so ensure that none of the configuration
@@ -61,9 +61,9 @@ systemctl stop matrix-synapse.target
 # Restart the master alone
 systemctl start matrix-synapse.service
 
-# Restart a specific worker (eg. federation_reader); the master is
+# Restart a specific worker (eg. generic_worker); the master is
 # unaffected by this.
-systemctl restart matrix-synapse-worker@federation_reader.service
+systemctl restart matrix-synapse-worker@generic_worker.service
 
 # Add a new worker (assuming all configs are set up already)
 systemctl enable matrix-synapse-worker@federation_writer.service
diff --git a/docs/systemd-with-workers/workers/background_worker.yaml b/docs/systemd-with-workers/workers/background_worker.yaml
new file mode 100644
index 000000000..9fbfbda7d
--- /dev/null
+++ b/docs/systemd-with-workers/workers/background_worker.yaml
@@ -0,0 +1,8 @@
+worker_app: synapse.app.generic_worker
+worker_name: background_worker
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_log_config: /etc/matrix-synapse/background-worker-log.yaml
diff --git a/docs/systemd-with-workers/workers/event_persister.yaml b/docs/systemd-with-workers/workers/event_persister.yaml
new file mode 100644
index 000000000..9bc6997ba
--- /dev/null
+++ b/docs/systemd-with-workers/workers/event_persister.yaml
@@ -0,0 +1,23 @@
+worker_app: synapse.app.generic_worker
+worker_name: event_persister1 
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: 8034
+    resources:
+      - names: [replication]
+
+  # Enable listener if this stream writer handles endpoints for the `typing` or
+  # `to_device` streams. Uses a different port to the `replication` listener to
+  # avoid exposing the `replication` listener publicly.
+  #
+  #- type: http
+  #  port: 8035
+  #  resources:
+  #    - names: [client]
+
+worker_log_config: /etc/matrix-synapse/event-persister-log.yaml
diff --git a/docs/systemd-with-workers/workers/federation_reader.yaml b/docs/systemd-with-workers/workers/federation_reader.yaml
deleted file mode 100644
index 13e69e62c..000000000
--- a/docs/systemd-with-workers/workers/federation_reader.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-worker_app: synapse.app.federation_reader
-worker_name: federation_reader1
-
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-    - type: http
-      port: 8011
-      resources:
-          - names: [federation]
-
-worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml
diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml
new file mode 100644
index 000000000..a82f9c161
--- /dev/null
+++ b/docs/systemd-with-workers/workers/generic_worker.yaml
@@ -0,0 +1,14 @@
+worker_app: synapse.app.generic_worker
+worker_name: generic_worker1
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: 8083
+    resources:
+      - names: [client, federation]
+
+worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
diff --git a/docs/turn-howto.md b/docs/turn-howto.md
index 3a2cd04e3..37a311ad9 100644
--- a/docs/turn-howto.md
+++ b/docs/turn-howto.md
@@ -302,14 +302,14 @@ Here are a few things to try:
 
    (Understanding the output is beyond the scope of this document!)
 
- * You can test your Matrix homeserver TURN setup with https://test.voip.librepush.net/.
+ * You can test your Matrix homeserver TURN setup with <https://test.voip.librepush.net/>.
    Note that this test is not fully reliable yet, so don't be discouraged if
    the test fails.
    [Here](https://github.com/matrix-org/voip-tester) is the github repo of the
    source of the tester, where you can file bug reports.
 
  * There is a WebRTC test tool at
-   https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To
+   <https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/>. To
    use it, you will need a username/password for your TURN server. You can
    either:
 
diff --git a/docs/upgrade.md b/docs/upgrade.md
index 023872490..3a8aeb039 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -19,32 +19,36 @@ this document.
     packages](setup/installation.md#prebuilt-packages), you will need to follow the
     normal process for upgrading those packages.
 
+-   If Synapse was installed using pip then upgrade to the latest
+    version by running:
+
+    ```bash
+    pip install --upgrade matrix-synapse
+    ```
+
 -   If Synapse was installed from source, then:
 
-    1.  Activate the virtualenv before upgrading. For example, if
-        Synapse is installed in a virtualenv in `~/synapse/env` then
+    1.  Obtain the latest version of the source code. Git users can run
+        `git pull` to do this.
+
+    2.  If you're running Synapse in a virtualenv, make sure to activate it before
+        upgrading. For example, if Synapse is installed in a virtualenv in `~/synapse/env` then
         run:
 
         ```bash
         source ~/synapse/env/bin/activate
-        ```
-
-    2.  If Synapse was installed using pip then upgrade to the latest
-        version by running:
-
-        ```bash
-        pip install --upgrade matrix-synapse
-        ```
-
-        If Synapse was installed using git then upgrade to the latest
-        version by running:
-
-        ```bash
-        git pull
         pip install --upgrade .
         ```
+        Include any relevant extras between square brackets, e.g. `pip install --upgrade ".[postgres,oidc]"`.
 
-    3.  Restart Synapse:
+    3.  If you're using `poetry` to manage a Synapse installation, run:
+        ```bash
+        poetry install
+        ```
+        Include any relevant extras with `--extras`, e.g. `poetry install --extras postgres --extras oidc`.
+        It's probably easiest to run `poetry install --extras all`.
+
+    4.  Restart Synapse:
 
         ```bash
         synctl restart
@@ -85,6 +89,13 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
+# Upgrading to v1.58.0
+
+## Groups/communities feature has been disabled by default
+
+The non-standard groups/communities feature in Synapse has been disabled by default
+and will be removed in Synapse v1.61.0.
+
 # Upgrading to v1.57.0
 
 ## Changes to database schema for application services
diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md
new file mode 100644
index 000000000..968b0fbfa
--- /dev/null
+++ b/docs/usage/configuration/config_documentation.md
@@ -0,0 +1,3454 @@
+# Configuring Synapse
+
+This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified 
+through the many configuration settings documented here — each config option is explained, 
+including what the default is, how to change the default and what sort of behaviour the setting governs.
+Also included is an example configuration for each setting. If you don't want to spend a lot of time 
+thinking about options, the config as generated sets sensible defaults for all values. Do note however that the
+database defaults to SQLite, which is not recommended for production usage. You can read more on this subject 
+[here](../../setup/installation.md#using-postgresql).
+
+## Config Conventions
+
+Configuration options that take a time period can be set using a number
+followed by a letter. Letters have the following meanings:
+
+* `s` = second
+* `m` = minute
+* `h` = hour
+* `d` = day
+* `w` = week
+* `y` = year
+
+For example, setting `redaction_retention_period: 5m` would remove redacted
+messages from the database after 5 minutes, rather than 5 months.
+
+### YAML 
+The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules
+apply if you want your config file to be read properly. A few helpful things to know:
+* `#` before any option in the config will comment out that setting and either a default (if available) will 
+   be applied or Synapse will ignore the setting. Thus, in example #1 below, the setting will be read and
+   applied, but in example #2 the setting will not be read and a default will be applied.  
+
+   Example #1:
+   ```yaml
+   pid_file: DATADIR/homeserver.pid
+   ```
+   Example #2:
+   ```yaml
+   #pid_file: DATADIR/homeserver.pid
+   ```
+* Indentation matters! The indentation before a setting
+  will determine whether a given setting is read as part of another
+  setting, or considered on its own. Thus, in example #1, the `enabled` setting
+  is read as a sub-option of the `presence` setting, and will be properly applied.
+  
+  However, the lack of indentation before the `enabled` setting in example #2 means
+  that when reading the config, Synapse will consider both `presence` and `enabled` as
+  different settings. In this case, `presence` has no value, and thus a default applied, and `enabled`
+  is an option that Synapse doesn't recognize and thus ignores.
+  
+  Example #1: 
+  ```yaml
+  presence:
+    enabled: false
+  ```
+  Example #2:
+  ```yaml
+  presence:
+  enabled: false
+  ```
+  In this manual, all top-level settings (ones with no indentation) are identified 
+  at the beginning of their section (i.e. "Config option: `example_setting`") and 
+  the sub-options, if any, are identified and listed in the body of the section. 
+  In addition, each setting has an example of its usage, with the proper indentation
+  shown. 
+
+## Contents
+[Modules](#modules)
+
+[Server](#server)
+
+[Homeserver Blocking](#homeserver-blocking)
+
+[TLS](#tls)
+
+[Federation](#federation)
+
+[Caching](#caching)
+
+[Database](#database)
+
+[Logging](#logging)
+
+[Ratelimiting](#ratelimiting)
+
+[Media Store](#media-store)
+
+[Captcha](#captcha)
+
+[TURN](#turn)
+
+[Registration](#registration)
+
+[API Configuration](#api-configuration)
+
+[Signing Keys](#signing-keys)
+
+[Single Sign On Integration](#single-sign-on-integration)
+
+[Push](#push)
+
+[Rooms](#rooms)
+
+[Opentracing](#opentracing)
+
+[Workers](#workers)
+
+[Background Updates](#background-updates)
+
+## Modules
+
+Server admins can expand Synapse's functionality with external modules.
+
+See [here](../../modules/index.md) for more
+documentation on how to configure or create custom modules for Synapse.
+
+
+---
+Config option: `modules`
+
+Use the `module` sub-option to add modules under this option to extend functionality. 
+The `module` setting then has a sub-option, `config`, which can be used to define some configuration
+for the `module`.
+
+Defaults to none.
+
+Example configuration:
+```yaml
+modules:
+  - module: my_super_module.MySuperClass
+    config:
+      do_thing: true
+  - module: my_other_super_module.SomeClass
+    config: {}
+```
+---
+## Server ##
+
+Define your homeserver name and other base options.
+
+---
+Config option: `server_name`
+
+This sets the public-facing domain of the server.
+
+The `server_name` name will appear at the end of usernames and room addresses
+created on your server. For example if the `server_name` was example.com,
+usernames on your server would be in the format `@user:example.com`
+
+In most cases you should avoid using a matrix specific subdomain such as
+matrix.example.com or synapse.example.com as the `server_name` for the same
+reasons you wouldn't use user@email.example.com as your email address.
+See [here](../../delegate.md)
+for information on how to host Synapse on a subdomain while preserving
+a clean `server_name`.
+
+The `server_name` cannot be changed later so it is important to
+configure this correctly before you start Synapse. It should be all
+lowercase and may contain an explicit port.
+
+There is no default for this option. 
+ 
+Example configuration #1:
+```yaml
+server_name: matrix.org 
+```
+Example configuration #2:
+```yaml
+server_name: localhost:8080
+```
+---
+Config option: `pid_file`
+
+When running Synapse as a daemon, the file to store the pid in. Defaults to none.
+
+Example configuration:
+```yaml
+pid_file: DATADIR/homeserver.pid
+```
+---
+Config option: `web_client_location`
+
+The absolute URL to the web client which `/` will redirect to. Defaults to none. 
+
+Example configuration:
+```yaml
+web_client_location: https://riot.example.com/
+```
+---
+Config option: `public_baseurl`
+
+The public-facing base URL that clients use to access this Homeserver (not
+including _matrix/...). This is the same URL a user might enter into the
+'Custom Homeserver URL' field on their client. If you use Synapse with a
+reverse proxy, this should be the URL to reach Synapse via the proxy.
+Otherwise, it should be the URL to reach Synapse's client HTTP listener (see
+'listeners' below).
+
+Defaults to `https://<server_name>/`.
+
+Example configuration:
+```yaml
+public_baseurl: https://example.com/
+```
+---
+Config option: `serve_server_wellknown`
+
+By default, other servers will try to reach our server on port 8448, which can
+be inconvenient in some environments.
+
+Provided `https://<server_name>/` on port 443 is routed to Synapse, this
+option configures Synapse to serve a file at `https://<server_name>/.well-known/matrix/server`. 
+This will tell other servers to send traffic to port 443 instead.
+
+This option currently defaults to false.
+
+See https://matrix-org.github.io/synapse/latest/delegate.html for more
+information.
+
+Example configuration:
+```yaml
+serve_server_wellknown: true
+```
+---
+Config option: `soft_file_limit`
+ 
+Set the soft limit on the number of file descriptors synapse can use.
+Zero is used to indicate synapse should set the soft limit to the hard limit.
+Defaults to 0. 
+
+Example configuration:
+```yaml
+soft_file_limit: 3
+```
+---
+Config option: `presence`
+
+Presence tracking allows users to see the state (e.g online/offline)
+of other local and remote users. Set the `enabled` sub-option to false to  
+disable presence tracking on this homeserver. Defaults to true. 
+This option replaces the previous top-level 'use_presence' option.
+
+Example configuration:
+```yaml
+presence:
+  enabled: false
+```
+---
+Config option: `require_auth_for_profile_requests`
+
+Whether to require authentication to retrieve profile data (avatars, display names) of other 
+users through the client API. Defaults to false. Note that profile data is also available 
+via the federation API, unless `allow_profile_lookup_over_federation` is set to false.
+
+Example configuration:
+```yaml
+require_auth_for_profile_requests: true
+```
+---
+Config option: `limit_profile_requests_to_users_who_share_rooms`
+
+Use this option to require a user to share a room with another user in order
+to retrieve their profile information. Only checked on Client-Server 
+requests. Profile requests from other servers should be checked by the
+requesting server. Defaults to false.
+
+Example configuration: 
+```yaml
+limit_profile_requests_to_users_who_share_rooms: true
+```
+---
+Config option: `include_profile_data_on_invite`
+
+Use this option to prevent a user's profile data from being retrieved and
+displayed in a room until they have joined it. By default, a user's
+profile data is included in an invite event, regardless of the values
+of the above two settings, and whether or not the users share a server.
+Defaults to true.
+
+Example configuration:
+```yaml
+include_profile_data_on_invite: false
+```
+---
+Config option: `allow_public_rooms_without_auth`
+
+If set to true, removes the need for authentication to access the server's
+public rooms directory through the client API, meaning that anyone can
+query the room directory. Defaults to false.
+
+Example configuration:
+```yaml
+allow_public_rooms_without_auth: true
+```
+---
+Config option: `allow_public_rooms_without_auth`
+
+If set to true, allows any other homeserver to fetch the server's public
+rooms directory via federation. Defaults to false.
+
+Example configuration:
+```yaml
+allow_public_rooms_over_federation: true
+```
+---
+Config option: `default_room_version`
+
+The default room version for newly created rooms on this server.
+
+Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions)
+
+For example, for room version 1, `default_room_version` should be set
+to "1". 
+
+Currently defaults to "9".
+
+Example configuration:
+```yaml
+default_room_version: "8"
+```
+---
+Config option: `gc_thresholds`
+
+The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined.
+Defaults to none. 
+
+Example configuration:
+```yaml
+gc_thresholds: [700, 10, 10]
+```
+---
+Config option: `gc_min_interval`
+
+The minimum time in seconds between each GC for a generation, regardless of
+the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` 
+indicates that a second must pass between consecutive generation 0 GCs, etc.
+
+Defaults to `[1s, 10s, 30s]`.
+
+Example configuration:
+```yaml
+gc_min_interval: [0.5s, 30s, 1m]
+```
+---
+Config option: `filter_timeline_limit`
+
+Set the limit on the returned events in the timeline in the get
+and sync operations. Defaults to 100. A value of -1 means no upper limit.
+
+
+Example configuration:
+```yaml
+filter_timeline_limit: 5000
+```
+---
+Config option: `block_non_admin_invites`
+
+Whether room invites to users on this server should be blocked
+(except those sent by local server admins). Defaults to false.
+
+Example configuration:
+```yaml
+block_non_admin_invites: true
+```
+---
+Config option: `enable_search`
+
+If set to false, new messages will not be indexed for searching and users
+will receive errors when searching for messages. Defaults to true.
+
+Example configuration:
+```yaml
+enable_search: false
+```
+---
+Config option: `ip_range_blacklist`
+ 
+This option prevents outgoing requests from being sent to the specified blacklisted IP address
+CIDR ranges. If this option is not specified then it defaults to private IP
+address ranges (see the example below).
+
+The blacklist applies to the outbound requests for federation, identity servers,
+push servers, and for checking key validity for third-party invite events.
+
+(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly
+listed here, since they correspond to unroutable addresses.)
+
+This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0.
+
+Note: The value is ignored when an HTTP proxy is in use.
+
+Example configuration:
+```yaml
+ip_range_blacklist:
+  - '127.0.0.0/8'
+  - '10.0.0.0/8'
+  - '172.16.0.0/12'
+  - '192.168.0.0/16'
+  - '100.64.0.0/10'
+  - '192.0.0.0/24'
+  - '169.254.0.0/16'
+  - '192.88.99.0/24'
+  - '198.18.0.0/15'
+  - '192.0.2.0/24'
+  - '198.51.100.0/24'
+  - '203.0.113.0/24'
+  - '224.0.0.0/4'
+  - '::1/128'
+  - 'fe80::/10'
+  - 'fc00::/7'
+  - '2001:db8::/32'
+  - 'ff00::/8'
+  - 'fec0::/10'
+```
+---
+Config option: `ip_range_whitelist`
+
+List of IP address CIDR ranges that should be allowed for federation,
+identity servers, push servers, and for checking key validity for
+third-party invite events. This is useful for specifying exceptions to
+wide-ranging blacklisted target IP ranges - e.g. for communication with
+a push server only visible in your network.
+
+This whitelist overrides `ip_range_blacklist` and defaults to an empty
+list.
+
+Example configuration:
+```yaml
+ip_range_whitelist:
+   - '192.168.1.1'
+```
+---
+Config option: `listeners`
+
+List of ports that Synapse should listen on, their purpose and their
+configuration.
+
+Sub-options for each listener include:
+
+* `port`: the TCP port to bind to. 
+
+* `bind_addresses`: a list of local addresses to listen on. The default is
+       'all local interfaces'.
+
+* `type`: the type of listener. Normally `http`, but other valid options are:
+    
+   * `manhole`: (see the docs [here](../../manhole.md)),
+
+   * `metrics`: (see the docs [here](../../metrics-howto.md)),
+
+   * `replication`: (see the docs [here](../../workers.md)).
+
+* `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path.
+
+* `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is
+   behind a reverse-proxy.
+
+* `resources`: Only valid for an 'http' listener. A list of resources to host
+   on this port. Sub-options for each resource are:
+
+   * `names`: a list of names of HTTP resources. See below for a list of valid resource names.
+
+   * `compress`: set to true to enable HTTP compression for this resource.
+
+* `additional_resources`: Only valid for an 'http' listener. A map of
+   additional endpoints which should be loaded via dynamic modules.
+
+Valid resource names are:
+
+* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies 'media' and 'static'.
+
+* `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more.
+
+* `federation`: the server-server API (/_matrix/federation). Also implies `media`, `keys`, `openid`
+
+* `keys`: the key discovery API (/_matrix/keys).
+
+* `media`: the media API (/_matrix/media).
+
+* `metrics`: the metrics interface. See [here](../../metrics-howto.md).
+
+* `openid`: OpenID authentication. See [here](../../openid.md).
+
+* `replication`: the HTTP replication API (/_synapse/replication). See [here](../../workers.md).
+
+* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for 'fallback authentication'.)
+
+Example configuration #1:
+```yaml
+listeners:
+  # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
+  #
+  # (Note that you will also need to give Synapse a TLS key and certificate: see the TLS section
+  # below.)
+  #
+  - port: 8448
+    type: http
+    tls: true
+    resources:
+      - names: [client, federation]
+```
+Example configuration #2:
+```yaml
+listeners:
+  # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy
+  # that unwraps TLS.
+  #
+  # If you plan to use a reverse proxy, please see
+  # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
+  #
+  - port: 8008
+    tls: false
+    type: http
+    x_forwarded: true
+    bind_addresses: ['::1', '127.0.0.1']
+
+    resources:
+      - names: [client, federation]
+        compress: false
+
+    # example additional_resources:
+    additional_resources:
+      "/_matrix/my/custom/endpoint":
+        module: my_module.CustomRequestHandler
+        config: {}
+
+  # Turn on the twisted ssh manhole service on localhost on the given
+  # port.
+  - port: 9000
+    bind_addresses: ['::1', '127.0.0.1']
+    type: manhole
+```
+---
+Config option: `manhole_settings`
+
+Connection settings for the manhole. You can find more information
+on the manhole [here](../../manhole.md). Manhole sub-options include:
+* `username` : the username for the manhole. This defaults to 'matrix'.
+* `password`: The password for the manhole. This defaults to 'rabbithole'.
+* `ssh_priv_key_path` and `ssh_pub_key_path`: The private and public SSH key pair used to encrypt the manhole traffic.
+  If these are left unset, then hardcoded and non-secret keys are used,
+  which could allow traffic to be intercepted if sent over a public network.
+
+Example configuration:
+```yaml
+manhole_settings:
+  username: manhole
+  password: mypassword
+  ssh_priv_key_path: CONFDIR/id_rsa
+  ssh_pub_key_path: CONFDIR/id_rsa.pub
+```
+---
+Config option: `dummy_events_threshold`
+
+Forward extremities can build up in a room due to networking delays between
+homeservers. Once this happens in a large room, calculation of the state of
+that room can become quite expensive. To mitigate this, once the number of
+forward extremities reaches a given threshold, Synapse will send an
+`org.matrix.dummy_event` event, which will reduce the forward extremities
+in the room.
+
+This setting defines the threshold (i.e. number of forward extremities in the room) at which dummy events are sent. 
+The default value is 10.
+
+Example configuration:
+```yaml
+dummy_events_threshold: 5
+```
+---
+## Homeserver blocking ##
+Useful options for Synapse admins.
+
+---
+
+Config option: `admin_contact`
+
+How to reach the server admin, used in `ResourceLimitError`. Defaults to none. 
+
+Example configuration:
+```yaml
+admin_contact: 'mailto:admin@server.com'
+```
+---
+Config option: `hs_disabled` and `hs_disabled_message`
+
+Blocks users from connecting to the homeserver and provides a human-readable reason
+why the connection was blocked. Defaults to false. 
+
+Example configuration:
+```yaml
+hs_disabled: true
+hs_disabled_message: 'Reason for why the HS is blocked'
+```
+---
+Config option: `limit_usage_by_mau`
+
+This option disables/enables monthly active user blocking. Used in cases where the admin or 
+server owner wants to limit to the number of monthly active users. When enabled and a limit is 
+reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`.
+Defaults to false. If this is enabled, a value for `max_mau_value` must also be set.
+
+Example configuration:
+```yaml
+limit_usage_by_mau: true 
+```
+---
+Config option: `max_mau_value`
+
+This option sets the hard limit of monthly active users above which the server will start 
+blocking user actions if `limit_usage_by_mau` is enabled. Defaults to 0.  
+
+Example configuration:
+```yaml
+max_mau_value: 50
+```
+---
+Config option: `mau_trial_days`
+
+The option `mau_trial_days` is a means to add a grace period for active users. It
+means that users must be active for the specified number of days before they
+can be considered active and guards against the case where lots of users
+sign up in a short space of time never to return after their initial
+session. Defaults to 0. 
+
+Example configuration:
+```yaml
+mau_trial_days: 5
+```
+---
+Config option: `mau_limit_alerting`
+
+The option `mau_limit_alerting` is a means of limiting client-side alerting
+should the mau limit be reached. This is useful for small instances
+where the admin has 5 mau seats (say) for 5 specific people and no
+interest increasing the mau limit further. Defaults to true, which
+means that alerting is enabled.
+
+Example configuration:
+```yaml
+mau_limit_alerting: false
+```
+---
+Config option: `mau_stats_only`
+
+If enabled, the metrics for the number of monthly active users will
+be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau`
+is true, this is implied to be true. Defaults to false. 
+
+Example configuration:
+```yaml
+mau_stats_only: true
+```
+---
+Config option: `mau_limit_reserved_threepids`
+
+Sometimes the server admin will want to ensure certain accounts are
+never blocked by mau checking. These accounts are specified by this option.
+Defaults to none. Add accounts by specifying the `medium` and `address` of the
+reserved threepid (3rd party identifier).
+
+Example configuration:
+```yaml
+mau_limit_reserved_threepids:
+  - medium: 'email'
+    address: 'reserved_user@example.com'
+```
+---
+Config option: `server_context`
+
+This option is used by phonehome stats to group together related servers.
+Defaults to none. 
+
+Example configuration:
+```yaml
+server_context: context
+```
+---
+Config option: `limit_remote_rooms`
+
+When this option is enabled, the room "complexity" will be checked before a user
+joins a new remote room. If it is above the complexity limit, the server will
+disallow joining, or will instantly leave. This is useful for homeservers that are
+resource-constrained. Options for this setting include:
+* `enabled`: whether this check is enabled. Defaults to false.
+* `complexity`: the limit above which rooms cannot be joined. The default is 1.0.
+* `complexity_error`: override the error which is returned when the room is too complex with a
+   custom message. 
+* `admins_can_join`: allow server admins to join complex rooms. Default is false.
+
+Room complexity is an arbitrary measure based on factors such as the number of
+users in the room. 
+
+Example configuration:
+```yaml
+limit_remote_rooms:
+  enabled: true
+  complexity: 0.5
+  complexity_error: "I can't let you do that, Dave."
+  admins_can_join: true
+```
+---
+Config option: `require_membership_for_aliases`
+
+Whether to require a user to be in the room to add an alias to it.
+Defaults to true.
+
+Example configuration:
+```yaml
+require_membership_for_aliases: false
+```
+---
+Config option: `allow_per_room_profiles`
+
+Whether to allow per-room membership profiles through the sending of membership
+events with profile information that differs from the target's global profile.
+Defaults to true.
+
+Example configuration:
+```yaml
+allow_per_room_profiles: false
+```
+---
+Config option: `max_avatar_size`
+
+The largest permissible file size in bytes for a user avatar. Defaults to no restriction.
+Use M for MB and K for KB. 
+
+Note that user avatar changes will not work if this is set without using Synapse's media repository.
+
+Example configuration:
+```yaml
+max_avatar_size: 10M
+```
+---
+Config option: `allowed_avatar_mimetypes`
+
+The MIME types allowed for user avatars. Defaults to no restriction.
+
+Note that user avatar changes will not work if this is set without
+using Synapse's media repository.
+
+Example configuration:
+```yaml
+allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"]
+```
+---
+Config option: `redaction_retention_period`
+
+How long to keep redacted events in unredacted form in the database. After
+this period redacted events get replaced with their redacted form in the DB.
+
+Defaults to `7d`. Set to `null` to disable.
+
+Example configuration:
+```yaml
+redaction_retention_period: 28d
+```
+---
+Config option: `user_ips_max_age` 
+
+How long to track users' last seen time and IPs in the database.
+
+Defaults to `28d`. Set to `null` to disable clearing out of old rows.
+
+Example configuration:
+```yaml
+user_ips_max_age: 14d
+```
+---
+Config option: `request_token_inhibit_3pid_errors`
+
+Inhibits the `/requestToken` endpoints from returning an error that might leak
+information about whether an e-mail address is in use or not on this
+homeserver. Defaults to false. 
+Note that for some endpoints the error situation is the e-mail already being
+used, and for others the error is entering the e-mail being unused.
+If this option is enabled, instead of returning an error, these endpoints will
+act as if no error happened and return a fake session ID ('sid') to clients.
+
+Example configuration:
+```yaml
+request_token_inhibit_3pid_errors: true
+```
+---
+Config option: `next_link_domain_whitelist`
+
+A list of domains that the domain portion of `next_link` parameters
+must match.
+
+This parameter is optionally provided by clients while requesting
+validation of an email or phone number, and maps to a link that
+users will be automatically redirected to after validation
+succeeds. Clients can make use this parameter to aid the validation
+process.
+
+The whitelist is applied whether the homeserver or an identity server is handling validation.
+
+The default value is no whitelist functionality; all domains are
+allowed. Setting this value to an empty list will instead disallow
+all domains.
+
+Example configuration:
+```yaml
+next_link_domain_whitelist: ["matrix.org"]
+```
+---
+Config option: `templates` and `custom_template_directory`
+
+These options define templates to use when generating email or HTML page contents.
+The `custom_template_directory` determines which directory Synapse will try to 
+find template files in to use to generate email or HTML page contents.
+If not set, or a file is not found within the template directory, a default 
+template from within the Synapse package will be used.
+
+See [here](../../templates.md) for more
+information about using custom templates.
+
+Example configuration:
+```yaml
+templates:
+  custom_template_directory: /path/to/custom/templates/
+```
+---
+Config option: `retention`
+
+This option and the associated options determine message retention policy at the
+server level.
+
+Room admins and mods can define a retention period for their rooms using the
+`m.room.retention` state event, and server admins can cap this period by setting
+the `allowed_lifetime_min` and `allowed_lifetime_max` config options.
+
+If this feature is enabled, Synapse will regularly look for and purge events
+which are older than the room's maximum retention period. Synapse will also
+filter events received over federation so that events that should have been 
+purged are ignored and not stored again. 
+
+The message retention policies feature is disabled by default.
+
+This setting has the following sub-options:
+* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the
+   'm.room.retention' state event. This option is further specified by the 
+   `min_lifetime` and `max_lifetime` sub-options associated with it. Note that the 
+    value of `min_lifetime` doesn't matter much because Synapse doesn't take it into account yet. 
+
+* `allowed_lifetime_min` and `allowed_lifetime_max`: Retention policy limits. If 
+   set, and the state of a room contains a `m.room.retention` event in its state 
+   which contains a `min_lifetime` or a `max_lifetime` that's out of these bounds,
+   Synapse will cap the room's policy to these limits when running purge jobs.
+
+* `purge_jobs` and the associated `shortest_max_lifetime` and `longest_max_lifetime` sub-options:
+   Server admins can define the settings of the background jobs purging the
+   events whose lifetime has expired under the `purge_jobs` section.
+   
+  If no configuration is provided for this option, a single job will be set up to delete
+  expired events in every room daily.
+
+  Each job's configuration defines which range of message lifetimes the job
+  takes care of. For example, if `shortest_max_lifetime` is '2d' and
+  `longest_max_lifetime` is '3d', the job will handle purging expired events in
+  rooms whose state defines a `max_lifetime` that's both higher than 2 days, and
+  lower than or equal to 3 days. Both the minimum and the maximum value of a
+  range are optional, e.g. a job with no `shortest_max_lifetime` and a
+  `longest_max_lifetime` of '3d' will handle every room with a retention policy
+  whose `max_lifetime` is lower than or equal to three days.
+  
+  The rationale for this per-job configuration is that some rooms might have a
+  retention policy with a low `max_lifetime`, where history needs to be purged
+  of outdated messages on a more frequent basis than for the rest of the rooms
+  (e.g. every 12h), but not want that purge to be performed by a job that's
+  iterating over every room it knows, which could be heavy on the server.
+
+  If any purge job is configured, it is strongly recommended to have at least
+  a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime`
+  set, or one job without `shortest_max_lifetime` and one job without
+  `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if
+  `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a
+  room's policy to these values is done after the policies are retrieved from
+  Synapse's database (which is done using the range specified in a purge job's
+  configuration).
+
+Example configuration:
+```yaml
+retention:
+  enabled: true
+  default_policy:
+    min_lifetime: 1d
+    max_lifetime: 1y
+  allowed_lifetime_min: 1d
+  allowed_lifetime_max: 1y
+  purge_jobs:
+    - longest_max_lifetime: 3d
+      interval: 12h
+    - shortest_max_lifetime: 3d
+      interval: 1d  
+```
+---
+## TLS ##
+
+Options related to TLS.
+
+---
+Config option: `tls_certificate_path`
+
+This option specifies a PEM-encoded X509 certificate for TLS.
+This certificate, as of Synapse 1.0, will need to be a valid and verifiable
+certificate, signed by a recognised Certificate Authority. Defaults to none. 
+
+Be sure to use a `.pem` file that includes the full certificate chain including
+any intermediate certificates (for instance, if using certbot, use
+`fullchain.pem` as your certificate, not `cert.pem`). 
+
+Example configuration:
+```yaml
+tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt"
+```
+---
+Config option: `tls_private_key_path`
+
+PEM-encoded private key for TLS. Defaults to none. 
+
+Example configuration:
+```yaml
+tls_private_key_path: "CONFDIR/SERVERNAME.tls.key"
+```
+---
+Config option: `federation_verify_certificates`
+Whether to verify TLS server certificates for outbound federation requests.
+
+Defaults to true. To disable certificate verification, set the option to false.
+
+Example configuration:
+```yaml
+federation_verify_certificates: false
+```
+---
+Config option: `federation_client_minimum_tls_version`
+
+The minimum TLS version that will be used for outbound federation requests.
+
+Defaults to `1`. Configurable to `1`, `1.1`, `1.2`, or `1.3`. Note
+that setting this value higher than `1.2` will prevent federation to most
+of the public Matrix network: only configure it to `1.3` if you have an
+entirely private federation setup and you can ensure TLS 1.3 support.
+
+Example configuration:
+```yaml
+federation_client_minimum_tls_version: 1.2
+```
+---
+Config option: `federation_certificate_verification_whitelist`
+
+Skip federation certificate verification on a given whitelist
+of domains.
+
+This setting should only be used in very specific cases, such as
+federation over Tor hidden services and similar. For private networks
+of homeservers, you likely want to use a private CA instead.
+
+Only effective if `federation_verify_certicates` is `true`.
+
+Example configuration:
+```yaml
+federation_certificate_verification_whitelist:
+  - lon.example.com
+  - "*.domain.com"
+  - "*.onion"
+```
+---
+Config option: `federation_custom_ca_list`
+
+List of custom certificate authorities for federation traffic.
+
+This setting should only normally be used within a private network of
+homeservers.
+
+Note that this list will replace those that are provided by your
+operating environment. Certificates must be in PEM format.
+
+Example configuration:
+```yaml
+federation_custom_ca_list:
+  - myCA1.pem
+  - myCA2.pem
+  - myCA3.pem
+```
+---
+## Federation ##
+
+Options related to federation.
+
+---
+Config option: `federation_domain_whitelist`
+
+Restrict federation to the given whitelist of domains.
+N.B. we recommend also firewalling your federation listener to limit
+inbound federation traffic as early as possible, rather than relying
+purely on this application-layer restriction.  If not specified, the
+default is to whitelist everything.
+
+Example configuration:
+```yaml
+federation_domain_whitelist:
+  - lon.example.com
+  - nyc.example.com
+  - syd.example.com
+```
+---
+Config option: `federation_metrics_domains`
+
+Report prometheus metrics on the age of PDUs being sent to and received from
+the given domains. This can be used to give an idea of "delay" on inbound
+and outbound federation, though be aware that any delay can be due to problems
+at either end or with the intermediate network.
+
+By default, no domains are monitored in this way.
+
+Example configuration:
+```yaml
+federation_metrics_domains:
+  - matrix.org
+  - example.com
+```
+---
+Config option: `allow_profile_lookup_over_federation`
+
+Set to false to disable profile lookup over federation. By default, the
+Federation API allows other homeservers to obtain profile data of any user
+on this homeserver.
+
+Example configuration:
+```yaml
+allow_profile_lookup_over_federation: false
+```
+---
+Config option: `allow_device_name_lookup_over_federation`
+
+Set this option to false to disable device display name lookup over federation. By default, the
+Federation API allows other homeservers to obtain device display names of any user
+on this homeserver.
+
+Example configuration:
+```yaml
+allow_device_name_lookup_over_federation: false
+```
+---
+## Caching ##
+
+Options related to caching
+
+---
+Config option: `event_cache_size`
+
+The number of events to cache in memory. Not affected by
+`caches.global_factor`. Defaults to 10K.
+
+Example configuration:
+```yaml
+event_cache_size: 15K
+```
+---
+Config option: `cache` and associated values
+
+A cache 'factor' is a multiplier that can be applied to each of
+Synapse's caches in order to increase or decrease the maximum
+number of entries that can be stored.
+
+Caching can be configured through the following sub-options:
+
+* `global_factor`: Controls the global cache factor, which is the default cache factor
+  for all caches if a specific factor for that cache is not otherwise
+  set.
+
+  This can also be set by the `SYNAPSE_CACHE_FACTOR` environment
+  variable. Setting by environment variable takes priority over
+  setting through the config file.
+  
+  Defaults to 0.5, which will halve the size of all caches.
+
+* `per_cache_factors`: A dictionary of cache name to cache factor for that individual
+   cache. Overrides the global cache factor for a given cache.
+  
+   These can also be set through environment variables comprised
+   of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital
+   letters and underscores. Setting by environment variable
+   takes priority over setting through the config file.
+   Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0`
+  
+   Some caches have '*' and other characters that are not
+   alphanumeric or underscores. These caches can be named with or
+   without the special characters stripped. For example, to specify
+   the cache factor for `*stateGroupCache*` via an environment
+   variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`.
+ 
+* `expire_caches`: Controls whether cache entries are evicted after a specified time
+   period. Defaults to true. Set to false to disable this feature. Note that never expiring
+   caches may result in excessive memory usage. 
+
+* `cache_entry_ttl`: If `expire_caches` is enabled, this flag controls how long an entry can
+  be in a cache without having been accessed before being evicted.
+  Defaults to 30m. 
+
+* `sync_response_cache_duration`: Controls how long the results of a /sync request are
+  cached for after a successful response is returned. A higher duration can help clients
+  with intermittent connections, at the cost of higher memory usage.
+  By default, this is zero, which means that sync responses are not cached
+  at all.
+
+
+Example configuration:
+```yaml
+caches:
+  global_factor: 1.0
+  per_cache_factors:
+    get_users_who_share_room_with_user: 2.0
+  expire_caches: false
+  sync_response_cache_duration: 2m
+```
+---
+## Database ##
+Config options related to database settings.
+
+---
+Config option: `database`
+
+The `database` setting defines the database that synapse uses to store all of
+its data.
+
+Associated sub-options:
+
+* `name`: this option specifies the database engine to use: either `sqlite3` (for SQLite)
+  or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. 
+
+* `txn_limit` gives the maximum number of transactions to run per connection
+  before reconnecting. Defaults to 0, which means no limit.
+
+* `allow_unsafe_locale` is an option specific to Postgres. Under the default behavior, Synapse will refuse to
+  start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended)
+  by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information
+  [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes).
+
+* `args` gives options which are passed through to the database engine,
+  except for options starting with `cp_`, which are used to configure the Twisted
+  connection pool. For a reference to valid arguments, see:
+    * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect)
+    * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS)
+    * for [the connection pool](https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__)
+
+For more information on using Synapse with Postgres,
+see [here](../../postgres.md).
+
+Example SQLite configuration:
+```
+database:
+  name: sqlite3
+  args:
+    database: /path/to/homeserver.db
+```
+
+Example Postgres configuration:
+```
+database:
+  name: psycopg2
+  txn_limit: 10000
+  args:
+    user: synapse_user
+    password: secretpassword
+    database: synapse
+    host: localhost
+    port: 5432
+    cp_min: 5
+    cp_max: 10
+```
+---
+## Logging ##
+Config options related to logging. 
+
+---
+Config option: `log_config`
+
+This option specifies a yaml python logging config file as described [here](https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema).
+
+Example configuration:
+```yaml
+log_config: "CONFDIR/SERVERNAME.log.config"
+```
+---
+## Ratelimiting ##
+Options related to ratelimiting in Synapse. 
+
+Each ratelimiting configuration is made of two parameters:
+   - `per_second`: number of requests a client can send per second.
+   - `burst_count`: number of requests a client can send before being throttled.
+---
+Config option: `rc_message`
+
+
+Ratelimiting settings for client messaging.
+   
+This is a ratelimiting option for messages that ratelimits sending based on the account the client
+is using. It defaults to: `per_second: 0.2`, `burst_count: 10`.
+
+Example configuration:
+```yaml
+rc_message:
+  per_second: 0.5
+  burst_count: 15
+```
+---
+Config option: `rc_registration`
+
+This option ratelimits registration requests based on the client's IP address.
+It defaults to `per_second: 0.17`, `burst_count: 3`. 
+
+Example configuration:
+```yaml
+rc_registration:
+  per_second: 0.15
+  burst_count: 2
+```
+---
+Config option: `rc_registration_token_validity`
+
+This option checks the validity of registration tokens that ratelimits requests based on 
+the client's IP address.
+Defaults to `per_second: 0.1`, `burst_count: 5`.
+
+Example configuration:
+```yaml
+rc_registration_token_validity:
+  per_second: 0.3
+  burst_count: 6
+```   
+---
+Config option: `rc_login`
+
+This option specifies several limits for login:
+* `address` ratelimits login requests based on the client's IP
+      address. Defaults to `per_second: 0.17`, `burst_count: 3`.
+    
+* `account` ratelimits login requests based on the account the
+  client is attempting to log into. Defaults to `per_second: 0.17`,
+  `burst_count: 3`.
+    
+* `failted_attempts` ratelimits login requests based on the account the
+  client is attempting to log into, based on the amount of failed login
+  attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`.
+
+Example configuration:
+```yaml
+rc_login:
+  address:
+    per_second: 0.15
+    burst_count: 5
+  account:
+    per_second: 0.18
+    burst_count: 4
+  failed_attempts:
+    per_second: 0.19
+    burst_count: 7
+```
+---
+Config option: `rc_admin_redaction`
+
+This option sets ratelimiting redactions by room admins. If this is not explicitly 
+set then it uses the same ratelimiting as per `rc_message`. This is useful
+to allow room admins to deal with abuse quickly. 
+
+Example configuration:
+```yaml
+rc_admin_redaction:
+  per_second: 1
+  burst_count: 50
+```
+---
+Config option: `rc_joins`
+
+This option allows for ratelimiting number of rooms a user can join. This setting has the following sub-options:
+
+* `local`: ratelimits when users are joining rooms the server is already in. 
+   Defaults to `per_second: 0.1`, `burst_count: 10`.
+
+* `remote`: ratelimits when users are trying to join rooms not on the server (which
+  can be more computationally expensive than restricting locally). Defaults to
+  `per_second: 0.01`, `burst_count: 10` 
+
+Example configuration:
+```yaml
+rc_joins:
+  local:
+    per_second: 0.2
+    burst_count: 15
+  remote:
+    per_second: 0.03
+    burst_count: 12
+```
+---
+Config option: `rc_3pid_validation`
+
+This option ratelimits how often a user or IP can attempt to validate a 3PID.
+Defaults to `per_second: 0.003`, `burst_count: 5`.
+
+Example configuration:
+```yaml
+rc_3pid_validation:
+  per_second: 0.003
+  burst_count: 5
+```
+---
+Config option: `rc_invites`
+
+This option sets ratelimiting how often invites can be sent in a room or to a 
+specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10` and
+`per_user` defaults to `per_second: 0.003`, `burst_count: 5`. 
+
+Example configuration:
+```yaml
+rc_invites:
+  per_room:
+    per_second: 0.5
+    burst_count: 5
+  per_user:
+    per_second: 0.004
+    burst_count: 3
+```
+---
+Config option: `rc_third_party_invite`
+
+This option ratelimits 3PID invites (i.e. invites sent to a third-party ID
+such as an email address or a phone number) based on the account that's
+sending the invite. Defaults to `per_second: 0.2`, `burst_count: 10`.
+
+Example configuration:
+```yaml
+rc_third_party_invite:
+  per_second: 0.2
+  burst_count: 10
+```
+---
+Config option: `rc_federation`
+
+Defines limits on federation requests. 
+
+The `rc_federation` configuration has the following sub-options:
+* `window_size`: window size in milliseconds. Defaults to 1000.
+* `sleep_limit`: number of federation requests from a single server in
+   a window before the server will delay processing the request. Defaults to 10.
+* `sleep_delay`: duration in milliseconds to delay processing events
+   from remote servers by if they go over the sleep limit. Defaults to 500.
+* `reject_limit`: maximum number of concurrent federation requests
+   allowed from a single server. Defaults to 50.
+* `concurrent`: number of federation requests to concurrently process
+   from a single server. Defaults to 3.
+
+Example configuration:
+```yaml
+rc_federation:
+  window_size: 750
+  sleep_limit: 15
+  sleep_delay: 400
+  reject_limit: 40
+  concurrent: 5
+```
+---
+Config option: `federation_rr_transactions_per_room_per_second`
+
+Sets outgoing federation transaction frequency for sending read-receipts,
+per-room.
+
+If we end up trying to send out more read-receipts, they will get buffered up
+into fewer transactions. Defaults to 50. 
+
+Example configuration:
+```yaml
+federation_rr_transactions_per_room_per_second: 40
+```
+---
+## Media Store ##
+Config options relating to Synapse media store.
+
+---
+Config option: `enable_media_repo` 
+
+Enable the media store service in the Synapse master. Defaults to true. 
+Set to false if you are using a separate media store worker.
+
+Example configuration:
+```yaml
+enable_media_repo: false
+```
+---
+Config option: `media_store_path`
+
+Directory where uploaded images and attachments are stored.
+
+Example configuration:
+```yaml
+media_store_path: "DATADIR/media_store"
+```
+---
+Config option: `media_storage_providers`
+
+Media storage providers allow media to be stored in different
+locations. Defaults to none. Associated sub-options are:
+* `module`: type of resource, e.g. `file_system`.
+* `store_local`: whether to store newly uploaded local files
+* `store_remote`: whether to store newly downloaded local files
+* `store_synchronous`: whether to wait for successful storage for local uploads
+* `config`: sets a path to the resource through the `directory` option 
+
+Example configuration:
+```yaml
+media_storage_providers:
+  - module: file_system
+    store_local: false
+    store_remote: false
+    store_synchronous: false
+    config:
+       directory: /mnt/some/other/directory
+```
+---
+Config option: `max_upload_size`
+
+The largest allowed upload size in bytes.
+
+If you are using a reverse proxy you may also need to set this value in
+your reverse proxy's config. Defaults to 50M. Notably Nginx has a small max body size by default.
+See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. 
+
+Example configuration:
+```yaml
+max_upload_size: 60M
+```
+---
+Config option: `max_image_pixels`
+
+Maximum number of pixels that will be thumbnailed. Defaults to 32M.
+
+Example configuration:
+```yaml
+max_image_pixels: 35M
+```
+---
+Config option: `dynamic_thumbnails`
+
+Whether to generate new thumbnails on the fly to precisely match
+the resolution requested by the client. If true then whenever
+a new resolution is requested by the client the server will
+generate a new thumbnail. If false the server will pick a thumbnail
+from a precalculated list. Defaults to false. 
+
+Example configuration:
+```yaml
+dynamic_thumbnails: true
+```
+---
+Config option: `thumbnail_sizes`  
+
+List of thumbnails to precalculate when an image is uploaded. Associated sub-options are:
+* `width`
+* `height`
+* `method`: i.e. `crop`, `scale`, etc.
+
+Example configuration:
+```yaml
+thumbnail_sizes:
+  - width: 32
+    height: 32
+    method: crop
+  - width: 96
+    height: 96
+    method: crop
+  - width: 320
+    height: 240
+    method: scale
+  - width: 640
+    height: 480
+    method: scale
+  - width: 800
+    height: 600
+    method: scale
+```
+Config option: `url_preview_enabled`
+
+This setting determines whether the preview URL API is enabled.
+It is disabled by default. Set to true to enable. If enabled you must specify a
+`url_preview_ip_range_blacklist` blacklist.
+
+Example configuration:
+```yaml
+url_preview_enabled: true
+```
+---
+Config option: `url_preview_ip_range_blacklist`
+
+List of IP address CIDR ranges that the URL preview spider is denied
+from accessing.  There are no defaults: you must explicitly
+specify a list for URL previewing to work.  You should specify any
+internal services in your network that you do not want synapse to try
+to connect to, otherwise anyone in any Matrix room could cause your
+synapse to issue arbitrary GET requests to your internal services,
+causing serious security issues.
+
+(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly
+listed here, since they correspond to unroutable addresses.)
+
+This must be specified if `url_preview_enabled` is set. It is recommended that
+you use the following example list as a starting point.
+
+Note: The value is ignored when an HTTP proxy is in use.
+
+Example configuration:
+```yaml
+url_preview_ip_range_blacklist:
+  - '127.0.0.0/8'
+  - '10.0.0.0/8'
+  - '172.16.0.0/12'
+  - '192.168.0.0/16'
+  - '100.64.0.0/10'
+  - '192.0.0.0/24'
+  - '169.254.0.0/16'
+  - '192.88.99.0/24'
+  - '198.18.0.0/15'
+  - '192.0.2.0/24'
+  - '198.51.100.0/24'
+  - '203.0.113.0/24'
+  - '224.0.0.0/4'
+  - '::1/128'
+  - 'fe80::/10'
+  - 'fc00::/7'
+  - '2001:db8::/32'
+  - 'ff00::/8'
+  - 'fec0::/10'
+```
+----
+Config option: `url_preview_ip_range_whitelist`
+
+This option sets a list of IP address CIDR ranges that the URL preview spider is allowed
+to access even if they are specified in `url_preview_ip_range_blacklist`.
+This is useful for specifying exceptions to wide-ranging blacklisted
+target IP ranges - e.g. for enabling URL previews for a specific private
+website only visible in your network. Defaults to none. 
+
+Example configuration:
+```yaml
+url_preview_ip_range_whitelist:
+   - '192.168.1.1'
+```
+---
+Config option: `url_preview_url_blacklist`
+
+Optional list of URL matches that the URL preview spider is
+denied from accessing.  You should use `url_preview_ip_range_blacklist`
+in preference to this, otherwise someone could define a public DNS
+entry that points to a private IP address and circumvent the blacklist.
+This is more useful if you know there is an entire shape of URL that
+you know that will never want synapse to try to spider.
+
+Each list entry is a dictionary of url component attributes as returned
+by urlparse.urlsplit as applied to the absolute form of the URL.  See 
+[here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more
+information. Some examples are:
+
+* `username`
+* `netloc`
+* `scheme`
+* `path`
+
+The values of the dictionary are treated as a filename match pattern
+applied to that component of URLs, unless they start with a ^ in which
+case they are treated as a regular expression match.  If all the
+specified component matches for a given list item succeed, the URL is
+blacklisted.
+
+Example configuration:
+```yaml
+url_preview_url_blacklist:
+  # blacklist any URL with a username in its URI
+  - username: '*'
+
+  # blacklist all *.google.com URLs
+  - netloc: 'google.com'
+  - netloc: '*.google.com'
+
+  # blacklist all plain HTTP URLs
+  - scheme: 'http'
+
+  # blacklist http(s)://www.acme.com/foo
+  - netloc: 'www.acme.com'
+    path: '/foo'
+
+  # blacklist any URL with a literal IPv4 address
+  - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
+```
+---
+Config option: `max_spider_size`
+
+The largest allowed URL preview spidering size in bytes. Defaults to 10M.
+
+Example configuration:
+```yaml
+max_spider_size: 8M
+```
+---
+Config option: `url_preview_language`
+
+A list of values for the Accept-Language HTTP header used when
+downloading webpages during URL preview generation. This allows
+Synapse to specify the preferred languages that URL previews should
+be in when communicating with remote servers.
+
+Each value is a IETF language tag; a 2-3 letter identifier for a
+language, optionally followed by subtags separated by '-', specifying
+a country or region variant.
+
+Multiple values can be provided, and a weight can be added to each by
+using quality value syntax (;q=). '*' translates to any language.
+
+Defaults to "en".
+
+Example configuration:
+```yaml
+ url_preview_accept_language:
+   - en-UK
+   - en-US;q=0.9
+   - fr;q=0.8
+   - *;q=0.7
+```
+----
+Config option: `oembed`
+
+oEmbed allows for easier embedding content from a website. It can be
+used for generating URLs previews of services which support it. A default list of oEmbed providers
+is included with Synapse. Set `disable_default_providers` to true to disable using
+these default oEmbed URLs. Use `additional_providers` to specify additional files with oEmbed configuration (each 
+should be in the form of providers.json). By default this list is empty. 
+
+Example configuration:
+```yaml
+oembed:
+  disable_default_providers: true
+  additional_providers:
+    - oembed/my_providers.json
+```
+---
+## Captcha ##
+
+See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha.
+
+---
+Config option: `recaptcha_public_key`
+
+This homeserver's ReCAPTCHA public key. Must be specified if `enable_registration_captcha` is 
+enabled.
+
+Example configuration:
+```yaml
+recaptcha_public_key: "YOUR_PUBLIC_KEY"
+```
+---
+Config option: `recaptcha_private_key` 
+
+This homeserver's ReCAPTCHA private key. Must be specified if `enable_registration_captcha` is 
+enabled.
+
+Example configuration:
+```yaml
+recaptcha_private_key: "YOUR_PRIVATE_KEY"
+```
+---
+Config option: `enable_registration_captcha`
+
+Set to true to enable ReCaptcha checks when registering, preventing signup
+unless a captcha is answered. Requires a valid ReCaptcha public/private key. 
+Defaults to false.
+
+Example configuration:
+```yaml
+enable_registration_captcha: true
+```
+---
+Config option: `recaptcha_siteverify_api`
+
+The API endpoint to use for verifying `m.login.recaptcha` responses.
+Defaults to `https://www.recaptcha.net/recaptcha/api/siteverify`.
+
+Example configuration:
+```yaml
+recaptcha_siteverify_api: "https://my.recaptcha.site"
+```
+---
+## TURN ##
+Options related to adding a TURN server to Synapse.
+
+---
+Config option: `turn_uris`
+
+The public URIs of the TURN server to give to clients.
+
+Example configuration:
+```yaml
+turn_uris: [turn:example.org]
+```
+---
+Config option: `turn_shared_secret`
+
+The shared secret used to compute passwords for the TURN server.
+
+Example configuration:
+```yaml
+turn_shared_secret: "YOUR_SHARED_SECRET"
+```
+----
+Config options: `turn_username` and `turn_password`
+
+The Username and password if the TURN server needs them and does not use a token.
+
+Example configuration:
+```yaml
+turn_username: "TURNSERVER_USERNAME"
+turn_password: "TURNSERVER_PASSWORD"
+```
+---
+Config option: `turn_user_lifetime`
+
+How long generated TURN credentials last. Defaults to 1h.
+
+Example configuration:
+```yaml
+turn_user_lifetime: 2h
+```
+---
+Config option: `turn_allow_guests`
+
+Whether guests should be allowed to use the TURN server. This defaults to true, otherwise
+VoIP will be unreliable for guests. However, it does introduce a slight security risk as
+it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA).
+
+Example configuration:
+```yaml
+turn_allow_guests: false
+```
+---
+## Registration ##
+
+Registration can be rate-limited using the parameters in the [Ratelimiting](#ratelimiting) section of this manual.
+
+---
+Config option: `enable_registration`
+
+Enable registration for new users. Defaults to false. It is highly recommended that if you enable registration,
+you use either captcha, email, or token-based verification to verify that new users are not bots. In order to enable registration 
+without any verification, you must also set `enable_registration_without_verification` to true.
+
+Example configuration:
+```yaml
+enable_registration: true
+```
+---
+Config option: `enable_registration_without_verification`
+Enable registration without email or captcha verification. Note: this option is *not* recommended,
+as registration without verification is a known vector for spam and abuse. Defaults to false. Has no effect
+unless `enable_registration` is also enabled.
+
+Example configuration:
+```yaml
+enable_registration_without_verification: true
+```
+---
+Config option: `session_lifetime`
+
+Time that a user's session remains valid for, after they log in.
+
+Note that this is not currently compatible with guest logins.
+
+Note also that this is calculated at login time: changes are not applied retrospectively to users who have already 
+logged in.
+
+By default, this is infinite.
+
+Example configuration:
+```yaml
+session_lifetime: 24h
+```
+----
+Config option: `refresh_access_token_lifetime`
+
+Time that an access token remains valid for, if the session is using refresh tokens.
+
+For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md).
+
+Note that this only applies to clients which advertise support for refresh tokens.
+
+Note also that this is calculated at login time and refresh time: changes are not applied to 
+existing sessions until they are refreshed.
+
+By default, this is 5 minutes.
+
+Example configuration:
+```yaml
+refreshable_access_token_lifetime: 10m
+```
+---
+Config option: `refresh_token_lifetime: 24h`
+
+Time that a refresh token remains valid for (provided that it is not
+exchanged for another one first).
+This option can be used to automatically log-out inactive sessions.
+Please see the manual for more information.
+
+Note also that this is calculated at login time and refresh time:
+changes are not applied to existing sessions until they are refreshed.
+
+By default, this is infinite.
+
+Example configuration:
+```yaml
+refresh_token_lifetime: 24h
+```
+---
+Config option: `nonrefreshable_access_token_lifetime`
+
+Time that an access token remains valid for, if the session is NOT
+using refresh tokens.
+
+Please note that not all clients support refresh tokens, so setting
+this to a short value may be inconvenient for some users who will
+then be logged out frequently.
+
+Note also that this is calculated at login time: changes are not applied
+retrospectively to existing sessions for users that have already logged in.
+
+By default, this is infinite.
+
+Example configuration:
+```yaml
+nonrefreshable_access_token_lifetime: 24h
+```
+---
+Config option: `registrations_require_3pid`
+
+If this is set, the user must provide all of the specified types of 3PID when registering.
+
+Example configuration:
+```yaml
+registrations_require_3pid:
+  - email
+  - msisdn
+```
+---
+Config option: `disable_msisdn_registration`
+
+Explicitly disable asking for MSISDNs from the registration
+flow (overrides `registrations_require_3pid` if MSISDNs are set as required).
+
+Example configuration:
+```yaml
+disable_msisdn_registration: true
+```
+---
+Config option: `allowed_local_3pids`
+
+Mandate that users are only allowed to associate certain formats of
+3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options.
+
+Example configuration:
+```yaml
+allowed_local_3pids:
+  - medium: email
+    pattern: '^[^@]+@matrix\.org$'
+  - medium: email
+    pattern: '^[^@]+@vector\.im$'
+  - medium: msisdn
+    pattern: '\+44'
+```
+---
+Config option: `enable_3pid_lookup`
+
+Enable 3PIDs lookup requests to identity servers from this server. Defaults to true.
+
+Example configuration:
+```yaml
+enable_3pid_lookup: false
+```
+---
+Config option: `registration_requires_token`
+
+Require users to submit a token during registration.
+Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md).
+Note that `enable_registration` must be set to true.
+Disabling this option will not delete any tokens previously generated.
+Defaults to false. Set to true to enable.  
+
+Example configuration:
+```yaml
+registration_requires_token: true
+```
+---
+Config option: `registration_shared_secret`
+
+If set, allows registration of standard or admin accounts by anyone who
+has the shared secret, even if registration is otherwise disabled.
+
+Example configuration:
+```yaml
+registration_shared_secret: <PRIVATE STRING>
+```
+---
+Config option: `bcrypt_rounds`
+
+Set the number of bcrypt rounds used to generate password hash.
+Larger numbers increase the work factor needed to generate the hash.
+The default number is 12 (which equates to 2^12 rounds).
+N.B. that increasing this will exponentially increase the time required
+to register or login - e.g. 24 => 2^24 rounds which will take >20 mins.
+Example configuration:
+```yaml
+bcrypt_rounds: 14
+```
+---
+Config option: `allow_guest_access`
+
+Allows users to register as guests without a password/email/etc, and
+participate in rooms hosted on this server which have been made
+accessible to anonymous users. Defaults to false.
+
+Example configuration:
+```yaml
+allow_guest_access: true
+```
+---
+Config option: `default_identity_server`
+
+The identity server which we suggest that clients should use when users log
+in on this server.
+
+(By default, no suggestion is made, so it is left up to the client.
+This setting is ignored unless `public_baseurl` is also explicitly set.)
+
+Example configuration:
+```yaml
+default_identity_server: https://matrix.org
+```
+---
+Config option: `account_threepid_delegates`
+
+Handle threepid (email/phone etc) registration and password resets through a set of
+*trusted* identity servers. Note that this allows the configured identity server to
+reset passwords for accounts!
+
+Be aware that if `email` is not set, and SMTP options have not been
+configured in the email config block, registration and user password resets via
+email will be globally disabled.
+
+Additionally, if `msisdn` is not set, registration and password resets via msisdn
+will be disabled regardless, and users will not be able to associate an msisdn
+identifier to their account. This is due to Synapse currently not supporting
+any method of sending SMS messages on its own.
+
+To enable using an identity server for operations regarding a particular third-party
+identifier type, set the value to the URL of that identity server as shown in the
+examples below.
+
+Servers handling the these requests must answer the `/requestToken` endpoints defined
+by the Matrix Identity Service API [specification](https://matrix.org/docs/spec/identity_service/latest).
+
+Example configuration:
+```yaml
+account_threepid_delegates:
+    email: https://example.com     # Delegate email sending to example.com
+    msisdn: http://localhost:8090  # Delegate SMS sending to this local process
+```
+---
+Config option: `enable_set_displayname`
+
+Whether users are allowed to change their displayname after it has
+been initially set. Useful when provisioning users based on the
+contents of a third-party directory.
+
+Does not apply to server administrators. Defaults to true.
+
+Example configuration:
+```yaml
+enable_set_displayname: false
+```
+---
+Config option: `enable_set_avatar_url`
+
+Whether users are allowed to change their avatar after it has been
+initially set. Useful when provisioning users based on the contents
+of a third-party directory.
+
+Does not apply to server administrators. Defaults to true.
+
+Example configuration:
+```yaml
+enable_set_avatar_url: false
+```
+---
+Config option: `enable_3pid_changes`
+
+Whether users can change the third-party IDs associated with their accounts
+(email address and msisdn).
+
+Defaults to true.
+
+Example configuration:
+```yaml
+enable_3pid_changes: false
+```
+---
+Config option: `auto_join_rooms`
+
+Users who register on this homeserver will automatically be joined
+to the rooms listed under this option.
+
+By default, any room aliases included in this list will be created
+as a publicly joinable room when the first user registers for the
+homeserver. If the room already exists, make certain it is a publicly joinable
+room, i.e. the join rule of the room must be set to 'public'. You can find more options
+relating to auto-joining rooms below. 
+
+Example configuration:
+```yaml
+auto_join_rooms:
+  - "#exampleroom:example.com"
+  - "#anotherexampleroom:example.com"
+```
+---
+Config option: `autocreate_auto_join_rooms`
+
+Where `auto_join_rooms` are specified, setting this flag ensures that
+the rooms exist by creating them when the first user on the
+homeserver registers.
+
+By default the auto-created rooms are publicly joinable from any federated
+server. Use the `autocreate_auto_join_rooms_federated` and
+`autocreate_auto_join_room_preset` settings to customise this behaviour.
+
+Setting to false means that if the rooms are not manually created,
+users cannot be auto-joined since they do not exist.
+
+Defaults to true.
+
+Example configuration:
+```yaml
+autocreate_auto_join_rooms: false
+```
+---
+Config option: `autocreate_auto_join_rooms_federated`
+
+Whether the rooms listen in `auto_join_rooms` that are auto-created are available
+via federation. Only has an effect if `autocreate_auto_join_rooms` is true.
+
+Note that whether a room is federated cannot be modified after
+creation.
+
+Defaults to true: the room will be joinable from other servers.
+Set to false to prevent users from other homeservers from
+joining these rooms.
+
+Example configuration:
+```yaml
+autocreate_auto_join_rooms_federated: false
+```
+---
+Config option: `autocreate_auto_join_room_preset`
+
+The room preset to use when auto-creating one of `auto_join_rooms`. Only has an
+effect if `autocreate_auto_join_rooms` is true.
+
+Possible values for this option are:
+* "public_chat": the room is joinable by anyone, including
+  federated servers if `autocreate_auto_join_rooms_federated` is true (the default).
+* "private_chat": an invitation is required to join these rooms. 
+* "trusted_private_chat": an invitation is required to join this room and the invitee is
+  assigned a power level of 100 upon joining the room. 
+
+If a value of "private_chat" or "trusted_private_chat" is used then
+`auto_join_mxid_localpart` must also be configured.
+
+Defaults to "public_chat".
+
+Example configuration:
+```yaml
+autocreate_auto_join_room_preset: private_chat
+```
+---
+Config option: `auto_join_mxid_localpart`
+
+The local part of the user id which is used to create `auto_join_rooms` if
+`autocreate_auto_join_rooms` is true. If this is not provided then the
+initial user account that registers will be used to create the rooms.
+
+The user id is also used to invite new users to any auto-join rooms which
+are set to invite-only.
+
+It *must* be configured if `autocreate_auto_join_room_preset` is set to
+"private_chat" or "trusted_private_chat".
+
+Note that this must be specified in order for new users to be correctly
+invited to any auto-join rooms which have been set to invite-only (either
+at the time of creation or subsequently).
+
+Note that, if the room already exists, this user must be joined and
+have the appropriate permissions to invite new members.
+
+Example configuration:
+```yaml
+auto_join_mxid_localpart: system
+```
+---
+Config option: `auto_join_rooms_for_guests`
+ 
+When `auto_join_rooms` is specified, setting this flag to false prevents
+guest accounts from being automatically joined to the rooms.
+
+Defaults to true.
+
+Example configuration:
+```yaml
+auto_join_rooms_for_guests: false
+```
+---
+Config option: `inhibit_user_in_use_error`
+ 
+Whether to inhibit errors raised when registering a new account if the user ID
+already exists. If turned on, requests to `/register/available` will always
+show a user ID as available, and Synapse won't raise an error when starting
+a registration with a user ID that already exists. However, Synapse will still
+raise an error if the registration completes and the username conflicts.
+
+Defaults to false.
+
+Example configuration:
+```yaml
+inhibit_user_in_use_error: true
+```
+---
+## Metrics ###
+Config options related to metrics.
+
+---
+Config option: `enable_metrics`
+
+Set to true to enable collection and rendering of performance metrics. 
+Defaults to false.
+
+Example configuration:
+```yaml
+enable_metrics: true
+```
+---
+Config option: `sentry`
+
+Use this option to enable sentry integration. Provide the DSN assigned to you by sentry
+with the `dsn` setting. 
+
+NOTE: While attempts are made to ensure that the logs don't contain
+any sensitive information, this cannot be guaranteed. By enabling
+this option the sentry server may therefore receive sensitive 
+information, and it in turn may then disseminate sensitive information
+through insecure notification channels if so configured.
+
+Example configuration:
+```yaml
+sentry:
+    dsn: "..."
+```
+---
+Config option: `metrics_flags`
+
+Flags to enable Prometheus metrics which are not suitable to be
+enabled by default, either for performance reasons or limited use.
+Currently the only option is `known_servers`, which publishes 
+`synapse_federation_known_servers`, a gauge of the number of
+servers this homeserver knows about, including itself. May cause
+performance problems on large homeservers.
+
+Example configuration:
+```yaml
+metrics_flags:
+    known_servers: true
+```
+---
+Config option: `report_stats`
+
+Whether or not to report anonymized homeserver usage statistics. This is originally
+set when generating the config. Set this option to true or false to change the current
+behavior. 
+
+Example configuration:
+```yaml
+report_stats: true
+```
+---
+Config option: `report_stats_endpoint`
+
+The endpoint to report the anonymized homeserver usage statistics to.
+Defaults to https://matrix.org/report-usage-stats/push
+
+Example configuration:
+```yaml
+report_stats_endpoint: https://example.com/report-usage-stats/push
+```
+---
+## API Configuration ##
+Config settings related to the client/server API
+
+---
+Config option: `room_prejoin_state:`
+
+Controls for the state that is shared with users who receive an invite
+to a room. By default, the following state event types are shared with users who 
+receive invites to the room:
+- m.room.join_rules
+- m.room.canonical_alias
+- m.room.avatar
+- m.room.encryption
+- m.room.name
+- m.room.create
+- m.room.topic
+
+To change the default behavior, use the following sub-options:
+* `disable_default_event_types`: set to true to disable the above defaults. If this 
+   is enabled, only the event types listed in `additional_event_types` are shared.
+   Defaults to false.
+* `additional_event_types`: Additional state event types to share with users when they are invited
+   to a room. By default, this list is empty (so only the default event types are shared).
+
+Example configuration:
+```yaml
+room_prejoin_state:
+   disable_default_event_types: true
+   additional_event_types:
+     - org.example.custom.event.type
+     - m.room.join_rules
+```
+---
+Config option: `track_puppeted_user_ips`
+
+We record the IP address of clients used to access the API for various
+reasons, including displaying it to the user in the "Where you're signed in"
+dialog.
+
+By default, when puppeting another user via the admin API, the client IP
+address is recorded against the user who created the access token (ie, the
+admin user), and *not* the puppeted user.
+
+Set this option to true to also record the IP address against the puppeted
+user. (This also means that the puppeted user will count as an "active" user
+for the purpose of monthly active user tracking - see `limit_usage_by_mau` etc
+above.)
+
+Example configuration:
+```yaml
+track_puppeted_user_ips: true
+```
+---
+Config option: `app_service_config_files`
+
+A list of application service config files to use.
+
+Example configuration:
+```yaml
+app_service_config_files:
+  - app_service_1.yaml
+  - app_service_2.yaml
+```
+---
+Config option: `track_appservice_user_ips`
+
+Defaults to false. Set to true to enable tracking of application service IP addresses.
+Implicitly enables MAU tracking for application service users.
+
+Example configuration:
+```yaml
+track_appservice_user_ips: true
+```
+---
+Config option: `macaroon_secret_key`
+
+A secret which is used to sign access tokens. If none is specified,
+the `registration_shared_secret` is used, if one is given; otherwise,
+a secret key is derived from the signing key.
+
+Example configuration:
+```yaml
+macaroon_secret_key: <PRIVATE STRING>
+```
+---
+Config option: `form_secret`
+
+A secret which is used to calculate HMACs for form values, to stop
+falsification of values. Must be specified for the User Consent
+forms to work.
+
+Example configuration:
+```yaml
+form_secret: <PRIVATE STRING>
+```
+---
+## Signing Keys ##
+Config options relating to signing keys
+
+---
+Config option: `signing_key_path`
+
+Path to the signing key to sign messages with.
+
+Example configuration:
+```yaml
+signing_key_path: "CONFDIR/SERVERNAME.signing.key"
+```
+--- 
+Config option: `old_signing_keys`
+
+The keys that the server used to sign messages with but won't use
+to sign new messages. For each key, `key` should be the base64-encoded public key, and
+`expired_ts`should be the time (in milliseconds since the unix epoch) that
+it was last used.
+
+It is possible to build an entry from an old `signing.key` file using the
+`export_signing_key` script which is provided with synapse.
+
+Example configuration:
+```yaml
+old_signing_keys:
+  "ed25519:id": { key: "base64string", expired_ts: 123456789123 }
+```
+---
+Config option: `key_refresh_interval`
+
+How long key response published by this server is valid for.
+Used to set the `valid_until_ts` in `/key/v2` APIs.
+Determines how quickly servers will query to check which keys
+are still valid. Defaults to 1d.
+
+Example configuration:
+```yaml
+key_refresh_interval: 2d
+```
+---
+Config option: `trusted_key_servers:`
+
+The trusted servers to download signing keys from.
+
+When we need to fetch a signing key, each server is tried in parallel.
+
+Normally, the connection to the key server is validated via TLS certificates.
+Additional security can be provided by configuring a `verify key`, which
+will make synapse check that the response is signed by that key.
+
+This setting supercedes an older setting named `perspectives`. The old format
+is still supported for backwards-compatibility, but it is deprecated.
+
+`trusted_key_servers` defaults to matrix.org, but using it will generate a
+warning on start-up. To suppress this warning, set
+`suppress_key_server_warning` to true.
+
+Options for each entry in the list include:
+* `server_name`: the name of the server. Required.
+* `verify_keys`: an optional map from key id to base64-encoded public key.
+   If specified, we will check that the response is signed by at least
+   one of the given keys.
+* `accept_keys_insecurely`: a boolean. Normally, if `verify_keys` is unset,
+   and `federation_verify_certificates` is not `true`, synapse will refuse 
+   to start, because this would allow anyone who can spoof DNS responses
+   to masquerade as the trusted key server. If you know what you are doing
+   and are sure that your network environment provides a secure connection
+   to the key server, you can set this to `true` to override this behaviour.
+
+Example configuration #1:
+```yaml
+trusted_key_servers:
+  - server_name: "my_trusted_server.example.com"
+    verify_keys:
+      "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr"
+  - server_name: "my_other_trusted_server.example.com"
+```
+Example configuration #2:
+```yaml
+trusted_key_servers:
+  - server_name: "matrix.org"
+```
+---
+Config option: `suppress_key_server_warning`
+
+Set the following to true to disable the warning that is emitted when the
+`trusted_key_servers` include 'matrix.org'. See above.
+
+Example configuration:
+```yaml
+suppress_key_server_warning: true
+```
+---
+Config option: `key_server_signing_keys_path`
+
+The signing keys to use when acting as a trusted key server. If not specified
+defaults to the server signing key.
+
+Can contain multiple keys, one per line.
+
+Example configuration:
+```yaml
+key_server_signing_keys_path: "key_server_signing_keys.key"
+```
+---
+## Single sign-on integration ##
+
+The following settings can be used to make Synapse use a single sign-on
+provider for authentication, instead of its internal password database.
+
+You will probably also want to set the following options to false to
+disable the regular login/registration flows:
+   * `enable_registration`
+   * `password_config.enabled`
+
+You will also want to investigate the settings under the "sso" configuration
+section below.
+
+---
+Config option: `saml2_config`
+
+Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and
+to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/).
+
+At least one of `sp_config` or `config_path` must be set in this section to
+enable SAML login. You can either put your entire pysaml config inline using the `sp_config`
+option, or you can specify a path to a psyaml config file with the sub-option `config_path`.
+This setting has the following sub-options:
+
+* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config.
+   Default values will be used for the `entityid` and `service` settings,
+   so it is not normally necessary to specify them unless you need to
+   override them. Here are a few useful sub-options for configuring pysaml:
+   * `metadata`: Point this to the IdP's metadata. You must provide either a local
+      file via the `local` attribute or (preferably) a URL via the
+      `remote` attribute.
+   * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP.
+      Defaults to 0.
+   * `service`: By default, the user has to go to our login page first. If you'd like
+     to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service`
+     section.
+* `config_path`: specify a separate pysaml2 configuration file thusly: 
+  `config_path: "CONFDIR/sp_conf.py"`
+* `saml_session_lifetime`: The lifetime of a SAML session. This defines how long a user has to
+   complete the authentication process, if `allow_unsolicited` is unset. The default is 15 minutes.
+* `user_mapping_provider`: Using this option, an external module can be provided as a 
+   custom solution to mapping attributes returned from a saml provider onto a matrix user. The 
+   `user_mapping_provider` has the following attributes:
+  * `module`: The custom module's class. 
+  * `config`: Custom configuration values for the module. Use the values provided in the 
+     example if you are using the built-in user_mapping_provider, or provide your own
+     config values for a custom class if you are using one. This section will be passed as a Python
+     dictionary to the module's `parse_config` method. The built-in provider takes the following two
+     options:
+      * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use
+          to derive the Matrix ID from. It is 'uid' by default. Note: This used to be configured by the
+          `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead.
+      * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a
+         matrix ID. Options include: `hexencode` (which maps unpermitted characters to '=xx')
+         and `dotreplace` (which replaces unpermitted characters with '.').
+         The default is `hexencode`. Note: This used to be configured by the
+         `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead.
+* `grandfathered_mxid_source_attribute`: In previous versions of synapse, the mapping from SAML attribute to
+   MXID was always calculated dynamically rather than stored in a table. For backwards- compatibility, we will look for `user_ids`
+   matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this
+   backwards-compatibility lookup. Typically it should be 'uid', but if the attribute maps are changed, it may be necessary to change it.
+   The default is 'uid'. 
+* `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes
+    match particular values. The requirements can be listed under
+   `attribute_requirements` as shown in the example. All of the listed attributes must
+    match for the login to be permitted.
+* `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid`
+   option must be set to the entity to redirect users to.
+   Most deployments only have a single IdP entity and so should omit this option.
+  
+
+Once SAML support is enabled, a metadata file will be exposed at
+`https://<server>:<port>/_synapse/client/saml2/metadata.xml`, which you may be able to
+use to configure your SAML IdP with. Alternatively, you can manually configure
+the IdP to use an ACS location of
+`https://<server>:<port>/_synapse/client/saml2/authn_response`.
+
+Example configuration:
+```yaml
+saml2_config:
+  sp_config:
+    metadata:
+      local: ["saml2/idp.xml"]
+      remote:
+        - url: https://our_idp/metadata.xml
+    accepted_time_diff: 3
+
+    service:
+      sp:
+        allow_unsolicited: true
+
+    # The examples below are just used to generate our metadata xml, and you
+    # may well not need them, depending on your setup. Alternatively you
+    # may need a whole lot more detail - see the pysaml2 docs!
+    description: ["My awesome SP", "en"]
+    name: ["Test SP", "en"]
+
+    ui_info:
+      display_name:
+        - lang: en
+          text: "Display Name is the descriptive name of your service."
+      description:
+        - lang: en
+          text: "Description should be a short paragraph explaining the purpose of the service."
+      information_url:
+        - lang: en
+          text: "https://example.com/terms-of-service"
+      privacy_statement_url:
+        - lang: en
+          text: "https://example.com/privacy-policy"
+      keywords:
+        - lang: en
+          text: ["Matrix", "Element"]
+      logo:
+        - lang: en
+          text: "https://example.com/logo.svg"
+          width: "200"
+          height: "80"
+
+    organization:
+      name: Example com
+      display_name:
+        - ["Example co", "en"]
+      url: "http://example.com"
+
+    contact_person:
+      - given_name: Bob
+        sur_name: "the Sysadmin"
+        email_address": ["admin@example.com"]
+        contact_type": technical
+        
+  saml_session_lifetime: 5m
+  
+  user_mapping_provider:
+    # Below options are intended for the built-in provider, they should be 
+    # changed if using a custom module. 
+    config:
+      mxid_source_attribute: displayName
+      mxid_mapping: dotreplace
+  
+  grandfathered_mxid_source_attribute: upn
+
+  attribute_requirements:
+    - attribute: userGroup
+      value: "staff"
+    - attribute: department
+      value: "sales"
+
+  idp_entityid: 'https://our_idp/entityid'
+```
+---
+Config option: `oidc_providers`
+
+List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration
+and login. See [here](../../openid.md)
+for information on how to configure these options.
+
+For backwards compatibility, it is also possible to configure a single OIDC
+provider via an `oidc_config` setting. This is now deprecated and admins are
+advised to migrate to the `oidc_providers` format. (When doing that migration,
+use `oidc` for the `idp_id` to ensure that existing users continue to be
+recognised.)
+
+Options for each entry include:
+* `idp_id`: a unique identifier for this identity provider. Used internally
+   by Synapse; should be a single word such as 'github'.
+   Note that, if this is changed, users authenticating via that provider
+   will no longer be recognised as the same user!
+   (Use "oidc" here if you are migrating from an old `oidc_config` configuration.)
+
+* `idp_name`: A user-facing name for this identity provider, which is used to
+   offer the user a choice of login mechanisms.
+
+* `idp_icon`: An optional icon for this identity provider, which is presented
+   by clients and Synapse's own IdP picker page. If given, must be an
+   MXC URI of the format mxc://<server-name>/<media-id>. (An easy way to
+   obtain such an MXC URI is to upload an image to an (unencrypted) room
+   and then copy the "url" from the source of the event.)
+
+* `idp_brand`: An optional brand for this identity provider, allowing clients
+   to style the login flow according to the identity provider in question.
+   See the [spec](https://spec.matrix.org/latest/) for possible options here.
+
+* `discover`: set to false to disable the use of the OIDC discovery mechanism
+  to discover endpoints. Defaults to true.
+
+* `issuer`: Required. The OIDC issuer. Used to validate tokens and (if discovery
+   is enabled) to discover the provider's endpoints.
+
+* `client_id`: Required. oauth2 client id to use.
+
+* `client_secret`: oauth2 client secret to use. May be omitted if
+  `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'.
+
+* `client_secret_jwt_key`: Alternative to client_secret: details of a key used
+   to create a JSON Web Token to be used as an OAuth2 client secret. If
+   given, must be a dictionary with the following properties:
+
+  * `key`: a pem-encoded signing key. Must be a suitable key for the
+    algorithm specified. Required unless `key_file` is given.
+
+  * `key_file`: the path to file containing a pem-encoded signing key file.
+     Required unless `key` is given.
+
+  * `jwt_header`: a dictionary giving properties to include in the JWT
+     header. Must include the key `alg`, giving the algorithm used to
+     sign the JWT, such as "ES256", using the JWA identifiers in
+     RFC7518.
+
+  * `jwt_payload`: an optional dictionary giving properties to include in
+    the JWT payload. Normally this should include an `iss` key.
+
+* `client_auth_method`: auth method to use when exchanging the token. Valid
+   values are `client_secret_basic` (default), `client_secret_post` and
+   `none`.
+
+* `scopes`: list of scopes to request. This should normally include the "openid"
+   scope. Defaults to ["openid"].
+
+* `authorization_endpoint`: the oauth2 authorization endpoint. Required if
+   provider discovery is disabled.
+
+* `token_endpoint`: the oauth2 token endpoint. Required if provider discovery is
+   disabled.
+
+* `userinfo_endpoint`: the OIDC userinfo endpoint. Required if discovery is
+   disabled and the 'openid' scope is not requested.
+
+* `jwks_uri`: URI where to fetch the JWKS. Required if discovery is disabled and
+   the 'openid' scope is used.
+
+* `skip_verification`: set to 'true' to skip metadata verification. Use this if
+   you are connecting to a provider that is not OpenID Connect compliant.
+   Defaults to false. Avoid this in production.
+
+* `user_profile_method`: Whether to fetch the user profile from the userinfo
+   endpoint, or to rely on the data returned in the id_token from the `token_endpoint`.
+   Valid values are: `auto` or `userinfo_endpoint`.
+   Defaults to `auto`, which uses the userinfo endpoint if `openid` is
+   not included in `scopes`. Set to `userinfo_endpoint` to always use the
+   userinfo endpoint.
+
+* `allow_existing_users`: set to true to allow a user logging in via OIDC to
+   match a pre-existing account instead of failing. This could be used if
+   switching from password logins to OIDC. Defaults to false.
+
+* `user_mapping_provider`: Configuration for how attributes returned from a OIDC
+   provider are mapped onto a matrix user. This setting has the following
+   sub-properties:
+
+     * `module`: The class name of a custom mapping module. Default is
+       `synapse.handlers.oidc.JinjaOidcMappingProvider`.
+        See https://matrix-org.github.io/synapse/latest/sso_mapping_providers.html#openid-mapping-providers
+        for information on implementing a custom mapping provider.
+
+     * `config`: Configuration for the mapping provider module. This section will
+        be passed as a Python dictionary to the user mapping provider
+        module's `parse_config` method.
+
+        For the default provider, the following settings are available:
+
+       * subject_claim: name of the claim containing a unique identifier
+         for the user. Defaults to 'sub', which OpenID Connect
+         compliant providers should provide.
+
+       * `localpart_template`: Jinja2 template for the localpart of the MXID.
+          If this is not set, the user will be prompted to choose their
+          own username (see the documentation for the `sso_auth_account_details.html` 
+          template). This template can use the `localpart_from_email` filter.
+
+       * `confirm_localpart`: Whether to prompt the user to validate (or
+          change) the generated localpart (see the documentation for the
+          'sso_auth_account_details.html' template), instead of
+          registering the account right away.
+
+       * `display_name_template`: Jinja2 template for the display name to set
+          on first login. If unset, no displayname will be set.
+
+       * `email_template`: Jinja2 template for the email address of the user.
+          If unset, no email address will be added to the account.
+                 
+       * `extra_attributes`: a map of Jinja2 templates for extra attributes
+          to send back to the client during login. Note that these are non-standard and clients will ignore them
+          without modifications.
+
+     When rendering, the Jinja2 templates are given a 'user' variable,
+     which is set to the claims returned by the UserInfo Endpoint and/or
+     in the ID Token.
+
+
+It is possible to configure Synapse to only allow logins if certain attributes 
+match particular values in the OIDC userinfo. The requirements can be listed under
+`attribute_requirements` as shown here:
+```yaml
+attribute_requirements:
+     - attribute: family_name
+       value: "Stephensson"
+     - attribute: groups
+       value: "admin"
+```
+All of the listed attributes must match for the login to be permitted. Additional attributes can be added to
+userinfo by expanding the `scopes` section of the OIDC config to retrieve
+additional information from the OIDC provider.
+
+If the OIDC claim is a list, then the attribute must match any value in the list.
+Otherwise, it must exactly match the value of the claim. Using the example 
+above, the `family_name` claim MUST be "Stephensson", but the `groups`
+claim MUST contain "admin".
+
+Example configuration:
+```yaml
+oidc_providers:
+  # Generic example
+  #
+  - idp_id: my_idp
+    idp_name: "My OpenID provider"
+    idp_icon: "mxc://example.com/mediaid"
+    discover: false
+    issuer: "https://accounts.example.com/"
+    client_id: "provided-by-your-issuer"
+    client_secret: "provided-by-your-issuer"
+    client_auth_method: client_secret_post
+    scopes: ["openid", "profile"]
+    authorization_endpoint: "https://accounts.example.com/oauth2/auth"
+    token_endpoint: "https://accounts.example.com/oauth2/token"
+    userinfo_endpoint: "https://accounts.example.com/userinfo"
+    jwks_uri: "https://accounts.example.com/.well-known/jwks.json"
+    skip_verification: true
+    user_mapping_provider:
+      config:
+        subject_claim: "id"
+        localpart_template: "{{ user.login }}"
+        display_name_template: "{{ user.name }}"
+        email_template: "{{ user.email }}"
+    attribute_requirements:
+      - attribute: userGroup
+        value: "synapseUsers"
+```
+---
+Config option: `cas_config`
+
+Enable Central Authentication Service (CAS) for registration and login.
+Has the following sub-options:
+* `enabled`: Set this to true to enable authorization against a CAS server.
+   Defaults to false.
+* `server_url`: The URL of the CAS authorization endpoint.
+* `displayname_attribute`: The attribute of the CAS response to use as the display name.
+   If no name is given here, no displayname will be set.
+* `required_attributes`:  It is possible to configure Synapse to only allow logins if CAS attributes
+   match particular values. All of the keys given below must exist
+   and the values must match the given value. Alternately if the given value
+   is `None` then any value is allowed (the attribute just must exist).
+   All of the listed attributes must match for the login to be permitted.
+
+Example configuration:
+```yaml
+cas_config:
+  enabled: true
+  server_url: "https://cas-server.com"
+  displayname_attribute: name
+  required_attributes:
+    userGroup: "staff"
+    department: None
+```
+---
+Config option: `sso`
+
+Additional settings to use with single-sign on systems such as OpenID Connect,
+SAML2 and CAS.
+
+Server admins can configure custom templates for pages related to SSO. See 
+[here](../../templates.md) for more information.
+
+Options include:
+* `client_whitelist`: A list of client URLs which are whitelisted so that the user does not
+   have to confirm giving access to their account to the URL. Any client
+   whose URL starts with an entry in the following list will not be subject
+   to an additional confirmation step after the SSO login is completed.
+   WARNING: An entry such as "https://my.client" is insecure, because it
+   will also match "https://my.client.evil.site", exposing your users to
+   phishing attacks from evil.site. To avoid this, include a slash after the
+   hostname: "https://my.client/".
+   The login fallback page (used by clients that don't natively support the
+   required login flows) is whitelisted in addition to any URLs in this list.
+   By default, this list contains only the login fallback page.
+* `update_profile_information`: Use this setting to keep a user's profile fields in sync with information from
+   the identity provider. Currently only syncing the displayname is supported. Fields 
+   are checked on every SSO login, and are updated if necessary.
+   Note that enabling this option will override user profile information,
+   regardless of whether users have opted-out of syncing that
+   information when first signing in. Defaults to false.
+
+
+Example configuration:
+```yaml
+sso:
+    client_whitelist:
+      - https://riot.im/develop
+      - https://my.custom.client/
+    update_profile_information: true
+```
+---
+Config option: `jwt_config`
+
+JSON web token integration. The following settings can be used to make
+Synapse JSON web tokens for authentication, instead of its internal
+password database.
+
+Each JSON Web Token needs to contain a "sub" (subject) claim, which is
+used as the localpart of the mxid.
+
+Additionally, the expiration time ("exp"), not before time ("nbf"),
+and issued at ("iat") claims are validated if present.
+
+Note that this is a non-standard login type and client support is
+expected to be non-existent.
+
+See [here](../../jwt.md) for more.
+
+Additional sub-options for this setting include:
+* `enabled`: Set to true to enable authorization using JSON web
+   tokens. Defaults to false.
+* `secret`: This is either the private shared secret or the public key used to
+   decode the contents of the JSON web token. Required if `enabled` is set to true.
+* `algorithm`: The algorithm used to sign the JSON web token. Supported algorithms are listed at
+   https://pyjwt.readthedocs.io/en/latest/algorithms.html Required if `enabled` is set to true.
+* `subject_claim`: Name of the claim containing a unique identifier for the user.
+   Optional, defaults to `sub`.
+* `issuer`: The issuer to validate the "iss" claim against. Optional. If provided the 
+   "iss" claim will be required and validated for all JSON web tokens.
+* `audiences`: A list of audiences to validate the "aud" claim against. Optional.
+   If provided the "aud" claim will be required and validated for all JSON web tokens.
+   Note that if the "aud" claim is included in a JSON web token then
+   validation will fail without configuring audiences.
+
+Example configuration:
+```yaml
+jwt_config:
+    enabled: true 
+    secret: "provided-by-your-issuer"
+    algorithm: "provided-by-your-issuer"
+    subject_claim: "name_of_claim"
+    issuer: "provided-by-your-issuer"
+    audiences:
+        - "provided-by-your-issuer"
+```
+---
+Config option: `password_config`
+
+Use this setting to enable password-based logins. 
+
+This setting has the following sub-options:
+* `enabled`: Defaults to true.
+* `localdb_enabled`: Set to false to disable authentication against the local password
+   database. This is ignored if `enabled` is false, and is only useful
+   if you have other `password_providers`. Defaults to true. 
+* `pepper`: Set the value here to a secret random string for extra security. # Uncomment and change to a secret random string for extra security.
+   DO NOT CHANGE THIS AFTER INITIAL SETUP!
+* `policy`: Define and enforce a password policy, such as minimum lengths for passwords, etc. 
+   Each parameter is optional. This is an implementation of MSC2000. Parameters are as follows:
+   * `enabled`: Defaults to false. Set to true to enable.
+   * `minimum_length`: Minimum accepted length for a password. Defaults to 0.
+   * `require_digit`: Whether a password must contain at least one digit.
+      Defaults to false.
+   * `require_symbol`: Whether a password must contain at least one symbol.
+      A symbol is any character that's not a number or a letter. Defaults to false.
+   * `require_lowercase`: Whether a password must contain at least one lowercase letter.
+      Defaults to false.
+   * `require_uppercase`: Whether a password must contain at least one uppercase letter.
+      Defaults to false.
+      
+
+Example configuration:
+```yaml
+password_config:
+   enabled: false
+   localdb_enabled: false
+   pepper: "EVEN_MORE_SECRET"
+
+   policy:
+      enabled: true
+      minimum_length: 15
+      require_digit: true
+      require_symbol: true
+      require_lowercase: true
+      require_uppercase: true
+```
+---
+Config option: `ui_auth`
+
+The amount of time to allow a user-interactive authentication session to be active.
+
+This defaults to 0, meaning the user is queried for their credentials 
+before every action, but this can be overridden to allow a single
+validation to be re-used.  This weakens the protections afforded by
+the user-interactive authentication process, by allowing for multiple
+(and potentially different) operations to use the same validation session.
+
+This is ignored for potentially "dangerous" operations (including
+deactivating an account, modifying an account password, and
+adding a 3PID).
+
+Use the `session_timeout` sub-option here to change the time allowed for credential validation.
+
+Example configuration:
+```yaml
+ui_auth:
+    session_timeout: "15s"
+```
+---
+Config option: `email`
+
+Configuration for sending emails from Synapse.
+
+Server admins can configure custom templates for email content. See
+[here](../../templates.md) for more information.
+
+This setting has the following sub-options:
+* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'.
+* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 25.
+* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no
+   authentication is attempted.
+* `require_transport_security`: Set to true to require TLS transport security for SMTP.
+   By default, Synapse will connect over plain text, and will then switch to
+   TLS via STARTTLS *if the SMTP server supports it*. If this option is set,
+   Synapse will refuse to connect unless the server supports STARTTLS.
+* `enable_tls`: By default, if the server supports TLS, it will be used, and the server
+   must present a certificate that is valid for 'smtp_host'. If this option
+   is set to false, TLS will not be used.
+* `notif_from`: defines the "From" address to use when sending emails.
+    It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name,
+    which is normally set in `app_name`, but may be overridden by the
+    Matrix client application. Note that the placeholder must be written '%(app)s', including the
+    trailing 's'.
+* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email
+   subjects. It defaults to 'Matrix'.
+* `enable_notifs`: Set to true to enable sending emails for messages that the user
+   has missed. Disabled by default.
+* `notif_for_new_users`: Set to false to disable automatic subscription to email
+   notifications for new users. Enabled by default.
+* `client_base_url`: Custom URL for client links within the email notifications. By default
+   links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`;
+   the old name is still supported for backwards-compatibility but is now deprecated.)
+* `validation_token_lifetime`: Configures the time that a validation email will expire after sending.
+   Defaults to 1h.
+* `invite_client_location`: The web client location to direct users to during an invite. This is passed
+   to the identity server as the `org.matrix.web_client_location` key. Defaults
+   to unset, giving no guidance to the identity server.
+* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will
+   be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application.
+   In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname
+   of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the
+   message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will
+   can use the '%(server_name)s' placeholder, which will be replaced by the value of the
+   `server_name` setting in your Synapse configuration.
+   
+   Here is a list of subjects for notification emails that can be set: 
+     * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a
+        room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."
+     * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a
+        room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..."
+     * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in
+        a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..."
+     * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a
+        name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..."
+     * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms. 
+        Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..."
+     * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in
+        multiple rooms. This is similar to the setting above except it's used when
+        the room in which the notification was triggered has no name. Defaults to 
+        "[%(app)s] You have messages on %(app)s from %(person)s and others..."
+     * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name. 
+        Defaults to  "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."
+     * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a
+        name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..."
+     * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset"
+     * `email_validation`: Subject to use when sending a verification email to assert an address's
+        ownership. Defaults to "[%(server_name)s] Validate your email"
+
+Example configuration:
+```yaml
+email:
+  smtp_host: mail.server
+  smtp_port: 587
+  smtp_user: "exampleusername"
+  smtp_pass: "examplepassword"
+  require_transport_security: true
+  enable_tls: false
+  notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>"
+  app_name: my_branded_matrix_server
+  enable_notifs: true
+  notif_for_new_users: false
+  client_base_url: "http://localhost/riot"
+  validation_token_lifetime: 15m
+  invite_client_location: https://app.element.io
+
+  subjects:
+    message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."
+    message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..."
+    messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..."
+    messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..."
+    messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..."
+    messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..."
+    invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."
+    invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..."
+    password_reset: "[%(server_name)s] Password reset"
+    email_validation: "[%(server_name)s] Validate your email"
+```
+---
+## Push ##
+Configuration settings related to push notifications
+
+---
+Config option: `push`
+
+This setting defines options for push notifications. 
+
+This option has a number of sub-options. They are as follows:
+* `include_content`: Clients requesting push notifications can either have the body of
+   the message sent in the notification poke along with other details
+   like the sender, or just the event ID and room ID (`event_id_only`).
+   If clients choose the to have the body sent, this option controls whether the
+   notification request includes the content of the event (other details
+   like the sender are still included). If `event_id_only` is enabled, it
+   has no effect.
+   For modern android devices the notification content will still appear
+   because it is loaded by the app. iPhone, however will send a
+   notification saying only that a message arrived and who it came from.
+   Defaults to true. Set to false to only include the event ID and room ID in push notification payloads.
+* `group_unread_count_by_room: false`: When a push notification is received, an unread count is also sent.
+   This number can either be calculated as the number of unread messages  for the user, or the number of *rooms* the 
+   user has unread messages in. Defaults to true, meaning push clients will see the number of
+   rooms with unread messages in them. Set to false to instead send the number
+   of unread messages.
+
+Example configuration:
+```yaml
+push:
+  include_content: false
+  group_unread_count_by_room: false
+```
+---
+## Rooms ##
+Config options relating to rooms.
+
+---
+Config option: `encryption_enabled_by_default`
+
+Controls whether locally-created rooms should be end-to-end encrypted by
+default.
+
+Possible options are "all", "invite", and "off". They are defined as:
+
+* "all": any locally-created room
+* "invite": any room created with the `private_chat` or `trusted_private_chat`
+   room creation presets
+* "off": this option will take no effect
+
+The default value is "off".
+
+Note that this option will only affect rooms created after it is set. It
+will also not affect rooms created by other servers.
+
+Example configuration:
+```yaml
+encryption_enabled_by_default_for_room_type: invite
+```
+---
+Config option: `enable_group_creation`
+
+Set to true to allow non-server-admin users to create groups on this server
+
+Example configuration:
+```yaml
+enable_group_creation: true
+```
+---
+Config option: `group_creation_prefix`
+
+If enabled/present, non-server admins can only create groups with local parts
+starting with this prefix.
+
+Example configuration:
+```yaml
+group_creation_prefix: "unofficial_"
+```
+---
+Config option: `user_directory`
+
+This setting defines options related to the user directory. 
+
+This option has the following sub-options:
+* `enabled`:  Defines whether users can search the user directory. If false then
+   empty responses are returned to all queries. Defaults to true.
+* `search_all_users`: Defines whether to search all users visible to your HS when searching
+   the user directory. If false, search results will only contain users
+    visible in public rooms and users sharing a room with the requester.
+    Defaults to false.
+    NB. If you set this to true, and the last time the user_directory search
+    indexes were (re)built was before Synapse 1.44, you'll have to
+    rebuild the indexes in order to search through all known users.
+    These indexes are built the first time Synapse starts; admins can
+    manually trigger a rebuild via API following the instructions at
+         https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/background_updates.html#run
+    Set to true to return search results containing all known users, even if that
+    user does not share a room with the requester.
+* `prefer_local_users`: Defines whether to prefer local users in search query results.
+   If set to true, local users are more likely to appear above remote users when searching the 
+   user directory. Defaults to false.
+
+Example configuration:
+```yaml
+user_directory:
+    enabled: false
+    search_all_users: true
+    prefer_local_users: true
+```
+---
+Config option: `user_consent`
+
+For detailed instructions on user consent configuration, see [here](../../consent_tracking.md).
+
+Parts of this section are required if enabling the `consent` resource under
+`listeners`, in particular `template_dir` and `version`. # TODO: link `listeners`
+
+* `template_dir`: gives the location of the templates for the HTML forms.
+  This directory should contain one subdirectory per language (eg, `en`, `fr`),
+  and each language directory should contain the policy document (named as
+  <version>.html) and a success page (success.html).
+
+* `version`: specifies the 'current' version of the policy document. It defines
+   the version to be served by the consent resource if there is no 'v'
+   parameter.
+
+* `server_notice_content`: if enabled, will send a user a "Server Notice"
+   asking them to consent to the privacy policy. The `server_notices` section ##TODO: link
+   must also be configured for this to work. Notices will *not* be sent to
+   guest users unless `send_server_notice_to_guests` is set to true.
+
+* `block_events_error`, if set, will block any attempts to send events
+   until the user consents to the privacy policy. The value of the setting is
+   used as the text of the error.
+
+* `require_at_registration`, if enabled, will add a step to the registration
+   process, similar to how captcha works. Users will be required to accept the
+   policy before their account is created.
+
+* `policy_name` is the display name of the policy users will see when registering
+   for an account. Has no effect unless `require_at_registration` is enabled.
+   Defaults to "Privacy Policy".
+
+Example configuration:
+```yaml
+user_consent:
+  template_dir: res/templates/privacy
+  version: 1.0
+  server_notice_content:
+    msgtype: m.text
+    body: >-
+      To continue using this homeserver you must review and agree to the
+      terms and conditions at %(consent_uri)s
+  send_server_notice_to_guests: true
+  block_events_error: >-
+    To continue using this homeserver you must review and agree to the
+    terms and conditions at %(consent_uri)s
+  require_at_registration: false
+  policy_name: Privacy Policy
+```
+---
+Config option: `stats`
+
+Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md)
+for more. 
+
+* `enabled`: Set to false to disable room and user statistics. Note that doing
+   so may cause certain features (such as the room directory) not to work
+   correctly. Defaults to true. 
+
+Example configuration:
+```yaml
+stats:  
+  enabled: false
+```
+---
+Config option: `server_notices`
+
+Use this setting to enable a room which can be used to send notices
+from the server to users. It is a special room which users cannot leave; notices
+in the room come from a special "notices" user id.
+
+If you use this setting, you *must* define the `system_mxid_localpart`
+sub-setting, which defines the id of the user which will be used to send the
+notices.
+
+Sub-options for this setting include:
+* `system_mxid_display_name`: set the display name of the "notices" user
+* `system_mxid_avatar_url`: set the avatar for the "notices" user
+* `room_name`: set the room name of the server notices room
+
+Example configuration:
+```yaml
+server_notices:
+  system_mxid_localpart: notices
+  system_mxid_display_name: "Server Notices"
+  system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
+  room_name: "Server Notices"
+```
+---
+Config option: `enable_room_list_search`
+
+Set to false to disable searching the public room list. When disabled
+blocks searching local and remote room lists for local and remote
+users by always returning an empty list for all queries. Defaults to true. 
+
+Example configuration:
+```yaml
+enable_room_list_search: false
+```
+---
+Config option: `alias_creation`
+
+The `alias_creation` option controls who is allowed to create aliases
+on this server.
+
+The format of this option is a list of rules that contain globs that
+match against user_id, room_id and the new alias (fully qualified with
+server name). The action in the first rule that matches is taken,
+which can currently either be "allow" or "deny".
+
+Missing user_id/room_id/alias fields default to "*".
+
+If no rules match the request is denied. An empty list means no one
+can create aliases.
+
+Options for the rules include:
+* `user_id`: Matches against the creator of the alias. Defaults to "*".
+* `alias`: Matches against the alias being created. Defaults to "*".
+* `room_id`: Matches against the room ID the alias is being pointed at. Defaults to "*"
+* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. 
+
+Example configuration:
+```yaml
+alias_creation_rules:
+  - user_id: "bad_user"
+    alias: "spammy_alias"
+    room_id: "*"
+    action: deny
+```
+---
+Config options: `room_list_publication_rules`
+
+The `room_list_publication_rules` option controls who can publish and
+which rooms can be published in the public room list.
+
+The format of this option is the same as that for
+`alias_creation_rules`.
+
+If the room has one or more aliases associated with it, only one of
+the aliases needs to match the alias rule. If there are no aliases
+then only rules with `alias: *` match.
+
+If no rules match the request is denied. An empty list means no one
+can publish rooms.
+
+Options for the rules include:
+* `user_id`: Matches against the creator of the alias. Defaults to "*".
+* `alias`: Matches against any current local or canonical aliases associated with the room. Defaults to "*".
+* `room_id`: Matches against the room ID being published. Defaults to "*".
+* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. 
+
+Example configuration:
+```yaml
+room_list_publication_rules:
+  - user_id: "*"
+    alias: "*"
+    room_id: "*"
+    action: allow
+```
+---
+## Opentracing ##
+Configuration options related to Opentracing support.
+
+---
+Config option: `opentracing`
+
+These settings enable and configure opentracing, which implements distributed tracing.
+This allows you to observe the causal chains of events across servers
+including requests, key lookups etc., across any server running
+synapse or any other services which support opentracing
+(specifically those implemented with Jaeger).
+
+Sub-options include:
+* `enabled`: whether tracing is enabled. Set to true to enable. Disabled by default.
+* `homeserver_whitelist`: The list of homeservers we wish to send and receive span contexts and span baggage.
+   See [here](../../opentracing.md) for more. 
+   This is a list of regexes which are matched against the `server_name` of the homeserver.
+   By default, it is empty, so no servers are matched.
+* `force_tracing_for_users`: # A list of the matrix IDs of users whose requests will always be traced,
+   even if the tracing system would otherwise drop the traces due to probabilistic sampling.
+    By default, the list is empty.
+* `jaeger_config`: Jaeger can be configured to sample traces at different rates.
+   All configuration options provided by Jaeger can be set here. Jaeger's configuration is 
+   mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/).
+
+Example configuration:
+```yaml
+opentracing:
+    enabled: true
+    homeserver_whitelist:
+      - ".*"
+    force_tracing_for_users:
+      - "@user1:server_name"
+      - "@user2:server_name"
+
+    jaeger_config:
+      sampler:
+        type: const
+        param: 1
+      logging:
+        false
+```
+---
+## Workers ##
+Configuration options related to workers.
+
+---
+Config option: `send_federation`
+
+Controls sending of outbound federation transactions on the main process.
+Set to false if using a federation sender worker. Defaults to true. 
+
+Example configuration:
+```yaml
+send_federation: false
+```
+---
+Config option: `federation_sender_instances`
+
+It is possible to run multiple federation sender workers, in which case the
+work is balanced across them. Use this setting to list the senders. 
+
+This configuration setting must be shared between all federation sender workers, and if
+changed all federation sender workers must be stopped at the same time and then
+started, to ensure that all instances are running with the same config (otherwise
+events may be dropped). 
+
+Example configuration:
+```yaml
+federation_sender_instances:
+  - federation_sender1
+```
+---
+Config option: `instance_map`
+
+When using workers this should be a map from worker name to the
+HTTP replication listener of the worker, if configured. 
+
+Example configuration:
+```yaml
+instance_map:
+  worker1:
+    host: localhost
+    port: 8034
+```
+---
+Config option: `stream_writers`
+
+Experimental: When using workers you can define which workers should
+handle event persistence and typing notifications. Any worker
+specified here must also be in the `instance_map`.
+
+Example configuration:
+```yaml
+stream_writers:
+  events: worker1
+  typing: worker1
+```
+---
+Config option: `run_background_task_on`
+
+The worker that is used to run background tasks (e.g. cleaning up expired
+data). If not provided this defaults to the main process.
+
+Example configuration:
+```yaml
+run_background_tasks_on: worker1
+```
+---
+Config option: `worker_replication_secret`
+
+A shared secret used by the replication APIs to authenticate HTTP requests
+from workers.
+
+By default this is unused and traffic is not authenticated.
+
+Example configuration:
+```yaml
+worker_replication_secret: "secret_secret"
+```
+Config option: `redis`
+
+Configuration for Redis when using workers. This *must* be enabled when
+using workers (unless using old style direct TCP configuration).
+This setting has the following sub-options:
+* `enabled`: whether to use Redis support. Defaults to false. 
+* `host` and `port`: Optional host and port to use to connect to redis. Defaults to
+   localhost and 6379
+* `password`: Optional password if configured on the Redis instance.
+
+Example configuration:
+```yaml
+redis:
+  enabled: true
+  host: localhost
+  port: 6379
+  password: <secret_password>
+```
+## Background Updates ##
+Configuration settings related to background updates. 
+
+---
+Config option: `background_updates`
+
+Background updates are database updates that are run in the background in batches.
+The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to
+sleep can all be configured. This is helpful to speed up or slow down the updates.
+This setting has the following sub-options:
+* `background_update_duration_ms`: How long in milliseconds to run a batch of background updates for. Defaults to 100. 
+   Set a different time to change the default.
+* `sleep_enabled`: Whether to sleep between updates. Defaults to true. Set to false to change the default.
+* `sleep_duration_ms`: If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000.
+   Set a duration to change the default.
+* `min_batch_size`: Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1.
+   Set a size to change the default.
+* `default_batch_size`: The batch size to use for the first iteration of a new background update. The default is 100.
+   Set a size to change the default.
+
+Example configuration: 
+```yaml
+background_updates:
+    background_update_duration_ms: 500
+    sleep_enabled: false
+    sleep_duration_ms: 300
+    min_batch_size: 10
+    default_batch_size: 50
+```
diff --git a/docs/website_files/table-of-contents.js b/docs/website_files/table-of-contents.js
index 0de5960b2..772da97fb 100644
--- a/docs/website_files/table-of-contents.js
+++ b/docs/website_files/table-of-contents.js
@@ -75,6 +75,20 @@ function setTocEntry() {
  * Populate sidebar on load
  */
 window.addEventListener('load', () => {
+    // Prevent rendering the table of contents of the "print book" page, as it
+    // will end up being rendered into the output (in a broken-looking way)
+
+    // Get the name of the current page (i.e. 'print.html')
+    const pageNameExtension = window.location.pathname.split('/').pop();
+
+    // Split off the extension (as '.../print' is also a valid page name), which
+    // should result in 'print'
+    const pageName = pageNameExtension.split('.')[0];
+    if (pageName === "print") {
+        // Don't render the table of contents on this page
+        return;
+    }
+
     // Only create table of contents if there is more than one header on the page
     if (headers.length <= 1) {
         return;
diff --git a/docs/workers.md b/docs/workers.md
index caef44b61..afdcd785e 100644
--- a/docs/workers.md
+++ b/docs/workers.md
@@ -138,22 +138,7 @@ as the `listeners` option in the shared config.
 For example:
 
 ```yaml
-worker_app: synapse.app.generic_worker
-worker_name: worker1
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
- - type: http
-   port: 8083
-   resources:
-     - names:
-       - client
-       - federation
-
-worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml
+{{#include systemd-with-workers/workers/generic_worker.yaml}}
 ```
 
 ...is a full configuration for a generic worker instance, which will expose a
@@ -343,9 +328,9 @@ effects of bursts of events from that bridge on events sent by normal users.
 
 #### Stream writers
 
-Additionally, there is *experimental* support for moving writing of specific
-streams (such as events) off of the main process to a particular worker. (This
-is only supported with Redis-based replication.)
+Additionally, the writing of specific streams (such as events) can be moved off
+of the main process to a particular worker.
+(This is only supported with Redis-based replication.)
 
 To enable this, the worker must have a HTTP replication listener configured,
 have a `worker_name` and be listed in the `instance_map` config. The same worker
@@ -365,6 +350,12 @@ stream_writers:
     events: event_persister1
 ```
 
+An example for a stream writer instance:
+
+```yaml
+{{#include systemd-with-workers/workers/event_persister.yaml}}
+```
+
 Some of the streams have associated endpoints which, for maximum efficiency, should
 be routed to the workers handling that stream. See below for the currently supported
 streams and the endpoints associated with them:
@@ -422,7 +413,7 @@ the stream writer for the `presence` stream:
 
 #### Background tasks
 
-There is also *experimental* support for moving background tasks to a separate
+There is also support for moving background tasks to a separate
 worker. Background tasks are run periodically or started via replication. Exactly
 which tasks are configured to run depends on your Synapse configuration (e.g. if
 stats is enabled).
@@ -438,6 +429,12 @@ run_background_tasks_on: background_worker
 You might also wish to investigate the `update_user_directory` and
 `media_instance_running_background_jobs` settings.
 
+An example for a dedicated background worker instance:
+
+```yaml
+{{#include systemd-with-workers/workers/background_worker.yaml}}
+```
+
 ### `synapse.app.pusher`
 
 Handles sending push notifications to sygnal and email. Doesn't handle any
@@ -617,14 +614,14 @@ The following shows an example setup using Redis and a reverse proxy:
 |   Main       |  |   Generic    |  |   Generic    |  |  Event       |
 |   Process    |  |   Worker 1   |  |   Worker 2   |  |  Persister   |
 +--------------+  +--------------+  +--------------+  +--------------+
-      ^    ^          |   ^   |         |   ^   |          ^    ^
-      |    |          |   |   |         |   |   |          |    |
-      |    |          |   |   |  HTTP   |   |   |          |    |
-      |    +----------+<--|---|---------+   |   |          |    |
-      |                   |   +-------------|-->+----------+    |
-      |                   |                 |                   |
-      |                   |                 |                   |
-      v                   v                 v                   v
-====================================================================
+      ^    ^          |   ^   |         |   ^   |         |   ^   ^
+      |    |          |   |   |         |   |   |         |   |   |
+      |    |          |   |   |  HTTP   |   |   |         |   |   |
+      |    +----------+<--|---|---------+<--|---|---------+   |   |
+      |                   |   +-------------|-->+-------------+   |
+      |                   |                 |                     |
+      |                   |                 |                     |
+      v                   v                 v                     v
+======================================================================
                                                          Redis pub/sub channel
 ```
diff --git a/mypy.ini b/mypy.ini
index 5246f987c..a663bf697 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -13,7 +13,6 @@ no_implicit_optional = True
 files =
   docker/,
   scripts-dev/,
-  setup.py,
   synapse/,
   tests/
 
@@ -234,8 +233,8 @@ disallow_untyped_defs = True
 ;; The `typeshed` project maintains stubs here:
 ;;     https://github.com/python/typeshed/tree/master/stubs
 ;; and for each package `foo` there's a corresponding `types-foo` package on PyPI,
-;; which we can pull in as a dev dependency by adding to `setup.py`'s
-;; `CONDITIONAL_REQUIREMENTS["mypy"]` list.
+;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s
+;; `[tool.poetry.dev-dependencies]` list.
 
 [mypy-authlib.*]
 ignore_missing_imports = True
diff --git a/poetry.lock b/poetry.lock
index c11e2c684..8c7af1fa1 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,11 +1,3 @@
-[[package]]
-name = "appdirs"
-version = "1.4.4"
-description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
-category = "dev"
-optional = false
-python-versions = "*"
-
 [[package]]
 name = "attrs"
 version = "21.4.0"
@@ -49,17 +41,6 @@ six = "*"
 [package.extras]
 visualize = ["graphviz (>0.5.1)", "Twisted (>=16.1.1)"]
 
-[[package]]
-name = "baron"
-version = "0.10.1"
-description = "Full Syntax Tree for python to make writing refactoring code a realist task"
-category = "dev"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-rply = "*"
-
 [[package]]
 name = "bcrypt"
 version = "3.2.0"
@@ -720,7 +701,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock
 
 [[package]]
 name = "prometheus-client"
-version = "0.13.1"
+version = "0.14.0"
 description = "Python client for the Prometheus monitoring system."
 category = "main"
 optional = false
@@ -984,20 +965,6 @@ Pygments = ">=2.5.1"
 [package.extras]
 md = ["cmarkgfm (>=0.8.0)"]
 
-[[package]]
-name = "redbaron"
-version = "0.9.2"
-description = "Abstraction on top of baron, a FST for python to make writing refactoring code a realistic task"
-category = "dev"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-baron = ">=0.7"
-
-[package.extras]
-notebook = ["pygments"]
-
 [[package]]
 name = "requests"
 version = "2.27.1"
@@ -1038,17 +1005,6 @@ python-versions = ">=3.7"
 [package.extras]
 idna2008 = ["idna"]
 
-[[package]]
-name = "rply"
-version = "0.7.8"
-description = "A pure Python Lex/Yacc that works with RPython"
-category = "dev"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-appdirs = "*"
-
 [[package]]
 name = "secretstorage"
 version = "3.3.1"
@@ -1288,7 +1244,7 @@ urllib3 = ">=1.26.0"
 
 [[package]]
 name = "twisted"
-version = "22.2.0"
+version = "22.4.0"
 description = "An asynchronous networking framework written in Python"
 category = "main"
 optional = false
@@ -1308,19 +1264,20 @@ typing-extensions = ">=3.6.5"
 "zope.interface" = ">=4.4.2"
 
 [package.extras]
-all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
 conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"]
+conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"]
 contextvars = ["contextvars (>=2.4,<3)"]
 dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"]
 dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"]
-http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"]
-macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
-mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"]
-osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"]
+macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"]
+osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
 serial = ["pyserial (>=3.0)", "pywin32 (!=226)"]
 test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"]
 tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"]
-windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
 
 [[package]]
 name = "twisted-iocpsupport"
@@ -1596,13 +1553,9 @@ url_preview = ["lxml"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "964ad29eaf7fd02749a4e735818f3bc0ba729c2f4b9e3213f0daa02643508b16"
+content-hash = "f482a4f594a165dfe01ce253a22510d5faf38647ab0dcebc35789350cafd9bf0"
 
 [metadata.files]
-appdirs = [
-    {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
-    {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
-]
 attrs = [
     {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"},
     {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
@@ -1615,10 +1568,6 @@ automat = [
     {file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"},
     {file = "Automat-20.2.0.tar.gz", hash = "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33"},
 ]
-baron = [
-    {file = "baron-0.10.1-py2.py3-none-any.whl", hash = "sha256:befb33f4b9e832c7cd1e3cf0eafa6dd3cb6ed4cb2544245147c019936f4e0a8a"},
-    {file = "baron-0.10.1.tar.gz", hash = "sha256:af822ad44d4eb425c8516df4239ac4fdba9fdb398ef77e4924cd7c9b4045bc2f"},
-]
 bcrypt = [
     {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd"},
     {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"},
@@ -2229,8 +2178,8 @@ platformdirs = [
     {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"},
 ]
 prometheus-client = [
-    {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"},
-    {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"},
+    {file = "prometheus_client-0.14.0-py3-none-any.whl", hash = "sha256:f4aba3fdd1735852049f537c1f0ab177159b7ab76f271ecc4d2f45aa2a1d01f2"},
+    {file = "prometheus_client-0.14.0.tar.gz", hash = "sha256:8f7a922dd5455ad524b6ba212ce8eb2b4b05e073f4ec7218287f88b1cac34750"},
 ]
 psycopg2 = [
     {file = "psycopg2-2.9.3-cp310-cp310-win32.whl", hash = "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362"},
@@ -2411,10 +2360,6 @@ readme-renderer = [
     {file = "readme_renderer-33.0-py3-none-any.whl", hash = "sha256:f02cee0c4de9636b5a62b6be50c9742427ba1b956aad1d938bfb087d0d72ccdf"},
     {file = "readme_renderer-33.0.tar.gz", hash = "sha256:e3b53bc84bd6af054e4cc1fe3567dc1ae19f554134221043a3f8c674e22209db"},
 ]
-redbaron = [
-    {file = "redbaron-0.9.2-py2.py3-none-any.whl", hash = "sha256:d01032b6a848b5521a8d6ef72486315c2880f420956870cdd742e2b5a09b9bab"},
-    {file = "redbaron-0.9.2.tar.gz", hash = "sha256:472d0739ca6b2240bb2278ae428604a75472c9c12e86c6321e8c016139c0132f"},
-]
 requests = [
     {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
     {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
@@ -2427,10 +2372,6 @@ rfc3986 = [
     {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"},
     {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"},
 ]
-rply = [
-    {file = "rply-0.7.8-py2.py3-none-any.whl", hash = "sha256:28ffd11d656c48aeb8c508eb382acd6a0bd906662624b34388751732a27807e7"},
-    {file = "rply-0.7.8.tar.gz", hash = "sha256:2a808ac25a4580a9991fc304d64434e299a8fc75760574492f242cbb5bb301c9"},
-]
 secretstorage = [
     {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"},
     {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"},
@@ -2596,8 +2537,8 @@ twine = [
     {file = "twine-3.8.0.tar.gz", hash = "sha256:8efa52658e0ae770686a13b675569328f1fba9837e5de1867bfe5f46a9aefe19"},
 ]
 twisted = [
-    {file = "Twisted-22.2.0-py3-none-any.whl", hash = "sha256:5c63c149eb6b8fe1e32a0215b1cef96fabdba04f705d8efb9174b1ccf5b49d49"},
-    {file = "Twisted-22.2.0.tar.gz", hash = "sha256:57f32b1f6838facb8c004c89467840367ad38e9e535f8252091345dba500b4f2"},
+    {file = "Twisted-22.4.0-py3-none-any.whl", hash = "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"},
+    {file = "Twisted-22.4.0.tar.gz", hash = "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680"},
 ]
 twisted-iocpsupport = [
     {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"},
diff --git a/pyproject.toml b/pyproject.toml
index 65f5a5f59..cc4908fa2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,7 +54,7 @@ skip_gitignore = true
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.57.1"
+version = "1.58.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -270,7 +270,6 @@ idna = ">=2.5"
 
 # The following are used by the release script
 click = "==8.1.0"
-redbaron = "==0.9.2"
 GitPython = "==3.1.14"
 commonmark = "==0.9.1"
 pygithub = "==1.55"
@@ -280,5 +279,5 @@ twine = "*"
 towncrier = ">=18.6.0rc1"
 
 [build-system]
-requires = ["setuptools"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py
index 7ff96a1ee..e3e687868 100755
--- a/scripts-dev/build_debian_packages.py
+++ b/scripts-dev/build_debian_packages.py
@@ -26,6 +26,7 @@ DISTS = (
     "debian:sid",
     "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
     "ubuntu:impish",  # 21.10  (EOL 2022-07)
+    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
 )
 
 DESC = """\
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index d34e9f355..e0feba05f 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -64,4 +64,4 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"
-go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 "$@" ./tests/...
+go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "$@" ./tests/...
diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py
index c72e19f61..079d2f5ed 100755
--- a/scripts-dev/federation_client.py
+++ b/scripts-dev/federation_client.py
@@ -124,7 +124,12 @@ def request(
     authorization_headers = []
 
     for key, sig in signed_json["signatures"][origin_name].items():
-        header = 'X-Matrix origin=%s,key="%s",sig="%s"' % (origin_name, key, sig)
+        header = 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' % (
+            origin_name,
+            key,
+            sig,
+            destination,
+        )
         authorization_headers.append(header.encode("ascii"))
         print("Authorization: %s" % header, file=sys.stderr)
 
diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh
index 4698d2d5b..377348b10 100755
--- a/scripts-dev/lint.sh
+++ b/scripts-dev/lint.sh
@@ -79,8 +79,20 @@ else
   # If we were not asked to lint changed files, and no paths were found as a result,
   # then lint everything!
   if [[ -z ${files+x} ]]; then
-    # Lint all source code files and directories
-      files=( "." )
+      # CI runs each linter on the entire checkout, e.g. `black .`. So don't
+      # rely on this list to *find* lint targets if that misses a file; instead;
+      # use it to exclude files from linters when this can't be done by config.
+      #
+      # To check which files the linters examine, use:
+      #     black --verbose . 2>&1 | \grep -v ignored
+      #     isort --show-files .
+      #     flake8 --verbose .  # This isn't a great option
+      #     mypy has explicit config in mypy.ini; there is also mypy --verbose
+      files=(
+          "synapse" "docker" "tests"
+          "scripts-dev"
+          "contrib" "synmark" "stubs" ".ci"
+      )
   fi
 fi
 
diff --git a/scripts-dev/release.py b/scripts-dev/release.py
index 685fa32b0..9d7c7c445 100755
--- a/scripts-dev/release.py
+++ b/scripts-dev/release.py
@@ -25,13 +25,12 @@ import sys
 import urllib.request
 from os import path
 from tempfile import TemporaryDirectory
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 import attr
 import click
 import commonmark
 import git
-import redbaron
 from click.exceptions import ClickException
 from github import Github
 from packaging import version
@@ -69,11 +68,12 @@ def cli():
         # ... wait for assets to build ...
 
         ./scripts-dev/release.py publish
+
         ./scripts-dev/release.py upload
 
         # Optional: generate some nice links for the announcement
 
-        ./scripts-dev/release.py upload
+        ./scripts-dev/release.py announce
 
     If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
     `tag`/`publish` command, then a new draft release will be created/published.
@@ -99,7 +99,7 @@ def prepare():
     repo.remote().fetch()
 
     # Get the current version and AST from root Synapse module.
-    current_version, parsed_synapse_ast, version_node = parse_version_from_module()
+    current_version = get_package_version()
 
     # Figure out what sort of release we're doing and calcuate the new version.
     rc = click.confirm("RC", default=True)
@@ -161,7 +161,7 @@ def prepare():
         click.get_current_context().abort()
 
     # Switch to the release branch.
-    parsed_new_version = version.parse(new_version)
+    parsed_new_version: version.Version = version.parse(new_version)
 
     # We assume for debian changelogs that we only do RCs or full releases.
     assert not parsed_new_version.is_devrelease
@@ -206,17 +206,15 @@ def prepare():
         # Create the new release branch
         release_branch = repo.create_head(release_branch_name, commit=base_branch)
 
-    # Switch to the release branch and ensure its up to date.
+    # Switch to the release branch and ensure it's up to date.
     repo.git.checkout(release_branch_name)
     update_branch(repo)
 
-    # Update the `__version__` variable and write it back to the file.
-    version_node.value = '"' + new_version + '"'
-    with open("synapse/__init__.py", "w") as f:
-        f.write(parsed_synapse_ast.dumps())
+    # Update the version specified in pyproject.toml.
+    subprocess.check_output(["poetry", "version", new_version])
 
     # Generate changelogs.
-    generate_and_write_changelog(current_version)
+    generate_and_write_changelog(current_version, new_version)
 
     # Generate debian changelogs
     if parsed_new_version.pre is not None:
@@ -229,7 +227,7 @@ def prepare():
         debian_version = new_version
 
     run_until_successful(
-        f'dch -M -v {debian_version} "New synapse release {debian_version}."',
+        f'dch -M -v {debian_version} "New Synapse release {new_version}."',
         shell=True,
     )
     run_until_successful('dch -M -r -D stable ""', shell=True)
@@ -283,7 +281,7 @@ def tag(gh_token: Optional[str]):
     repo.remote().fetch()
 
     # Find out the version and tag name.
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     # Check we haven't released this version.
@@ -361,7 +359,7 @@ def publish(gh_token: str):
     if repo.is_dirty():
         raise click.ClickException("Uncommitted changes exist.")
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     if not click.confirm(f"Publish {tag_name}?", default=True):
@@ -395,7 +393,7 @@ def publish(gh_token: str):
 def upload():
     """Upload release to pypi."""
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     pypi_asset_names = [
@@ -423,7 +421,7 @@ def upload():
 def announce():
     """Generate markdown to announce the release."""
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     click.echo(
@@ -454,37 +452,11 @@ Announce the release in
         )
 
 
-def parse_version_from_module() -> Tuple[
-    version.Version, redbaron.RedBaron, redbaron.Node
-]:
-    # Parse the AST and load the `__version__` node so that we can edit it
-    # later.
-    with open("synapse/__init__.py") as f:
-        red = redbaron.RedBaron(f.read())
-
-    version_node = None
-    for node in red:
-        if node.type != "assignment":
-            continue
-
-        if node.target.type != "name":
-            continue
-
-        if node.target.value != "__version__":
-            continue
-
-        version_node = node
-        break
-
-    if not version_node:
-        print("Failed to find '__version__' definition in synapse/__init__.py")
-        sys.exit(1)
-
-    # Parse the current version.
-    current_version = version.parse(version_node.value.value.strip('"'))
-    assert isinstance(current_version, version.Version)
-
-    return current_version, red, version_node
+def get_package_version() -> version.Version:
+    version_string = subprocess.check_output(["poetry", "version", "--short"]).decode(
+        "utf-8"
+    )
+    return version.Version(version_string)
 
 
 def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
@@ -564,11 +536,13 @@ def get_changes_for_version(wanted_version: version.Version) -> str:
     return "\n".join(version_changelog)
 
 
-def generate_and_write_changelog(current_version: version.Version):
+def generate_and_write_changelog(current_version: version.Version, new_version: str):
     # We do this by getting a draft so that we can edit it before writing to the
     # changelog.
     result = run_until_successful(
-        "python3 -m towncrier --draft", shell=True, capture_output=True
+        f"python3 -m towncrier build --draft --version {new_version}",
+        shell=True,
+        capture_output=True,
     )
     new_changes = result.stdout.decode("utf-8")
     new_changes = new_changes.replace(
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 6213f3265..000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,9 +0,0 @@
-[check-manifest]
-ignore =
-    .git-blame-ignore-revs
-    contrib
-    contrib/*
-    docs/*
-    pylint.cfg
-    tox.ini
-
diff --git a/setup.py b/setup.py
deleted file mode 100755
index ecd30247e..000000000
--- a/setup.py
+++ /dev/null
@@ -1,183 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2014-2017 OpenMarket Ltd
-# Copyright 2017 Vector Creations Ltd
-# Copyright 2017-2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-from typing import Any, Dict
-
-from setuptools import Command, find_packages, setup
-
-here = os.path.abspath(os.path.dirname(__file__))
-
-
-# Some notes on `setup.py test`:
-#
-# Once upon a time we used to try to make `setup.py test` run `tox` to run the
-# tests. That's a bad idea for three reasons:
-#
-# 1: `setup.py test` is supposed to find out whether the tests work in the
-#    *current* environmentt, not whatever tox sets up.
-# 2: Empirically, trying to install tox during the test run wasn't working ("No
-#    module named virtualenv").
-# 3: The tox documentation advises against it[1].
-#
-# Even further back in time, we used to use setuptools_trial [2]. That has its
-# own set of issues: for instance, it requires installation of Twisted to build
-# an sdist (because the recommended mode of usage is to add it to
-# `setup_requires`). That in turn means that in order to successfully run tox
-# you have to have the python header files installed for whichever version of
-# python tox uses (which is python3 on recent ubuntus, for example).
-#
-# So, for now at least, we stick with what appears to be the convention among
-# Twisted projects, and don't attempt to do anything when someone runs
-# `setup.py test`; instead we direct people to run `trial` directly if they
-# care.
-#
-# [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command
-# [2]: https://pypi.python.org/pypi/setuptools_trial
-class TestCommand(Command):
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        print(
-            """Synapse's tests cannot be run via setup.py. To run them, try:
-     PYTHONPATH="." trial tests
-"""
-        )
-
-
-def read_file(path_segments):
-    """Read a file from the package. Takes a list of strings to join to
-    make the path"""
-    file_path = os.path.join(here, *path_segments)
-    with open(file_path) as f:
-        return f.read()
-
-
-def exec_file(path_segments):
-    """Execute a single python file to get the variables defined in it"""
-    result: Dict[str, Any] = {}
-    code = read_file(path_segments)
-    exec(code, result)
-    return result
-
-
-version = exec_file(("synapse", "__init__.py"))["__version__"]
-dependencies = exec_file(("synapse", "python_dependencies.py"))
-long_description = read_file(("README.rst",))
-
-REQUIREMENTS = dependencies["REQUIREMENTS"]
-CONDITIONAL_REQUIREMENTS = dependencies["CONDITIONAL_REQUIREMENTS"]
-ALL_OPTIONAL_REQUIREMENTS = dependencies["ALL_OPTIONAL_REQUIREMENTS"]
-
-# Make `pip install matrix-synapse[all]` install all the optional dependencies.
-CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS)
-
-# Developer dependencies should not get included in "all".
-#
-# We pin black so that our tests don't start failing on new releases.
-CONDITIONAL_REQUIREMENTS["lint"] = [
-    "isort==5.7.0",
-    "black==22.3.0",
-    "flake8-comprehensions",
-    "flake8-bugbear==21.3.2",
-    "flake8",
-]
-
-CONDITIONAL_REQUIREMENTS["mypy"] = [
-    "mypy==0.931",
-    "mypy-zope==0.3.5",
-    "types-bleach>=4.1.0",
-    "types-jsonschema>=3.2.0",
-    "types-opentracing>=2.4.2",
-    "types-Pillow>=8.3.4",
-    "types-psycopg2>=2.9.9",
-    "types-pyOpenSSL>=20.0.7",
-    "types-PyYAML>=5.4.10",
-    "types-requests>=2.26.0",
-    "types-setuptools>=57.4.0",
-]
-
-# Dependencies which are exclusively required by unit test code. This is
-# NOT a list of all modules that are necessary to run the unit tests.
-# Tests assume that all optional dependencies are installed.
-#
-# parameterized_class decorator was introduced in parameterized 0.7.0
-CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "idna>=2.5"]
-
-CONDITIONAL_REQUIREMENTS["dev"] = (
-    CONDITIONAL_REQUIREMENTS["lint"]
-    + CONDITIONAL_REQUIREMENTS["mypy"]
-    + CONDITIONAL_REQUIREMENTS["test"]
-    + [
-        # The following are used by the release script
-        "click==8.1.0",
-        "redbaron==0.9.2",
-        "GitPython==3.1.14",
-        "commonmark==0.9.1",
-        "pygithub==1.55",
-        # The following are executed as commands by the release script.
-        "twine",
-        "towncrier",
-    ]
-)
-
-setup(
-    name="matrix-synapse",
-    version=version,
-    packages=find_packages(exclude=["tests", "tests.*"]),
-    description="Reference homeserver for the Matrix decentralised comms protocol",
-    install_requires=REQUIREMENTS,
-    extras_require=CONDITIONAL_REQUIREMENTS,
-    include_package_data=True,
-    zip_safe=False,
-    long_description=long_description,
-    long_description_content_type="text/x-rst",
-    python_requires="~=3.7",
-    entry_points={
-        "console_scripts": [
-            # Application
-            "synapse_homeserver = synapse.app.homeserver:main",
-            "synapse_worker = synapse.app.generic_worker:main",
-            "synctl = synapse._scripts.synctl:main",
-            # Scripts
-            "export_signing_key = synapse._scripts.export_signing_key:main",
-            "generate_config = synapse._scripts.generate_config:main",
-            "generate_log_config = synapse._scripts.generate_log_config:main",
-            "generate_signing_key = synapse._scripts.generate_signing_key:main",
-            "hash_password = synapse._scripts.hash_password:main",
-            "register_new_matrix_user = synapse._scripts.register_new_matrix_user:main",
-            "synapse_port_db = synapse._scripts.synapse_port_db:main",
-            "synapse_review_recent_signups = synapse._scripts.review_recent_signups:main",
-            "update_synapse_database = synapse._scripts.update_synapse_database:main",
-        ]
-    },
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Topic :: Communications :: Chat",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-    ],
-    cmdclass={"test": TestCommand},
-)
diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi
index 344d55cce..e18d61728 100644
--- a/stubs/sortedcontainers/sorteddict.pyi
+++ b/stubs/sortedcontainers/sorteddict.pyi
@@ -103,7 +103,7 @@ class SortedDict(Dict[_KT, _VT]):
         self,
         start: Optional[int] = ...,
         stop: Optional[int] = ...,
-        reverse=bool,
+        reverse: bool = ...,
     ) -> Iterator[_KT]: ...
     def bisect_left(self, value: _KT) -> int: ...
     def bisect_right(self, value: _KT) -> int: ...
diff --git a/stubs/sortedcontainers/sortedlist.pyi b/stubs/sortedcontainers/sortedlist.pyi
index f80a3a72c..403897e39 100644
--- a/stubs/sortedcontainers/sortedlist.pyi
+++ b/stubs/sortedcontainers/sortedlist.pyi
@@ -81,7 +81,7 @@ class SortedList(MutableSequence[_T]):
         self,
         start: Optional[int] = ...,
         stop: Optional[int] = ...,
-        reverse=bool,
+        reverse: bool = ...,
     ) -> Iterator[_T]: ...
     def _islice(
         self,
@@ -153,14 +153,14 @@ class SortedKeyList(SortedList[_T]):
         maximum: Optional[int] = ...,
         inclusive: Tuple[bool, bool] = ...,
         reverse: bool = ...,
-    ): ...
+    ) -> Iterator[_T]: ...
     def irange_key(
         self,
         min_key: Optional[Any] = ...,
         max_key: Optional[Any] = ...,
         inclusive: Tuple[bool, bool] = ...,
         reserve: bool = ...,
-    ): ...
+    ) -> Iterator[_T]: ...
     def bisect_left(self, value: _T) -> int: ...
     def bisect_right(self, value: _T) -> int: ...
     def bisect(self, value: _T) -> int: ...
diff --git a/stubs/sortedcontainers/sortedset.pyi b/stubs/sortedcontainers/sortedset.pyi
index f9c290838..43c860f42 100644
--- a/stubs/sortedcontainers/sortedset.pyi
+++ b/stubs/sortedcontainers/sortedset.pyi
@@ -103,7 +103,7 @@ class SortedSet(MutableSet[_T], Sequence[_T]):
         self,
         start: Optional[int] = ...,
         stop: Optional[int] = ...,
-        reverse=bool,
+        reverse: bool = ...,
     ) -> Iterator[_T]: ...
     def irange(
         self,
diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi
index 2d8ca018f..695a2307c 100644
--- a/stubs/txredisapi.pyi
+++ b/stubs/txredisapi.pyi
@@ -18,6 +18,8 @@ from typing import Any, List, Optional, Type, Union
 
 from twisted.internet import protocol
 from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IAddress
+from twisted.python.failure import Failure
 
 class RedisProtocol(protocol.Protocol):
     def publish(self, channel: str, message: bytes) -> "Deferred[None]": ...
@@ -34,11 +36,14 @@ class RedisProtocol(protocol.Protocol):
     def get(self, key: str) -> "Deferred[Any]": ...
 
 class SubscriberProtocol(RedisProtocol):
-    def __init__(self, *args, **kwargs): ...
+    def __init__(self, *args: object, **kwargs: object): ...
     password: Optional[str]
-    def subscribe(self, channels: Union[str, List[str]]): ...
-    def connectionMade(self): ...
-    def connectionLost(self, reason): ...
+    def subscribe(self, channels: Union[str, List[str]]) -> "Deferred[None]": ...
+    def connectionMade(self) -> None: ...
+    # type-ignore: twisted.internet.protocol.Protocol provides a default argument for
+    # `reason`. txredisapi's LineReceiver Protocol doesn't. But that's fine: it's what's
+    # actually specified in twisted.internet.interfaces.IProtocol.
+    def connectionLost(self, reason: Failure) -> None: ...  # type: ignore[override]
 
 def lazyConnection(
     host: str = ...,
@@ -74,7 +79,7 @@ class RedisFactory(protocol.ReconnectingClientFactory):
         replyTimeout: Optional[int] = None,
         convertNumbers: Optional[int] = True,
     ): ...
-    def buildProtocol(self, addr) -> RedisProtocol: ...
+    def buildProtocol(self, addr: IAddress) -> RedisProtocol: ...
 
 class SubscriberFactory(RedisFactory):
     def __init__(self) -> None: ...
diff --git a/synapse/__init__.py b/synapse/__init__.py
index b62eed66e..161394175 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -20,6 +20,8 @@ import json
 import os
 import sys
 
+from matrix_common.versionstring import get_distribution_version_string
+
 # Check that we're not running on an unsupported Python version.
 if sys.version_info < (3, 7):
     print("Synapse requires Python 3.7 or above.")
@@ -68,7 +70,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.57.1"
+__version__ = get_distribution_version_string("matrix-synapse")
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 92907415e..0172eb60b 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -179,8 +179,6 @@ class RelationTypes:
     REPLACE: Final = "m.replace"
     REFERENCE: Final = "m.reference"
     THREAD: Final = "m.thread"
-    # TODO Remove this in Synapse >= v1.57.0.
-    UNSTABLE_THREAD: Final = "io.element.thread"
 
 
 class LimitBlockingTypes:
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index e92db29f6..cb3b7323d 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -79,6 +79,8 @@ class Codes:
     UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN"
     UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN"
 
+    UNREDACTED_CONTENT_DELETED = "FI.MAU.MSC2815_UNREDACTED_CONTENT_DELETED"
+
 
 class CodeMessageException(RuntimeError):
     """An exception with integer code and message string attributes.
@@ -483,6 +485,22 @@ class RequestSendFailed(RuntimeError):
         self.can_retry = can_retry
 
 
+class UnredactedContentDeletedError(SynapseError):
+    def __init__(self, content_keep_ms: Optional[int] = None):
+        super().__init__(
+            404,
+            "The content for that event has already been erased from the database",
+            errcode=Codes.UNREDACTED_CONTENT_DELETED,
+        )
+        self.content_keep_ms = content_keep_ms
+
+    def error_dict(self) -> "JsonDict":
+        extra = {}
+        if self.content_keep_ms is not None:
+            extra = {"fi.mau.msc2815.content_keep_ms": self.content_keep_ms}
+        return cs_error(self.msg, self.errcode, **extra)
+
+
 def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs: Any) -> "JsonDict":
     """Utility method for constructing an error response for client-server
     interactions.
diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 27e97d6f3..4a808e33f 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -89,9 +89,7 @@ ROOM_EVENT_FILTER_SCHEMA = {
         "org.matrix.not_labels": {"type": "array", "items": {"type": "string"}},
         # MSC3440, filtering by event relations.
         "related_by_senders": {"type": "array", "items": {"type": "string"}},
-        "io.element.relation_senders": {"type": "array", "items": {"type": "string"}},
         "related_by_rel_types": {"type": "array", "items": {"type": "string"}},
-        "io.element.relation_types": {"type": "array", "items": {"type": "string"}},
     },
 }
 
@@ -323,16 +321,6 @@ class Filter:
         self.related_by_senders = self.filter_json.get("related_by_senders", None)
         self.related_by_rel_types = self.filter_json.get("related_by_rel_types", None)
 
-        # Fallback to the unstable prefix if the stable version is not given.
-        if hs.config.experimental.msc3440_enabled:
-            self.related_by_senders = self.related_by_senders or self.filter_json.get(
-                "io.element.relation_senders", None
-            )
-            self.related_by_rel_types = (
-                self.related_by_rel_types
-                or self.filter_json.get("io.element.relation_types", None)
-            )
-
     def filters_all_types(self) -> bool:
         return "*" in self.not_types
 
diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py
index d23d9221b..a610fb785 100644
--- a/synapse/appservice/__init__.py
+++ b/synapse/appservice/__init__.py
@@ -42,7 +42,7 @@ logger = logging.getLogger(__name__)
 #   user ID -> {device ID -> {algorithm -> count}}
 TransactionOneTimeKeyCounts = Dict[str, Dict[str, Dict[str, int]]]
 
-# Type for the `device_unused_fallback_keys` field in an appservice transaction
+# Type for the `device_unused_fallback_key_types` field in an appservice transaction
 #   user ID -> {device ID -> [algorithm]}
 TransactionUnusedFallbackKeys = Dict[str, Dict[str, List[str]]]
 
diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py
index 0cdbb04bf..adc6b074d 100644
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -278,7 +278,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                 ] = one_time_key_counts
             if unused_fallback_keys:
                 body[
-                    "org.matrix.msc3202.device_unused_fallback_keys"
+                    "org.matrix.msc3202.device_unused_fallback_key_types"
                 ] = unused_fallback_keys
             if device_list_summary:
                 body["org.matrix.msc3202.device_lists"] = {
diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py
index 447476fbf..421ed7481 100644
--- a/synapse/config/experimental.py
+++ b/synapse/config/experimental.py
@@ -26,9 +26,6 @@ class ExperimentalConfig(Config):
     def read_config(self, config: JsonDict, **kwargs: Any) -> None:
         experimental = config.get("experimental_features") or {}
 
-        # MSC3440 (thread relation)
-        self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False)
-
         # MSC3026 (busy presence state)
         self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)
 
@@ -77,7 +74,10 @@ class ExperimentalConfig(Config):
         self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False)
 
         # The deprecated groups feature.
-        self.groups_enabled: bool = experimental.get("groups_enabled", True)
+        self.groups_enabled: bool = experimental.get("groups_enabled", False)
 
         # MSC2654: Unread counts
         self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False)
+
+        # MSC2815 (allow room moderators to view redacted event content)
+        self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False)
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 415279d26..d771045b5 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -680,14 +680,6 @@ class ServerConfig(Config):
             config.get("use_account_validity_in_account_status") or False
         )
 
-        # This is a temporary option that enables fully using the new
-        # `device_lists_changes_in_room` without the backwards compat code. This
-        # is primarily for testing. If enabled the server should *not* be
-        # downgraded, as it may lead to missing device list updates.
-        self.use_new_device_lists_changes_in_room = (
-            config.get("use_new_device_lists_changes_in_room") or False
-        )
-
         self.rooms_to_exclude_from_sync: List[str] = (
             config.get("exclude_rooms_from_sync") or []
         )
diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index 918e87ed9..f8d3ba545 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -39,7 +39,6 @@ from . import EventBase
 
 if TYPE_CHECKING:
     from synapse.handlers.relations import BundledAggregations
-    from synapse.server import HomeServer
 
 
 # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\'
@@ -396,9 +395,6 @@ class EventClientSerializer:
     clients.
     """
 
-    def __init__(self, hs: "HomeServer"):
-        self._msc3440_enabled = hs.config.experimental.msc3440_enabled
-
     def serialize_event(
         self,
         event: Union[JsonDict, EventBase],
@@ -406,6 +402,7 @@ class EventClientSerializer:
         *,
         config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG,
         bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None,
+        apply_edits: bool = True,
     ) -> JsonDict:
         """Serializes a single event.
 
@@ -413,10 +410,10 @@ class EventClientSerializer:
             event: The event being serialized.
             time_now: The current time in milliseconds
             config: Event serialization config
-            bundle_aggregations: Whether to include the bundled aggregations for this
-                event. Only applies to non-state events. (State events never include
-                bundled aggregations.)
-
+            bundle_aggregations: A map from event_id to the aggregations to be bundled
+               into the event.
+            apply_edits: Whether the content of the event should be modified to reflect
+               any replacement in `bundle_aggregations[<event_id>].replace`.
         Returns:
             The serialized event
         """
@@ -434,8 +431,9 @@ class EventClientSerializer:
                     event,
                     time_now,
                     config,
-                    bundle_aggregations[event.event_id],
+                    event_aggregations,
                     serialized_event,
+                    apply_edits=apply_edits,
                 )
 
         return serialized_event
@@ -474,16 +472,18 @@ class EventClientSerializer:
         config: SerializeEventConfig,
         aggregations: "BundledAggregations",
         serialized_event: JsonDict,
+        apply_edits: bool,
     ) -> None:
         """Potentially injects bundled aggregations into the unsigned portion of the serialized event.
 
         Args:
             event: The event being serialized.
             time_now: The current time in milliseconds
+            config: Event serialization config
             aggregations: The bundled aggregation to serialize.
             serialized_event: The serialized event which may be modified.
-            config: Event serialization config
-
+            apply_edits: Whether the content of the event should be modified to reflect
+               any replacement in `aggregations.replace`.
         """
         serialized_aggregations = {}
 
@@ -494,9 +494,10 @@ class EventClientSerializer:
             serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references
 
         if aggregations.replace:
-            # If there is an edit, apply it to the event.
+            # If there is an edit, optionally apply it to the event.
             edit = aggregations.replace
-            self._apply_edit(event, serialized_event, edit)
+            if apply_edits:
+                self._apply_edit(event, serialized_event, edit)
 
             # Include information about it in the relations dict.
             serialized_aggregations[RelationTypes.REPLACE] = {
@@ -525,8 +526,6 @@ class EventClientSerializer:
                 "current_user_participated": thread.current_user_participated,
             }
             serialized_aggregations[RelationTypes.THREAD] = thread_summary
-            if self._msc3440_enabled:
-                serialized_aggregations[RelationTypes.UNSTABLE_THREAD] = thread_summary
 
         # Include the bundled aggregations in the event.
         if serialized_aggregations:
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 69d833585..beab1227b 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -515,7 +515,7 @@ class FederationServer(FederationBase):
         )
 
     async def on_room_state_request(
-        self, origin: str, room_id: str, event_id: Optional[str]
+        self, origin: str, room_id: str, event_id: str
     ) -> Tuple[int, JsonDict]:
         origin_host, _ = parse_server_name(origin)
         await self.check_server_matches_acl(origin_host, room_id)
@@ -530,18 +530,13 @@ class FederationServer(FederationBase):
         # - but that's non-trivial to get right, and anyway somewhat defeats
         # the point of the linearizer.
         async with self._server_linearizer.queue((origin, room_id)):
-            resp: JsonDict = dict(
-                await self._state_resp_cache.wrap(
-                    (room_id, event_id),
-                    self._on_context_state_request_compute,
-                    room_id,
-                    event_id,
-                )
+            resp = await self._state_resp_cache.wrap(
+                (room_id, event_id),
+                self._on_context_state_request_compute,
+                room_id,
+                event_id,
             )
 
-        room_version = await self.store.get_room_version_id(room_id)
-        resp["room_version"] = room_version
-
         return 200, resp
 
     async def on_state_ids_request(
@@ -574,14 +569,11 @@ class FederationServer(FederationBase):
         return {"pdu_ids": state_ids, "auth_chain_ids": list(auth_chain_ids)}
 
     async def _on_context_state_request_compute(
-        self, room_id: str, event_id: Optional[str]
+        self, room_id: str, event_id: str
     ) -> Dict[str, list]:
         pdus: Collection[EventBase]
-        if event_id:
-            event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id)
-            pdus = await self.store.get_events_as_list(event_ids)
-        else:
-            pdus = (await self.state.get_current_state(room_id)).values()
+        event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id)
+        pdus = await self.store.get_events_as_list(event_ids)
 
         auth_chain = await self.store.get_auth_chain(
             room_id, [pdu.event_id for pdu in pdus]
@@ -687,8 +679,6 @@ class FederationServer(FederationBase):
         time_now = self._clock.time_msec()
         event_json = event.get_pdu_json(time_now)
         resp = {
-            # TODO Remove the unstable prefix when servers have updated.
-            "org.matrix.msc3083.v2.event": event_json,
             "event": event_json,
             "state": [p.get_pdu_json(time_now) for p in state_events],
             "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events],
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 01dc5ca94..1421050b9 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -1380,16 +1380,6 @@ class SendJoinParser(ByteParser[SendJoinResponse]):
                 prefix + "auth_chain.item",
                 use_float=True,
             ),
-            # TODO Remove the unstable prefix when servers have updated.
-            #
-            # By re-using the same event dictionary this will cause the parsing of
-            # org.matrix.msc3083.v2.event and event to stomp over each other.
-            # Generally this should be fine.
-            ijson.kvitems_coro(
-                _event_parser(self._response.event_dict),
-                prefix + "org.matrix.msc3083.v2.event",
-                use_float=True,
-            ),
             ijson.kvitems_coro(
                 _event_parser(self._response.event_dict),
                 prefix + "event",
diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py
index 2529dee61..d629a3ecb 100644
--- a/synapse/federation/transport/server/_base.py
+++ b/synapse/federation/transport/server/_base.py
@@ -16,7 +16,8 @@ import functools
 import logging
 import re
 import time
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast
 
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
 from synapse.api.urls import FEDERATION_V1_PREFIX
@@ -86,15 +87,24 @@ class Authenticator:
 
         if not auth_headers:
             raise NoAuthenticationError(
-                401, "Missing Authorization headers", Codes.UNAUTHORIZED
+                HTTPStatus.UNAUTHORIZED,
+                "Missing Authorization headers",
+                Codes.UNAUTHORIZED,
             )
 
         for auth in auth_headers:
             if auth.startswith(b"X-Matrix"):
-                (origin, key, sig) = _parse_auth_header(auth)
+                (origin, key, sig, destination) = _parse_auth_header(auth)
                 json_request["origin"] = origin
                 json_request["signatures"].setdefault(origin, {})[key] = sig
 
+                # if the origin_server sent a destination along it needs to match our own server_name
+                if destination is not None and destination != self.server_name:
+                    raise AuthenticationError(
+                        HTTPStatus.UNAUTHORIZED,
+                        "Destination mismatch in auth header",
+                        Codes.UNAUTHORIZED,
+                    )
         if (
             self.federation_domain_whitelist is not None
             and origin not in self.federation_domain_whitelist
@@ -103,7 +113,9 @@ class Authenticator:
 
         if origin is None or not json_request["signatures"]:
             raise NoAuthenticationError(
-                401, "Missing Authorization headers", Codes.UNAUTHORIZED
+                HTTPStatus.UNAUTHORIZED,
+                "Missing Authorization headers",
+                Codes.UNAUTHORIZED,
             )
 
         await self.keyring.verify_json_for_server(
@@ -142,13 +154,14 @@ class Authenticator:
             logger.exception("Error resetting retry timings on %s", origin)
 
 
-def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:
+def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str]]:
     """Parse an X-Matrix auth header
 
     Args:
         header_bytes: header value
 
     Returns:
+        origin, key id, signature, destination.
         origin, key id, signature.
 
     Raises:
@@ -157,7 +170,9 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:
     try:
         header_str = header_bytes.decode("utf-8")
         params = header_str.split(" ")[1].split(",")
-        param_dict = {k: v for k, v in (kv.split("=", maxsplit=1) for kv in params)}
+        param_dict: Dict[str, str] = {
+            k: v for k, v in [param.split("=", maxsplit=1) for param in params]
+        }
 
         def strip_quotes(value: str) -> str:
             if value.startswith('"'):
@@ -172,7 +187,15 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:
 
         key = strip_quotes(param_dict["key"])
         sig = strip_quotes(param_dict["sig"])
-        return origin, key, sig
+
+        # get the destination server_name from the auth header if it exists
+        destination = param_dict.get("destination")
+        if destination is not None:
+            destination = strip_quotes(destination)
+        else:
+            destination = None
+
+        return origin, key, sig, destination
     except Exception as e:
         logger.warning(
             "Error parsing auth header '%s': %s",
@@ -180,7 +203,7 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:
             e,
         )
         raise AuthenticationError(
-            400, "Malformed Authorization header", Codes.UNAUTHORIZED
+            HTTPStatus.BAD_REQUEST, "Malformed Authorization header", Codes.UNAUTHORIZED
         )
 
 
diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py
index aed3d5069..6fbc7b5f1 100644
--- a/synapse/federation/transport/server/federation.py
+++ b/synapse/federation/transport/server/federation.py
@@ -160,7 +160,7 @@ class FederationStateV1Servlet(BaseFederationServerServlet):
         return await self.handler.on_room_state_request(
             origin,
             room_id,
-            parse_string_from_args(query, "event_id", None, required=False),
+            parse_string_from_args(query, "event_id", None, required=True),
         )
 
 
diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index ffa28b2a3..3c0fc756d 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -291,12 +291,6 @@ class DeviceHandler(DeviceWorkerHandler):
         # On start up check if there are any updates pending.
         hs.get_reactor().callWhenRunning(self._handle_new_device_update_async)
 
-        # Used to decide if we calculate outbound pokes up front or not. By
-        # default we do to allow safely downgrading Synapse.
-        self.use_new_device_lists_changes_in_room = (
-            hs.config.server.use_new_device_lists_changes_in_room
-        )
-
     def _check_device_name_length(self, name: Optional[str]) -> None:
         """
         Checks whether a device name is longer than the maximum allowed length.
@@ -490,23 +484,9 @@ class DeviceHandler(DeviceWorkerHandler):
 
         room_ids = await self.store.get_rooms_for_user(user_id)
 
-        hosts: Optional[Set[str]] = None
-        if not self.use_new_device_lists_changes_in_room:
-            hosts = set()
-
-            if self.hs.is_mine_id(user_id):
-                for room_id in room_ids:
-                    joined_users = await self.store.get_users_in_room(room_id)
-                    hosts.update(get_domain_from_id(u) for u in joined_users)
-
-                set_tag("target_hosts", hosts)
-
-                hosts.discard(self.server_name)
-
         position = await self.store.add_device_change_to_streams(
             user_id,
             device_ids,
-            hosts=hosts,
             room_ids=room_ids,
         )
 
@@ -528,14 +508,6 @@ class DeviceHandler(DeviceWorkerHandler):
         # We may need to do some processing asynchronously.
         self._handle_new_device_update_async()
 
-        if hosts:
-            logger.info(
-                "Sending device list update notif for %r to: %r", user_id, hosts
-            )
-            for host in hosts:
-                self.federation_sender.send_device_messages(host, immediate=False)
-                log_kv({"message": "sent device update to host", "host": host})
-
     async def notify_user_signature_update(
         self, from_user_id: str, user_ids: List[str]
     ) -> None:
@@ -677,9 +649,13 @@ class DeviceHandler(DeviceWorkerHandler):
                         return
 
                 for user_id, device_id, room_id, stream_id, opentracing_context in rows:
-                    joined_user_ids = await self.store.get_users_in_room(room_id)
-                    hosts = {get_domain_from_id(u) for u in joined_user_ids}
-                    hosts.discard(self.server_name)
+                    hosts = set()
+
+                    # Ignore any users that aren't ours
+                    if self.hs.is_mine_id(user_id):
+                        joined_user_ids = await self.store.get_users_in_room(room_id)
+                        hosts = {get_domain_from_id(u) for u in joined_user_ids}
+                        hosts.discard(self.server_name)
 
                     # Check if we've already sent this update to some hosts
                     if current_stream_id == stream_id:
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index d2ccb5c5d..5b94b00bc 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -16,11 +16,12 @@ import logging
 import random
 from typing import TYPE_CHECKING, Iterable, List, Optional
 
-from synapse.api.constants import EduTypes, EventTypes, Membership
+from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState
 from synapse.api.errors import AuthError, SynapseError
 from synapse.events import EventBase
 from synapse.events.utils import SerializeEventConfig
 from synapse.handlers.presence import format_user_presence_state
+from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, UserID
 from synapse.visibility import filter_events_for_client
@@ -67,7 +68,9 @@ class EventStreamHandler:
         presence_handler = self.hs.get_presence_handler()
 
         context = await presence_handler.user_syncing(
-            auth_user_id, affect_presence=affect_presence
+            auth_user_id,
+            affect_presence=affect_presence,
+            presence_state=PresenceState.ONLINE,
         )
         with context:
             if timeout:
@@ -139,7 +142,11 @@ class EventHandler:
         self.storage = hs.get_storage()
 
     async def get_event(
-        self, user: UserID, room_id: Optional[str], event_id: str
+        self,
+        user: UserID,
+        room_id: Optional[str],
+        event_id: str,
+        show_redacted: bool = False,
     ) -> Optional[EventBase]:
         """Retrieve a single specified event.
 
@@ -148,6 +155,7 @@ class EventHandler:
             room_id: The expected room id. We'll return None if the
                 event's room does not match.
             event_id: The event ID to obtain.
+            show_redacted: Should the full content of redacted events be returned?
         Returns:
             An event, or None if there is no event matching this ID.
         Raises:
@@ -155,7 +163,12 @@ class EventHandler:
             AuthError if the user does not have the rights to inspect this
             event.
         """
-        event = await self.store.get_event(event_id, check_room_id=room_id)
+        redact_behaviour = (
+            EventRedactBehaviour.AS_IS if show_redacted else EventRedactBehaviour.REDACT
+        )
+        event = await self.store.get_event(
+            event_id, check_room_id=room_id, redact_behaviour=redact_behaviour
+        )
 
         if not event:
             return None
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 0924def3e..f9d69f50d 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2014-2022 The Matrix.org Foundation C.I.C.
 # Copyright 2020 Sorunome
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,14 @@
 
 """Contains handlers for federation events."""
 
+import enum
+import itertools
 import logging
+from enum import Enum
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union
 
+import attr
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
 from unpaddedbase64 import decode_base64
@@ -92,6 +96,24 @@ def get_domains_from_state(state: StateMap[EventBase]) -> List[Tuple[str, int]]:
     return sorted(joined_domains.items(), key=lambda d: d[1])
 
 
+class _BackfillPointType(Enum):
+    # a regular backwards extremity (ie, an event which we don't yet have, but which
+    # is referred to by other events in the DAG)
+    BACKWARDS_EXTREMITY = enum.auto()
+
+    # an MSC2716 "insertion event"
+    INSERTION_PONT = enum.auto()
+
+
+@attr.s(slots=True, auto_attribs=True, frozen=True)
+class _BackfillPoint:
+    """A potential point we might backfill from"""
+
+    event_id: str
+    depth: int
+    type: _BackfillPointType
+
+
 class FederationHandler:
     """Handles general incoming federation requests
 
@@ -157,89 +179,51 @@ class FederationHandler:
     async def _maybe_backfill_inner(
         self, room_id: str, current_depth: int, limit: int
     ) -> bool:
-        oldest_events_with_depth = (
-            await self.store.get_oldest_event_ids_with_depth_in_room(room_id)
-        )
+        backwards_extremities = [
+            _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY)
+            for event_id, depth in await self.store.get_oldest_event_ids_with_depth_in_room(
+                room_id
+            )
+        ]
 
-        insertion_events_to_be_backfilled: Dict[str, int] = {}
+        insertion_events_to_be_backfilled: List[_BackfillPoint] = []
         if self.hs.config.experimental.msc2716_enabled:
-            insertion_events_to_be_backfilled = (
-                await self.store.get_insertion_event_backward_extremities_in_room(
+            insertion_events_to_be_backfilled = [
+                _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT)
+                for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room(
                     room_id
                 )
-            )
+            ]
         logger.debug(
-            "_maybe_backfill_inner: extremities oldest_events_with_depth=%s insertion_events_to_be_backfilled=%s",
-            oldest_events_with_depth,
+            "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s",
+            backwards_extremities,
             insertion_events_to_be_backfilled,
         )
 
-        if not oldest_events_with_depth and not insertion_events_to_be_backfilled:
+        if not backwards_extremities and not insertion_events_to_be_backfilled:
             logger.debug("Not backfilling as no extremeties found.")
             return False
 
-        # We only want to paginate if we can actually see the events we'll get,
-        # as otherwise we'll just spend a lot of resources to get redacted
-        # events.
-        #
-        # We do this by filtering all the backwards extremities and seeing if
-        # any remain. Given we don't have the extremity events themselves, we
-        # need to actually check the events that reference them.
-        #
-        # *Note*: the spec wants us to keep backfilling until we reach the start
-        # of the room in case we are allowed to see some of the history. However
-        # in practice that causes more issues than its worth, as a) its
-        # relatively rare for there to be any visible history and b) even when
-        # there is its often sufficiently long ago that clients would stop
-        # attempting to paginate before backfill reached the visible history.
-        #
-        # TODO: If we do do a backfill then we should filter the backwards
-        #   extremities to only include those that point to visible portions of
-        #   history.
-        #
-        # TODO: Correctly handle the case where we are allowed to see the
-        #   forward event but not the backward extremity, e.g. in the case of
-        #   initial join of the server where we are allowed to see the join
-        #   event but not anything before it. This would require looking at the
-        #   state *before* the event, ignoring the special casing certain event
-        #   types have.
-
-        forward_event_ids = await self.store.get_successor_events(
-            list(oldest_events_with_depth)
+        # we now have a list of potential places to backpaginate from. We prefer to
+        # start with the most recent (ie, max depth), so let's sort the list.
+        sorted_backfill_points: List[_BackfillPoint] = sorted(
+            itertools.chain(
+                backwards_extremities,
+                insertion_events_to_be_backfilled,
+            ),
+            key=lambda e: -int(e.depth),
         )
 
-        extremities_events = await self.store.get_events(
-            forward_event_ids,
-            redact_behaviour=EventRedactBehaviour.AS_IS,
-            get_prev_content=False,
-        )
-
-        # We set `check_history_visibility_only` as we might otherwise get false
-        # positives from users having been erased.
-        filtered_extremities = await filter_events_for_server(
-            self.storage,
-            self.server_name,
-            list(extremities_events.values()),
-            redact=False,
-            check_history_visibility_only=True,
-        )
         logger.debug(
-            "_maybe_backfill_inner: filtered_extremities %s", filtered_extremities
+            "_maybe_backfill_inner: room_id: %s: current_depth: %s, limit: %s, "
+            "backfill points (%d): %s",
+            room_id,
+            current_depth,
+            limit,
+            len(sorted_backfill_points),
+            sorted_backfill_points,
         )
 
-        if not filtered_extremities and not insertion_events_to_be_backfilled:
-            return False
-
-        extremities = {
-            **oldest_events_with_depth,
-            # TODO: insertion_events_to_be_backfilled is currently skipping the filtered_extremities checks
-            **insertion_events_to_be_backfilled,
-        }
-
-        # Check if we reached a point where we should start backfilling.
-        sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1]))
-        max_depth = sorted_extremeties_tuple[0][1]
-
         # If we're approaching an extremity we trigger a backfill, otherwise we
         # no-op.
         #
@@ -249,6 +233,11 @@ class FederationHandler:
         # chose more than one times the limit in case of failure, but choosing a
         # much larger factor will result in triggering a backfill request much
         # earlier than necessary.
+        #
+        # XXX: shouldn't we do this *after* the filter by depth below? Again, we don't
+        # care about events that have happened after our current position.
+        #
+        max_depth = sorted_backfill_points[0].depth
         if current_depth - 2 * limit > max_depth:
             logger.debug(
                 "Not backfilling as we don't need to. %d < %d - 2 * %d",
@@ -265,31 +254,98 @@ class FederationHandler:
         #    2. we have likely previously tried and failed to backfill from that
         #       extremity, so to avoid getting "stuck" requesting the same
         #       backfill repeatedly we drop those extremities.
-        filtered_sorted_extremeties_tuple = [
-            t for t in sorted_extremeties_tuple if int(t[1]) <= current_depth
-        ]
-
-        logger.debug(
-            "room_id: %s, backfill: current_depth: %s, limit: %s, max_depth: %s, extrems (%d): %s filtered_sorted_extremeties_tuple: %s",
-            room_id,
-            current_depth,
-            limit,
-            max_depth,
-            len(sorted_extremeties_tuple),
-            sorted_extremeties_tuple,
-            filtered_sorted_extremeties_tuple,
-        )
-
+        #
         # However, we need to check that the filtered extremities are non-empty.
         # If they are empty then either we can a) bail or b) still attempt to
         # backfill. We opt to try backfilling anyway just in case we do get
         # relevant events.
-        if filtered_sorted_extremeties_tuple:
-            sorted_extremeties_tuple = filtered_sorted_extremeties_tuple
+        #
+        filtered_sorted_backfill_points = [
+            t for t in sorted_backfill_points if t.depth <= current_depth
+        ]
+        if filtered_sorted_backfill_points:
+            logger.debug(
+                "_maybe_backfill_inner: backfill points before current depth: %s",
+                filtered_sorted_backfill_points,
+            )
+            sorted_backfill_points = filtered_sorted_backfill_points
+        else:
+            logger.debug(
+                "_maybe_backfill_inner: all backfill points are *after* current depth. Backfilling anyway."
+            )
 
-        # We don't want to specify too many extremities as it causes the backfill
-        # request URI to be too long.
-        extremities = dict(sorted_extremeties_tuple[:5])
+        # For performance's sake, we only want to paginate from a particular extremity
+        # if we can actually see the events we'll get. Otherwise, we'd just spend a lot
+        # of resources to get redacted events. We check each extremity in turn and
+        # ignore those which users on our server wouldn't be able to see.
+        #
+        # Additionally, we limit ourselves to backfilling from at most 5 extremities,
+        # for two reasons:
+        #
+        # - The check which determines if we can see an extremity's events can be
+        #   expensive (we load the full state for the room at each of the backfill
+        #   points, or (worse) their successors)
+        # - We want to avoid the server-server API request URI becoming too long.
+        #
+        # *Note*: the spec wants us to keep backfilling until we reach the start
+        # of the room in case we are allowed to see some of the history. However,
+        # in practice that causes more issues than its worth, as (a) it's
+        # relatively rare for there to be any visible history and (b) even when
+        # there is it's often sufficiently long ago that clients would stop
+        # attempting to paginate before backfill reached the visible history.
+
+        extremities_to_request: List[str] = []
+        for bp in sorted_backfill_points:
+            if len(extremities_to_request) >= 5:
+                break
+
+            # For regular backwards extremities, we don't have the extremity events
+            # themselves, so we need to actually check the events that reference them -
+            # their "successor" events.
+            #
+            # TODO: Correctly handle the case where we are allowed to see the
+            #   successor event but not the backward extremity, e.g. in the case of
+            #   initial join of the server where we are allowed to see the join
+            #   event but not anything before it. This would require looking at the
+            #   state *before* the event, ignoring the special casing certain event
+            #   types have.
+            if bp.type == _BackfillPointType.INSERTION_PONT:
+                event_ids_to_check = [bp.event_id]
+            else:
+                event_ids_to_check = await self.store.get_successor_events(bp.event_id)
+
+            events_to_check = await self.store.get_events_as_list(
+                event_ids_to_check,
+                redact_behaviour=EventRedactBehaviour.AS_IS,
+                get_prev_content=False,
+            )
+
+            # We set `check_history_visibility_only` as we might otherwise get false
+            # positives from users having been erased.
+            filtered_extremities = await filter_events_for_server(
+                self.storage,
+                self.server_name,
+                events_to_check,
+                redact=False,
+                check_history_visibility_only=True,
+            )
+            if filtered_extremities:
+                extremities_to_request.append(bp.event_id)
+            else:
+                logger.debug(
+                    "_maybe_backfill_inner: skipping extremity %s as it would not be visible",
+                    bp,
+                )
+
+        if not extremities_to_request:
+            logger.debug(
+                "_maybe_backfill_inner: found no extremities which would be visible"
+            )
+            return False
+
+        logger.debug(
+            "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request
+        )
 
         # Now we need to decide which hosts to hit first.
 
@@ -309,7 +365,7 @@ class FederationHandler:
             for dom in domains:
                 try:
                     await self._federation_event_handler.backfill(
-                        dom, room_id, limit=100, extremities=extremities
+                        dom, room_id, limit=100, extremities=extremities_to_request
                     )
                     # If this succeeded then we probably already have the
                     # appropriate stuff.
@@ -466,6 +522,8 @@ class FederationHandler:
             )
 
             if ret.partial_state:
+                # TODO(faster_joins): roll this back if we don't manage to start the
+                #   background resync (eg process_remote_join fails)
                 await self.store.store_partial_state_room(room_id, ret.servers_in_room)
 
             max_stream_id = await self._federation_event_handler.process_remote_join(
@@ -478,6 +536,18 @@ class FederationHandler:
                 partial_state=ret.partial_state,
             )
 
+            if ret.partial_state:
+                # Kick off the process of asynchronously fetching the state for this
+                # room.
+                #
+                # TODO(faster_joins): pick this up again on restart
+                run_as_background_process(
+                    desc="sync_partial_state_room",
+                    func=self._sync_partial_state_room,
+                    destination=origin,
+                    room_id=room_id,
+                )
+
             # We wait here until this instance has seen the events come down
             # replication (if we're using replication) as the below uses caches.
             await self._replication.wait_for_stream_position(
@@ -1370,3 +1440,64 @@ class FederationHandler:
         # We fell off the bottom, couldn't get the complexity from anyone. Oh
         # well.
         return None
+
+    async def _sync_partial_state_room(
+        self,
+        destination: str,
+        room_id: str,
+    ) -> None:
+        """Background process to resync the state of a partial-state room
+
+        Args:
+            destination: homeserver to pull the state from
+            room_id: room to be resynced
+        """
+
+        # TODO(faster_joins): do we need to lock to avoid races? What happens if other
+        #   worker processes kick off a resync in parallel? Perhaps we should just elect
+        #   a single worker to do the resync.
+        #
+        # TODO(faster_joins): what happens if we leave the room during a resync? if we
+        #   really leave, that might mean we have difficulty getting the room state over
+        #   federation.
+        #
+        # TODO(faster_joins): try other destinations if the one we have fails
+
+        logger.info("Syncing state for room %s via %s", room_id, destination)
+
+        # we work through the queue in order of increasing stream ordering.
+        while True:
+            batch = await self.store.get_partial_state_events_batch(room_id)
+            if not batch:
+                # all the events are updated, so we can update current state and
+                # clear the lazy-loading flag.
+                logger.info("Updating current state for %s", room_id)
+                assert (
+                    self.storage.persistence is not None
+                ), "TODO(faster_joins): support for workers"
+                await self.storage.persistence.update_current_state(room_id)
+
+                logger.info("Clearing partial-state flag for %s", room_id)
+                success = await self.store.clear_partial_state_room(room_id)
+                if success:
+                    logger.info("State resync complete for %s", room_id)
+
+                    # TODO(faster_joins) update room stats and user directory?
+                    return
+
+                # we raced against more events arriving with partial state. Go round
+                # the loop again. We've already logged a warning, so no need for more.
+                # TODO(faster_joins): there is still a race here, whereby incoming events which raced
+                #   with us will fail to be persisted after the call to `clear_partial_state_room` due to
+                #   having partial state.
+                continue
+
+            events = await self.store.get_events_as_list(
+                batch,
+                redact_behaviour=EventRedactBehaviour.AS_IS,
+                allow_rejected=True,
+            )
+            for event in events:
+                await self._federation_event_handler.update_state_for_partial_state_event(
+                    destination, event
+                )
diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py
index 03c1197c9..693b54428 100644
--- a/synapse/handlers/federation_event.py
+++ b/synapse/handlers/federation_event.py
@@ -477,6 +477,46 @@ class FederationEventHandler:
 
             return await self.persist_events_and_notify(room_id, [(event, context)])
 
+    async def update_state_for_partial_state_event(
+        self, destination: str, event: EventBase
+    ) -> None:
+        """Recalculate the state at an event as part of a de-partial-stating process
+
+        Args:
+            destination: server to request full state from
+            event: partial-state event to be de-partial-stated
+        """
+        logger.info("Updating state for %s", event.event_id)
+        with nested_logging_context(suffix=event.event_id):
+            # if we have all the event's prev_events, then we can work out the
+            # state based on their states. Otherwise, we request it from the destination
+            # server.
+            #
+            # This is the same operation as we do when we receive a regular event
+            # over federation.
+            state = await self._resolve_state_at_missing_prevs(destination, event)
+
+            # build a new state group for it if need be
+            context = await self._state_handler.compute_event_context(
+                event,
+                old_state=state,
+            )
+            if context.partial_state:
+                # this can happen if some or all of the event's prev_events still have
+                # partial state - ie, an event has an earlier stream_ordering than one
+                # or more of its prev_events, so we de-partial-state it before its
+                # prev_events.
+                #
+                # TODO(faster_joins): we probably need to be more intelligent, and
+                #    exclude partial-state prev_events from consideration
+                logger.warning(
+                    "%s still has partial state: can't de-partial-state it yet",
+                    event.event_id,
+                )
+                return
+            await self._store.update_state_for_partial_state_event(event, context)
+            self._state_store.notify_event_un_partial_stated(event.event_id)
+
     async def backfill(
         self, dest: str, room_id: str, limit: int, extremities: Collection[str]
     ) -> None:
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index f73bf4130..74b6c3da2 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -175,17 +175,13 @@ class MessageHandler:
         state_filter = state_filter or StateFilter.all()
 
         if at_token:
-            # FIXME this claims to get the state at a stream position, but
-            # get_recent_events_for_room operates by topo ordering. This therefore
-            # does not reliably give you the state at the given stream position.
-            # (https://github.com/matrix-org/synapse/issues/3305)
-            last_events, _ = await self.store.get_recent_events_for_room(
-                room_id, end_token=at_token.room_key, limit=1
+            last_event = await self.store.get_last_event_in_room_before_stream_ordering(
+                room_id,
+                end_token=at_token.room_key,
             )
 
-            if not last_events:
+            if not last_event:
                 raise NotFoundError("Can't find event for token %s" % (at_token,))
-            last_event = last_events[0]
 
             # check whether the user is in the room at that time to determine
             # whether they should be treated as peeking.
@@ -204,7 +200,7 @@ class MessageHandler:
             visible_events = await filter_events_for_client(
                 self.storage,
                 user_id,
-                last_events,
+                [last_event],
                 filter_send_to_client=False,
                 is_peeking=is_peeking,
             )
@@ -1104,10 +1100,7 @@ class EventCreationHandler:
                 raise SynapseError(400, "Can't send same reaction twice")
 
         # Don't attempt to start a thread if the parent event is a relation.
-        elif (
-            relation_type == RelationTypes.THREAD
-            or relation_type == RelationTypes.UNSTABLE_THREAD
-        ):
+        elif relation_type == RelationTypes.THREAD:
             if await self.store.event_includes_relation(relates_to):
                 raise SynapseError(
                     400, "Cannot start threads from an event with a relation"
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 209a4b0e5..d078162c2 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -151,7 +151,7 @@ class BasePresenceHandler(abc.ABC):
 
     @abc.abstractmethod
     async def user_syncing(
-        self, user_id: str, affect_presence: bool
+        self, user_id: str, affect_presence: bool, presence_state: str
     ) -> ContextManager[None]:
         """Returns a context manager that should surround any stream requests
         from the user.
@@ -165,6 +165,7 @@ class BasePresenceHandler(abc.ABC):
             affect_presence: If false this function will be a no-op.
                 Useful for streams that are not associated with an actual
                 client that is being used by a user.
+            presence_state: The presence state indicated in the sync request
         """
 
     @abc.abstractmethod
@@ -228,6 +229,11 @@ class BasePresenceHandler(abc.ABC):
 
         return states
 
+    async def current_state_for_user(self, user_id: str) -> UserPresenceState:
+        """Get the current presence state for a user."""
+        res = await self.current_state_for_users([user_id])
+        return res[user_id]
+
     @abc.abstractmethod
     async def set_state(
         self,
@@ -461,7 +467,7 @@ class WorkerPresenceHandler(BasePresenceHandler):
                 self.send_user_sync(user_id, False, last_sync_ms)
 
     async def user_syncing(
-        self, user_id: str, affect_presence: bool
+        self, user_id: str, affect_presence: bool, presence_state: str
     ) -> ContextManager[None]:
         """Record that a user is syncing.
 
@@ -471,6 +477,17 @@ class WorkerPresenceHandler(BasePresenceHandler):
         if not affect_presence or not self._presence_enabled:
             return _NullContextManager()
 
+        prev_state = await self.current_state_for_user(user_id)
+        if prev_state != PresenceState.BUSY:
+            # We set state here but pass ignore_status_msg = True as we don't want to
+            # cause the status message to be cleared.
+            # Note that this causes last_active_ts to be incremented which is not
+            # what the spec wants: see comment in the BasePresenceHandler version
+            # of this function.
+            await self.set_state(
+                UserID.from_string(user_id), {"presence": presence_state}, True
+            )
+
         curr_sync = self._user_to_num_current_syncs.get(user_id, 0)
         self._user_to_num_current_syncs[user_id] = curr_sync + 1
 
@@ -942,7 +959,10 @@ class PresenceHandler(BasePresenceHandler):
         await self._update_states([prev_state.copy_and_replace(**new_fields)])
 
     async def user_syncing(
-        self, user_id: str, affect_presence: bool = True
+        self,
+        user_id: str,
+        affect_presence: bool = True,
+        presence_state: str = PresenceState.ONLINE,
     ) -> ContextManager[None]:
         """Returns a context manager that should surround any stream requests
         from the user.
@@ -956,6 +976,7 @@ class PresenceHandler(BasePresenceHandler):
             affect_presence: If false this function will be a no-op.
                 Useful for streams that are not associated with an actual
                 client that is being used by a user.
+            presence_state: The presence state indicated in the sync request
         """
         # Override if it should affect the user's presence, if presence is
         # disabled.
@@ -967,9 +988,25 @@ class PresenceHandler(BasePresenceHandler):
             self.user_to_num_current_syncs[user_id] = curr_sync + 1
 
             prev_state = await self.current_state_for_user(user_id)
+
+            # If they're busy then they don't stop being busy just by syncing,
+            # so just update the last sync time.
+            if prev_state.state != PresenceState.BUSY:
+                # XXX: We set_state separately here and just update the last_active_ts above
+                # This keeps the logic as similar as possible between the worker and single
+                # process modes. Using set_state will actually cause last_active_ts to be
+                # updated always, which is not what the spec calls for, but synapse has done
+                # this for... forever, I think.
+                await self.set_state(
+                    UserID.from_string(user_id), {"presence": presence_state}, True
+                )
+                # Retrieve the new state for the logic below. This should come from the
+                # in-memory cache.
+                prev_state = await self.current_state_for_user(user_id)
+
+            # To keep the single process behaviour consistent with worker mode, run the
+            # same logic as `update_external_syncs_row`, even though it looks weird.
             if prev_state.state == PresenceState.OFFLINE:
-                # If they're currently offline then bring them online, otherwise
-                # just update the last sync times.
                 await self._update_states(
                     [
                         prev_state.copy_and_replace(
@@ -979,6 +1016,10 @@ class PresenceHandler(BasePresenceHandler):
                         )
                     ]
                 )
+            # otherwise, set the new presence state & update the last sync time,
+            # but don't update last_active_ts as this isn't an indication that
+            # they've been active (even though it's probably been updated by
+            # set_state above)
             else:
                 await self._update_states(
                     [
@@ -1086,11 +1127,6 @@ class PresenceHandler(BasePresenceHandler):
             )
             self.external_process_last_updated_ms.pop(process_id, None)
 
-    async def current_state_for_user(self, user_id: str) -> UserPresenceState:
-        """Get the current presence state for a user."""
-        res = await self.current_state_for_users([user_id])
-        return res[user_id]
-
     async def _persist_and_notify(self, states: List[UserPresenceState]) -> None:
         """Persist states in the database, poke the notifier and send to
         interested remote servers
diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py
index 0be231957..5efb56127 100644
--- a/synapse/handlers/relations.py
+++ b/synapse/handlers/relations.py
@@ -256,64 +256,6 @@ class RelationsHandler:
 
         return filtered_results
 
-    async def _get_bundled_aggregation_for_event(
-        self, event: EventBase, ignored_users: FrozenSet[str]
-    ) -> Optional[BundledAggregations]:
-        """Generate bundled aggregations for an event.
-
-        Note that this does not use a cache, but depends on cached methods.
-
-        Args:
-            event: The event to calculate bundled aggregations for.
-            ignored_users: The users ignored by the requesting user.
-
-        Returns:
-            The bundled aggregations for an event, if bundled aggregations are
-            enabled and the event can have bundled aggregations.
-        """
-
-        # Do not bundle aggregations for an event which represents an edit or an
-        # annotation. It does not make sense for them to have related events.
-        relates_to = event.content.get("m.relates_to")
-        if isinstance(relates_to, (dict, frozendict)):
-            relation_type = relates_to.get("rel_type")
-            if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE):
-                return None
-
-        event_id = event.event_id
-        room_id = event.room_id
-
-        # The bundled aggregations to include, a mapping of relation type to a
-        # type-specific value. Some types include the direct return type here
-        # while others need more processing during serialization.
-        aggregations = BundledAggregations()
-
-        annotations = await self.get_annotations_for_event(
-            event_id, room_id, ignored_users=ignored_users
-        )
-        if annotations:
-            aggregations.annotations = {"chunk": annotations}
-
-        references, next_token = await self.get_relations_for_event(
-            event_id,
-            event,
-            room_id,
-            RelationTypes.REFERENCE,
-            ignored_users=ignored_users,
-        )
-        if references:
-            aggregations.references = {
-                "chunk": [{"event_id": event.event_id} for event in references]
-            }
-
-            if next_token:
-                aggregations.references["next_batch"] = await next_token.to_string(
-                    self._main_store
-                )
-
-        # Store the bundled aggregations in the event metadata for later use.
-        return aggregations
-
     async def get_threads_for_events(
         self, event_ids: Collection[str], user_id: str, ignored_users: FrozenSet[str]
     ) -> Dict[str, _ThreadAggregation]:
@@ -435,11 +377,39 @@ class RelationsHandler:
 
         # Fetch other relations per event.
         for event in events_by_id.values():
-            event_result = await self._get_bundled_aggregation_for_event(
-                event, ignored_users
+            # Do not bundle aggregations for an event which represents an edit or an
+            # annotation. It does not make sense for them to have related events.
+            relates_to = event.content.get("m.relates_to")
+            if isinstance(relates_to, (dict, frozendict)):
+                relation_type = relates_to.get("rel_type")
+                if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE):
+                    continue
+
+            annotations = await self.get_annotations_for_event(
+                event.event_id, event.room_id, ignored_users=ignored_users
             )
-            if event_result:
-                results[event.event_id] = event_result
+            if annotations:
+                results.setdefault(
+                    event.event_id, BundledAggregations()
+                ).annotations = {"chunk": annotations}
+
+            references, next_token = await self.get_relations_for_event(
+                event.event_id,
+                event,
+                event.room_id,
+                RelationTypes.REFERENCE,
+                ignored_users=ignored_users,
+            )
+            if references:
+                aggregations = results.setdefault(event.event_id, BundledAggregations())
+                aggregations.references = {
+                    "chunk": [{"event_id": ev.event_id} for ev in references]
+                }
+
+                if next_token:
+                    aggregations.references["next_batch"] = await next_token.to_string(
+                        self._main_store
+                    )
 
         # Fetch any edits (but not for redacted events).
         #
diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py
index deb3a86af..1c4eca5a4 100644
--- a/synapse/handlers/room_batch.py
+++ b/synapse/handlers/room_batch.py
@@ -55,7 +55,7 @@ class RoomBatchHandler:
         # it has a larger `depth` but before the successor event because the `stream_ordering`
         # is negative before the successor event.
         successor_event_ids = await self.store.get_successor_events(
-            [most_recent_prev_event_id]
+            most_recent_prev_event_id
         )
 
         # If we can't find any successor events, then it's a forward extremity of
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 398d6110e..f6def883c 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -661,16 +661,15 @@ class SyncHandler:
             stream_position: point at which to get state
             state_filter: The state filter used to fetch state from the database.
         """
-        # FIXME this claims to get the state at a stream position, but
-        # get_recent_events_for_room operates by topo ordering. This therefore
-        # does not reliably give you the state at the given stream position.
-        # (https://github.com/matrix-org/synapse/issues/3305)
-        last_events, _ = await self.store.get_recent_events_for_room(
-            room_id, end_token=stream_position.room_key, limit=1
+        # FIXME: This gets the state at the latest event before the stream ordering,
+        # which might not be the same as the "current state" of the room at the time
+        # of the stream token if there were multiple forward extremities at the time.
+        last_event = await self.store.get_last_event_in_room_before_stream_ordering(
+            room_id,
+            end_token=stream_position.room_key,
         )
 
-        if last_events:
-            last_event = last_events[-1]
+        if last_event:
             state = await self.get_state_after_event(
                 last_event, state_filter=state_filter or StateFilter.all()
             )
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 5097b3ca5..e68644595 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -704,6 +704,9 @@ class MatrixFederationHttpClient:
         Returns:
             A list of headers to be added as "Authorization:" headers
         """
+        if destination is None and destination_is is None:
+            raise ValueError("destination and destination_is cannot both be None!")
+
         request: JsonDict = {
             "method": method.decode("ascii"),
             "uri": url_bytes.decode("ascii"),
@@ -726,8 +729,13 @@ class MatrixFederationHttpClient:
         for key, sig in request["signatures"][self.server_name].items():
             auth_headers.append(
                 (
-                    'X-Matrix origin=%s,key="%s",sig="%s"'
-                    % (self.server_name, key, sig)
+                    'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"'
+                    % (
+                        self.server_name,
+                        key,
+                        sig,
+                        request.get("destination") or request["destination_is"],
+                    )
                 ).encode("ascii")
             )
         return auth_headers
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
deleted file mode 100644
index ec199a161..000000000
--- a/synapse/python_dependencies.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# Copyright 2015, 2016 OpenMarket Ltd
-# Copyright 2017 Vector Creations Ltd
-# Copyright 2018 New Vector Ltd
-# Copyright 2020 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import itertools
-import logging
-from typing import Set
-
-logger = logging.getLogger(__name__)
-
-
-# REQUIREMENTS is a simple list of requirement specifiers[1], and must be
-# installed. It is passed to setup() as install_requires in setup.py.
-#
-# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict
-# of lists. The dict key is the optional dependency name and can be passed to
-# pip when installing. The list is a series of requirement specifiers[1] to be
-# installed when that optional dependency requirement is specified. It is passed
-# to setup() as extras_require in setup.py
-#
-# Note that these both represent runtime dependencies (and the versions
-# installed are checked at runtime).
-#
-# Also note that we replicate these constraints in the Synapse Dockerfile while
-# pre-installing dependencies. If these constraints are updated here, the same
-# change should be made in the Dockerfile.
-#
-# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers.
-
-REQUIREMENTS = [
-    # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
-    "jsonschema>=3.0.0",
-    # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
-    "frozendict>=1,!=2.1.2",
-    "unpaddedbase64>=1.1.0",
-    "canonicaljson>=1.4.0",
-    # we use the type definitions added in signedjson 1.1.
-    "signedjson>=1.1.0",
-    "pynacl>=1.2.1",
-    # validating SSL certs for IP addresses requires service_identity 18.1.
-    "service_identity>=18.1.0",
-    # Twisted 18.9 introduces some logger improvements that the structured
-    # logger utilises
-    "Twisted[tls]>=18.9.0",
-    "treq>=15.1",
-    # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
-    "pyopenssl>=16.0.0",
-    "pyyaml>=3.11",
-    "pyasn1>=0.1.9",
-    "pyasn1-modules>=0.0.7",
-    "bcrypt>=3.1.0",
-    "pillow>=5.4.0",
-    "sortedcontainers>=1.4.4",
-    "pymacaroons>=0.13.0",
-    "msgpack>=0.5.2",
-    "phonenumbers>=8.2.0",
-    # we use GaugeHistogramMetric, which was added in prom-client 0.4.0.
-    "prometheus_client>=0.4.0",
-    # we use `order`, which arrived in attrs 19.2.0.
-    # Note: 21.1.0 broke `/sync`, see #9936
-    "attrs>=19.2.0,!=21.1.0",
-    "netaddr>=0.7.18",
-    # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not
-    # end up with a broken installation, with recent MarkupSafe but old Jinja, we
-    # add a lower bound to the Jinja2 dependency.
-    "Jinja2>=3.0",
-    "bleach>=1.4.3",
-    # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0.
-    "typing-extensions>=3.10.0",
-    # We enforce that we have a `cryptography` version that bundles an `openssl`
-    # with the latest security patches.
-    "cryptography>=3.4.7",
-    # ijson 3.1.4 fixes a bug with "." in property names
-    "ijson>=3.1.4",
-    "matrix-common~=1.1.0",
-    # We need packaging.requirements.Requirement, added in 16.1.
-    "packaging>=16.1",
-    # At the time of writing, we only use functions from the version `importlib.metadata`
-    # which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
-    "importlib_metadata>=1.4 ; python_version < '3.8'",
-]
-
-CONDITIONAL_REQUIREMENTS = {
-    "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"],
-    "postgres": [
-        # we use execute_values with the fetch param, which arrived in psycopg 2.8.
-        "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'",
-        "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'",
-        "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'",
-    ],
-    "saml2": [
-        "pysaml2>=4.5.0",
-    ],
-    "oidc": ["authlib>=0.14.0"],
-    # systemd-python is necessary for logging to the systemd journal via
-    # `systemd.journal.JournalHandler`, as is documented in
-    # `contrib/systemd/log_config.yaml`.
-    "systemd": ["systemd-python>=231"],
-    "url_preview": ["lxml>=4.2.0"],
-    "sentry": ["sentry-sdk>=0.7.2"],
-    "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"],
-    "jwt": ["pyjwt>=1.6.4"],
-    # hiredis is not a *strict* dependency, but it makes things much faster.
-    # (if it is not installed, we fall back to slow code.)
-    "redis": ["txredisapi>=1.4.7", "hiredis"],
-    # Required to use experimental `caches.track_memory_usage` config option.
-    "cache_memory": ["pympler"],
-}
-
-ALL_OPTIONAL_REQUIREMENTS: Set[str] = set()
-
-for name, optional_deps in CONDITIONAL_REQUIREMENTS.items():
-    # Exclude systemd as it's a system-based requirement.
-    # Exclude lint as it's a dev-based requirement.
-    if name not in ["systemd"]:
-        ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
-
-
-# ensure there are no double-quote characters in any of the deps (otherwise the
-# 'pip install' incantation in DependencyException will break)
-for dep in itertools.chain(
-    REQUIREMENTS,
-    *CONDITIONAL_REQUIREMENTS.values(),
-):
-    if '"' in dep:
-        raise Exception(
-            "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,)
-        )
-
-
-def list_requirements():
-    return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
-
-
-if __name__ == "__main__":
-    import sys
-
-    sys.stdout.writelines(req + "\n" for req in list_requirements())
diff --git a/synapse/res/templates/notif.html b/synapse/res/templates/notif.html
index 0aaef97df..7d86681fe 100644
--- a/synapse/res/templates/notif.html
+++ b/synapse/res/templates/notif.html
@@ -30,7 +30,7 @@
                     {%- elif message.msgtype == "m.notice" %}
                         {{ message.body_text_html }}
                     {%- elif message.msgtype == "m.image" and message.image_url %}
-                        <img src="{{ message.image_url|mxc_to_http(640, 480, scale) }}" />
+                        <img src="{{ message.image_url|mxc_to_http(640, 480, 'scale') }}" />
                     {%- elif message.msgtype == "m.file" %}
                         <span class="filename">{{ message.body_text_plain }}</span>
                     {%- else %}
diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py
index c9d44c596..4a4dbe75d 100644
--- a/synapse/rest/client/login.py
+++ b/synapse/rest/client/login.py
@@ -342,6 +342,15 @@ class LoginRestServlet(RestServlet):
             user_id = canonical_uid
 
         device_id = login_submission.get("device_id")
+
+        # If device_id is present, check that device_id is not longer than a reasonable 512 characters
+        if device_id and len(device_id) > 512:
+            raise LoginError(
+                400,
+                "device_id cannot be longer than 512 characters.",
+                errcode=Codes.INVALID_PARAM,
+            )
+
         initial_display_name = login_submission.get("initial_device_display_name")
         (
             device_id,
diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
index fa60440a9..73598d13d 100644
--- a/synapse/rest/client/room.py
+++ b/synapse/rest/client/room.py
@@ -21,6 +21,7 @@ from urllib import parse as urlparse
 
 from twisted.web.server import Request
 
+from synapse import event_auth
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import (
     AuthError,
@@ -29,6 +30,7 @@ from synapse.api.errors import (
     MissingClientTokenError,
     ShadowBanError,
     SynapseError,
+    UnredactedContentDeletedError,
 )
 from synapse.api.filtering import Filter
 from synapse.events.utils import format_event_for_client_v2
@@ -647,18 +649,55 @@ class RoomEventServlet(RestServlet):
         super().__init__()
         self.clock = hs.get_clock()
         self._store = hs.get_datastores().main
+        self._state = hs.get_state_handler()
         self.event_handler = hs.get_event_handler()
         self._event_serializer = hs.get_event_client_serializer()
         self._relations_handler = hs.get_relations_handler()
         self.auth = hs.get_auth()
+        self.content_keep_ms = hs.config.server.redaction_retention_period
+        self.msc2815_enabled = hs.config.experimental.msc2815_enabled
 
     async def on_GET(
         self, request: SynapseRequest, room_id: str, event_id: str
     ) -> Tuple[int, JsonDict]:
         requester = await self.auth.get_user_by_req(request, allow_guest=True)
+
+        include_unredacted_content = self.msc2815_enabled and (
+            parse_string(
+                request,
+                "fi.mau.msc2815.include_unredacted_content",
+                allowed_values=("true", "false"),
+            )
+            == "true"
+        )
+        if include_unredacted_content and not await self.auth.is_server_admin(
+            requester.user
+        ):
+            power_level_event = await self._state.get_current_state(
+                room_id, EventTypes.PowerLevels, ""
+            )
+
+            auth_events = {}
+            if power_level_event:
+                auth_events[(EventTypes.PowerLevels, "")] = power_level_event
+
+            redact_level = event_auth.get_named_level(auth_events, "redact", 50)
+            user_level = event_auth.get_user_power_level(
+                requester.user.to_string(), auth_events
+            )
+            if user_level < redact_level:
+                raise SynapseError(
+                    403,
+                    "You don't have permission to view redacted events in this room.",
+                    errcode=Codes.FORBIDDEN,
+                )
+
         try:
             event = await self.event_handler.get_event(
-                requester.user, room_id, event_id
+                requester.user,
+                room_id,
+                event_id,
+                show_redacted=include_unredacted_content,
             )
         except AuthError:
             # This endpoint is supposed to return a 404 when the requester does
@@ -667,14 +706,21 @@ class RoomEventServlet(RestServlet):
             raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND)
 
         if event:
+            if include_unredacted_content and await self._store.have_censored_event(
+                event_id
+            ):
+                raise UnredactedContentDeletedError(self.content_keep_ms)
+
             # Ensure there are bundled aggregations available.
             aggregations = await self._relations_handler.get_bundled_aggregations(
                 [event], requester.user.to_string()
             )
 
             time_now = self.clock.time_msec()
+            # per MSC2676, /rooms/{roomId}/event/{eventId}, should return the
+            # *original* event, rather than the edited version
             event_dict = self._event_serializer.serialize_event(
-                event, time_now, bundle_aggregations=aggregations
+                event, time_now, bundle_aggregations=aggregations, apply_edits=False
             )
             return 200, event_dict
 
diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py
index 2e25e8638..e8772f86e 100644
--- a/synapse/rest/client/sync.py
+++ b/synapse/rest/client/sync.py
@@ -180,13 +180,10 @@ class SyncRestServlet(RestServlet):
 
         affect_presence = set_presence != PresenceState.OFFLINE
 
-        if affect_presence:
-            await self.presence_handler.set_state(
-                user, {"presence": set_presence}, True
-            )
-
         context = await self.presence_handler.user_syncing(
-            user.to_string(), affect_presence=affect_presence
+            user.to_string(),
+            affect_presence=affect_presence,
+            presence_state=set_presence,
         )
         with context:
             sync_result = await self.sync_handler.wait_for_sync_for_user(
diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py
index 9a65aa484..bfc1d4ee0 100644
--- a/synapse/rest/client/versions.py
+++ b/synapse/rest/client/versions.py
@@ -86,7 +86,7 @@ class VersionsRestServlet(RestServlet):
                     # Implements additional endpoints as described in MSC2432
                     "org.matrix.msc2432": True,
                     # Implements additional endpoints as described in MSC2666
-                    "uk.half-shot.msc2666": True,
+                    "uk.half-shot.msc2666.mutual_rooms": True,
                     # Whether new rooms will be set to encrypted or not (based on presets).
                     "io.element.e2ee_forced.public": self.e2ee_forced_public,
                     "io.element.e2ee_forced.private": self.e2ee_forced_private,
@@ -100,8 +100,9 @@ class VersionsRestServlet(RestServlet):
                     # Adds support for jump to date endpoints (/timestamp_to_event) as per MSC3030
                     "org.matrix.msc3030": self.config.experimental.msc3030_enabled,
                     # Adds support for thread relations, per MSC3440.
-                    "org.matrix.msc3440": self.config.experimental.msc3440_enabled,
                     "org.matrix.msc3440.stable": True,  # TODO: remove when "v1.3" is added above
+                    # Allows moderators to fetch redacted event content as described in MSC2815
+                    "fi.mau.msc2815": self.config.experimental.msc2815_enabled,
                 },
             },
         )
diff --git a/synapse/server.py b/synapse/server.py
index 380369db9..37c72bd83 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -758,7 +758,7 @@ class HomeServer(metaclass=abc.ABCMeta):
 
     @cache_in_self
     def get_event_client_serializer(self) -> EventClientSerializer:
-        return EventClientSerializer(self)
+        return EventClientSerializer()
 
     @cache_in_self
     def get_password_policy_handler(self) -> PasswordPolicyHandler:
diff --git a/synapse/storage/database.py b/synapse/storage/database.py
index 5eb545c86..df1e9c1b8 100644
--- a/synapse/storage/database.py
+++ b/synapse/storage/database.py
@@ -41,7 +41,6 @@ from prometheus_client import Histogram
 from typing_extensions import Literal
 
 from twisted.enterprise import adbapi
-from twisted.internet import defer
 
 from synapse.api.errors import StoreError
 from synapse.config.database import DatabaseConnectionConfig
@@ -794,7 +793,7 @@ class DatabasePool:
         # We also wait until everything above is done before releasing the
         # `CancelledError`, so that logging contexts won't get used after they have been
         # finished.
-        return await delay_cancellation(defer.ensureDeferred(_runInteraction()))
+        return await delay_cancellation(_runInteraction())
 
     async def runWithConnection(
         self,
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index dc8009b23..318e4df37 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -1582,7 +1582,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
         self,
         user_id: str,
         device_ids: Collection[str],
-        hosts: Optional[Collection[str]],
         room_ids: Collection[str],
     ) -> Optional[int]:
         """Persist that a user's devices have been updated, and which hosts
@@ -1592,9 +1591,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
             user_id: The ID of the user whose device changed.
             device_ids: The IDs of any changed devices. If empty, this function will
                 return None.
-            hosts: The remote destinations that should be notified of the change. If
-                None then the set of hosts have *not* been calculated, and will be
-                calculated later by a background task.
             room_ids: The rooms that the user is in
 
         Returns:
@@ -1606,14 +1602,12 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
 
         context = get_active_span_text_map()
 
-        def add_device_changes_txn(
-            txn, stream_ids_for_device_change, stream_ids_for_outbound_pokes
-        ):
+        def add_device_changes_txn(txn, stream_ids):
             self._add_device_change_to_stream_txn(
                 txn,
                 user_id,
                 device_ids,
-                stream_ids_for_device_change,
+                stream_ids,
             )
 
             self._add_device_outbound_room_poke_txn(
@@ -1621,43 +1615,17 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
                 user_id,
                 device_ids,
                 room_ids,
-                stream_ids_for_device_change,
-                context,
-                hosts_have_been_calculated=hosts is not None,
-            )
-
-            # If the set of hosts to send to has not been calculated yet (and so
-            # `hosts` is None) or there are no `hosts` to send to, then skip
-            # trying to persist them to the DB.
-            if not hosts:
-                return
-
-            self._add_device_outbound_poke_to_stream_txn(
-                txn,
-                user_id,
-                device_ids,
-                hosts,
-                stream_ids_for_outbound_pokes,
+                stream_ids,
                 context,
             )
 
-        # `device_lists_stream` wants a stream ID per device update.
-        num_stream_ids = len(device_ids)
-
-        if hosts:
-            # `device_lists_outbound_pokes` wants a different stream ID for
-            # each row, which is a row per host per device update.
-            num_stream_ids += len(hosts) * len(device_ids)
-
-        async with self._device_list_id_gen.get_next_mult(num_stream_ids) as stream_ids:
-            stream_ids_for_device_change = stream_ids[: len(device_ids)]
-            stream_ids_for_outbound_pokes = stream_ids[len(device_ids) :]
-
+        async with self._device_list_id_gen.get_next_mult(
+            len(device_ids)
+        ) as stream_ids:
             await self.db_pool.runInteraction(
                 "add_device_change_to_stream",
                 add_device_changes_txn,
-                stream_ids_for_device_change,
-                stream_ids_for_outbound_pokes,
+                stream_ids,
             )
 
         return stream_ids[-1]
@@ -1735,7 +1703,9 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
                     next(stream_id_iterator),
                     user_id,
                     device_id,
-                    False,
+                    not self.hs.is_mine_id(
+                        user_id
+                    ),  # We only need to send out update for *our* users
                     now,
                     encoded_context if whitelisted_homeserver(destination) else "{}",
                 )
@@ -1752,19 +1722,8 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
         room_ids: Collection[str],
         stream_ids: List[str],
         context: Dict[str, str],
-        hosts_have_been_calculated: bool,
     ) -> None:
-        """Record the user in the room has updated their device.
-
-        Args:
-            hosts_have_been_calculated: True if `device_lists_outbound_pokes`
-                has been updated already with the updates.
-        """
-
-        # We only need to convert to outbound pokes if they are our user.
-        converted_to_destinations = (
-            hosts_have_been_calculated or not self.hs.is_mine_id(user_id)
-        )
+        """Record the user in the room has updated their device."""
 
         encoded_context = json_encoder.encode(context)
 
@@ -1789,7 +1748,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
                     device_id,
                     room_id,
                     stream_id,
-                    converted_to_destinations,
+                    False,
                     encoded_context,
                 )
                 for room_id in room_ids
diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py
index 634e19e03..471022470 100644
--- a/synapse/storage/databases/main/event_federation.py
+++ b/synapse/storage/databases/main/event_federation.py
@@ -695,7 +695,9 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
         # Return all events where not all sets can reach them.
         return {eid for eid, n in event_to_missing_sets.items() if n}
 
-    async def get_oldest_event_ids_with_depth_in_room(self, room_id) -> Dict[str, int]:
+    async def get_oldest_event_ids_with_depth_in_room(
+        self, room_id
+    ) -> List[Tuple[str, int]]:
         """Gets the oldest events(backwards extremities) in the room along with the
         aproximate depth.
 
@@ -708,7 +710,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             room_id: Room where we want to find the oldest events
 
         Returns:
-            Map from event_id to depth
+            List of (event_id, depth) tuples
         """
 
         def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id):
@@ -741,7 +743,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
 
             txn.execute(sql, (room_id, False))
 
-            return dict(txn)
+            return txn.fetchall()
 
         return await self.db_pool.runInteraction(
             "get_oldest_event_ids_with_depth_in_room",
@@ -751,7 +753,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
 
     async def get_insertion_event_backward_extremities_in_room(
         self, room_id
-    ) -> Dict[str, int]:
+    ) -> List[Tuple[str, int]]:
         """Get the insertion events we know about that we haven't backfilled yet.
 
         We use this function so that we can compare and see if someones current
@@ -763,7 +765,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             room_id: Room where we want to find the oldest events
 
         Returns:
-            Map from event_id to depth
+            List of (event_id, depth) tuples
         """
 
         def get_insertion_event_backward_extremities_in_room_txn(txn, room_id):
@@ -778,8 +780,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             """
 
             txn.execute(sql, (room_id,))
-
-            return dict(txn)
+            return txn.fetchall()
 
         return await self.db_pool.runInteraction(
             "get_insertion_event_backward_extremities_in_room",
@@ -1295,22 +1296,19 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
         event_results.reverse()
         return event_results
 
-    async def get_successor_events(self, event_ids: Iterable[str]) -> List[str]:
-        """Fetch all events that have the given events as a prev event
+    async def get_successor_events(self, event_id: str) -> List[str]:
+        """Fetch all events that have the given event as a prev event
 
         Args:
-            event_ids: The events to use as the previous events.
+            event_id: The event to search for as a prev_event.
         """
-        rows = await self.db_pool.simple_select_many_batch(
+        return await self.db_pool.simple_select_onecol(
             table="event_edges",
-            column="prev_event_id",
-            iterable=event_ids,
-            retcols=("event_id",),
+            keyvalues={"prev_event_id": event_id},
+            retcol="event_id",
             desc="get_successor_events",
         )
 
-        return [row["event_id"] for row in rows]
-
     @wrap_as_background_process("delete_old_forward_extrem_cache")
     async def _delete_old_forward_extrem_cache(self) -> None:
         def _delete_old_forward_extrem_cache_txn(txn):
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 3fcd5f5b9..2a1e567ce 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -963,6 +963,21 @@ class PersistEventsStore:
                 values=to_insert,
             )
 
+    async def update_current_state(
+        self,
+        room_id: str,
+        state_delta: DeltaState,
+        stream_id: int,
+    ) -> None:
+        """Update the current state stored in the datatabase for the given room"""
+
+        await self.db_pool.runInteraction(
+            "update_current_state",
+            self._update_current_state_txn,
+            state_delta_by_room={room_id: state_delta},
+            stream_id=stream_id,
+        )
+
     def _update_current_state_txn(
         self,
         txn: LoggingTransaction,
@@ -1819,10 +1834,7 @@ class PersistEventsStore:
         if rel_type == RelationTypes.REPLACE:
             txn.call_after(self.store.get_applicable_edit.invalidate, (parent_id,))
 
-        if (
-            rel_type == RelationTypes.THREAD
-            or rel_type == RelationTypes.UNSTABLE_THREAD
-        ):
+        if rel_type == RelationTypes.THREAD:
             txn.call_after(self.store.get_thread_summary.invalidate, (parent_id,))
             # It should be safe to only invalidate the cache if the user has not
             # previously participated in the thread, but that's difficult (and
diff --git a/synapse/storage/databases/main/events_forward_extremities.py b/synapse/storage/databases/main/events_forward_extremities.py
index 68901b433..f851bff60 100644
--- a/synapse/storage/databases/main/events_forward_extremities.py
+++ b/synapse/storage/databases/main/events_forward_extremities.py
@@ -66,13 +66,15 @@ class EventForwardExtremitiesStore(
             """
 
             txn.execute(sql, (event_id, room_id))
+
+            deleted_count = txn.rowcount
             logger.info(
                 "Deleted %s extra forward extremities for room %s",
-                txn.rowcount,
+                deleted_count,
                 room_id,
             )
 
-            if txn.rowcount > 0:
+            if deleted_count > 0:
                 # Invalidate the cache
                 self._invalidate_cache_and_stream(
                     txn,
@@ -80,7 +82,7 @@ class EventForwardExtremitiesStore(
                     (room_id,),
                 )
 
-            return txn.rowcount
+            return deleted_count
 
         return await self.db_pool.runInteraction(
             "delete_forward_extremities_for_room",
diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py
index a60e3f4fd..c31fc00ea 100644
--- a/synapse/storage/databases/main/events_worker.py
+++ b/synapse/storage/databases/main/events_worker.py
@@ -75,7 +75,7 @@ from synapse.storage.util.id_generators import (
 from synapse.storage.util.sequence import build_sequence_generator
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util import unwrapFirstError
-from synapse.util.async_helpers import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred, delay_cancellation
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.iterutils import batch_iter
@@ -303,6 +303,24 @@ class EventsWorkerStore(SQLBaseStore):
             desc="get_received_ts",
         )
 
+    async def have_censored_event(self, event_id: str) -> bool:
+        """Check if an event has been censored, i.e. if the content of the event has been erased
+        from the database due to a redaction.
+
+        Args:
+            event_id: The event ID that was redacted.
+
+        Returns:
+            True if the event has been censored, False otherwise.
+        """
+        censored_redactions_list = await self.db_pool.simple_select_onecol(
+            table="redactions",
+            keyvalues={"redacts": event_id},
+            retcol="have_censored",
+            desc="get_have_censored",
+        )
+        return any(censored_redactions_list)
+
     # Inform mypy that if allow_none is False (the default) then get_event
     # always returns an EventBase.
     @overload
@@ -622,42 +640,57 @@ class EventsWorkerStore(SQLBaseStore):
         missing_events_ids.difference_update(already_fetching_ids)
 
         if missing_events_ids:
-            log_ctx = current_context()
-            log_ctx.record_event_fetch(len(missing_events_ids))
 
-            # Add entries to `self._current_event_fetches` for each event we're
-            # going to pull from the DB. We use a single deferred that resolves
-            # to all the events we pulled from the DB (this will result in this
-            # function returning more events than requested, but that can happen
-            # already due to `_get_events_from_db`).
-            fetching_deferred: ObservableDeferred[
-                Dict[str, EventCacheEntry]
-            ] = ObservableDeferred(defer.Deferred(), consumeErrors=True)
-            for event_id in missing_events_ids:
-                self._current_event_fetches[event_id] = fetching_deferred
+            async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]:
+                """Fetches the events in `missing_event_ids` from the database.
 
-            # Note that _get_events_from_db is also responsible for turning db rows
-            # into FrozenEvents (via _get_event_from_row), which involves seeing if
-            # the events have been redacted, and if so pulling the redaction event out
-            # of the database to check it.
-            #
-            try:
-                missing_events = await self._get_events_from_db(
-                    missing_events_ids,
-                )
+                Also creates entries in `self._current_event_fetches` to allow
+                concurrent `_get_events_from_cache_or_db` calls to reuse the same fetch.
+                """
+                log_ctx = current_context()
+                log_ctx.record_event_fetch(len(missing_events_ids))
 
-                event_entry_map.update(missing_events)
-            except Exception as e:
-                with PreserveLoggingContext():
-                    fetching_deferred.errback(e)
-                raise e
-            finally:
-                # Ensure that we mark these events as no longer being fetched.
+                # Add entries to `self._current_event_fetches` for each event we're
+                # going to pull from the DB. We use a single deferred that resolves
+                # to all the events we pulled from the DB (this will result in this
+                # function returning more events than requested, but that can happen
+                # already due to `_get_events_from_db`).
+                fetching_deferred: ObservableDeferred[
+                    Dict[str, EventCacheEntry]
+                ] = ObservableDeferred(defer.Deferred(), consumeErrors=True)
                 for event_id in missing_events_ids:
-                    self._current_event_fetches.pop(event_id, None)
+                    self._current_event_fetches[event_id] = fetching_deferred
 
-            with PreserveLoggingContext():
-                fetching_deferred.callback(missing_events)
+                # Note that _get_events_from_db is also responsible for turning db rows
+                # into FrozenEvents (via _get_event_from_row), which involves seeing if
+                # the events have been redacted, and if so pulling the redaction event
+                # out of the database to check it.
+                #
+                try:
+                    missing_events = await self._get_events_from_db(
+                        missing_events_ids,
+                    )
+                except Exception as e:
+                    with PreserveLoggingContext():
+                        fetching_deferred.errback(e)
+                    raise e
+                finally:
+                    # Ensure that we mark these events as no longer being fetched.
+                    for event_id in missing_events_ids:
+                        self._current_event_fetches.pop(event_id, None)
+
+                with PreserveLoggingContext():
+                    fetching_deferred.callback(missing_events)
+
+                return missing_events
+
+            # We must allow the database fetch to complete in the presence of
+            # cancellations, since multiple `_get_events_from_cache_or_db` calls can
+            # reuse the same fetch.
+            missing_events: Dict[str, EventCacheEntry] = await delay_cancellation(
+                get_missing_events_from_db()
+            )
+            event_entry_map.update(missing_events)
 
         if already_fetching_deferreds:
             # Wait for the other event requests to finish and add their results
@@ -1956,7 +1989,15 @@ class EventsWorkerStore(SQLBaseStore):
     async def get_partial_state_events(
         self, event_ids: Collection[str]
     ) -> Dict[str, bool]:
-        """Checks which of the given events have partial state"""
+        """Checks which of the given events have partial state
+
+        Args:
+            event_ids: the events we want to check for partial state.
+
+        Returns:
+            a dict mapping from event id to partial-stateness. We return True for
+            any of the events which are unknown (or are outliers).
+        """
         result = await self.db_pool.simple_select_many_batch(
             table="partial_state_events",
             column="event_id",
@@ -1979,3 +2020,27 @@ class EventsWorkerStore(SQLBaseStore):
             desc="is_partial_state_event",
         )
         return result is not None
+
+    async def get_partial_state_events_batch(self, room_id: str) -> List[str]:
+        """Get a list of events in the given room that have partial state"""
+        return await self.db_pool.runInteraction(
+            "get_partial_state_events_batch",
+            self._get_partial_state_events_batch_txn,
+            room_id,
+        )
+
+    @staticmethod
+    def _get_partial_state_events_batch_txn(
+        txn: LoggingTransaction, room_id: str
+    ) -> List[str]:
+        txn.execute(
+            """
+            SELECT event_id FROM partial_state_events AS pse
+                JOIN events USING (event_id)
+            WHERE pse.room_id = ?
+            ORDER BY events.stream_ordering
+            LIMIT 100
+            """,
+            (room_id,),
+        )
+        return [row[0] for row in txn]
diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py
index 407158cee..a5c31f678 100644
--- a/synapse/storage/databases/main/relations.py
+++ b/synapse/storage/databases/main/relations.py
@@ -14,7 +14,6 @@
 
 import logging
 from typing import (
-    TYPE_CHECKING,
     Collection,
     Dict,
     FrozenSet,
@@ -32,20 +31,12 @@ import attr
 from synapse.api.constants import RelationTypes
 from synapse.events import EventBase
 from synapse.storage._base import SQLBaseStore
-from synapse.storage.database import (
-    DatabasePool,
-    LoggingDatabaseConnection,
-    LoggingTransaction,
-    make_in_list_sql_clause,
-)
+from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause
 from synapse.storage.databases.main.stream import generate_pagination_where_clause
 from synapse.storage.engines import PostgresEngine
 from synapse.types import JsonDict, RoomStreamToken, StreamToken
 from synapse.util.caches.descriptors import cached, cachedList
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
 
 
@@ -63,16 +54,6 @@ class _RelatedEvent:
 
 
 class RelationsWorkerStore(SQLBaseStore):
-    def __init__(
-        self,
-        database: DatabasePool,
-        db_conn: LoggingDatabaseConnection,
-        hs: "HomeServer",
-    ):
-        super().__init__(database, db_conn, hs)
-
-        self._msc3440_enabled = hs.config.experimental.msc3440_enabled
-
     @cached(uncached_args=("event",), tree=True)
     async def get_relations_for_event(
         self,
@@ -497,7 +478,7 @@ class RelationsWorkerStore(SQLBaseStore):
                         AND parent.room_id = child.room_id
                     WHERE
                         %s
-                        AND %s
+                        AND relation_type = ?
                     ORDER BY parent.event_id, child.topological_ordering DESC, child.stream_ordering DESC
                 """
             else:
@@ -512,22 +493,16 @@ class RelationsWorkerStore(SQLBaseStore):
                         AND parent.room_id = child.room_id
                     WHERE
                         %s
-                        AND %s
+                        AND relation_type = ?
                     ORDER BY child.topological_ordering DESC, child.stream_ordering DESC
                 """
 
             clause, args = make_in_list_sql_clause(
                 txn.database_engine, "relates_to_id", event_ids
             )
+            args.append(RelationTypes.THREAD)
 
-            if self._msc3440_enabled:
-                relations_clause = "(relation_type = ? OR relation_type = ?)"
-                args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD))
-            else:
-                relations_clause = "relation_type = ?"
-                args.append(RelationTypes.THREAD)
-
-            txn.execute(sql % (clause, relations_clause), args)
+            txn.execute(sql % (clause,), args)
             latest_event_ids = {}
             for parent_event_id, child_event_id in txn:
                 # Only consider the latest threaded reply (by topological ordering).
@@ -547,7 +522,7 @@ class RelationsWorkerStore(SQLBaseStore):
                     AND parent.room_id = child.room_id
                 WHERE
                     %s
-                    AND %s
+                    AND relation_type = ?
                 GROUP BY parent.event_id
             """
 
@@ -556,15 +531,9 @@ class RelationsWorkerStore(SQLBaseStore):
             clause, args = make_in_list_sql_clause(
                 txn.database_engine, "relates_to_id", latest_event_ids.keys()
             )
+            args.append(RelationTypes.THREAD)
 
-            if self._msc3440_enabled:
-                relations_clause = "(relation_type = ? OR relation_type = ?)"
-                args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD))
-            else:
-                relations_clause = "relation_type = ?"
-                args.append(RelationTypes.THREAD)
-
-            txn.execute(sql % (clause, relations_clause), args)
+            txn.execute(sql % (clause,), args)
             counts = dict(cast(List[Tuple[str, int]], txn.fetchall()))
 
             return counts, latest_event_ids
@@ -622,7 +591,7 @@ class RelationsWorkerStore(SQLBaseStore):
                 parent.event_id = relates_to_id
                 AND parent.room_id = child.room_id
             WHERE
-                %s
+                relation_type = ?
                 AND %s
                 AND %s
             GROUP BY parent.event_id, child.sender
@@ -638,16 +607,9 @@ class RelationsWorkerStore(SQLBaseStore):
                 txn.database_engine, "relates_to_id", event_ids
             )
 
-            if self._msc3440_enabled:
-                relations_clause = "(relation_type = ? OR relation_type = ?)"
-                relations_args = [RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD]
-            else:
-                relations_clause = "relation_type = ?"
-                relations_args = [RelationTypes.THREAD]
-
             txn.execute(
-                sql % (users_sql, events_clause, relations_clause),
-                users_args + events_args + relations_args,
+                sql % (users_sql, events_clause),
+                [RelationTypes.THREAD] + users_args + events_args,
             )
             return {(row[0], row[1]): row[2] for row in txn}
 
@@ -677,7 +639,7 @@ class RelationsWorkerStore(SQLBaseStore):
             user participated in that event's thread, otherwise false.
         """
 
-        def _get_thread_summary_txn(txn: LoggingTransaction) -> Set[str]:
+        def _get_threads_participated_txn(txn: LoggingTransaction) -> Set[str]:
             # Fetch whether the requester has participated or not.
             sql = """
                 SELECT DISTINCT relates_to_id
@@ -688,28 +650,20 @@ class RelationsWorkerStore(SQLBaseStore):
                     AND parent.room_id = child.room_id
                 WHERE
                     %s
-                    AND %s
+                    AND relation_type = ?
                     AND child.sender = ?
             """
 
             clause, args = make_in_list_sql_clause(
                 txn.database_engine, "relates_to_id", event_ids
             )
+            args.extend([RelationTypes.THREAD, user_id])
 
-            if self._msc3440_enabled:
-                relations_clause = "(relation_type = ? OR relation_type = ?)"
-                args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD))
-            else:
-                relations_clause = "relation_type = ?"
-                args.append(RelationTypes.THREAD)
-
-            args.append(user_id)
-
-            txn.execute(sql % (clause, relations_clause), args)
+            txn.execute(sql % (clause,), args)
             return {row[0] for row in txn.fetchall()}
 
         participated_threads = await self.db_pool.runInteraction(
-            "get_thread_summary", _get_thread_summary_txn
+            "get_threads_participated", _get_threads_participated_txn
         )
 
         return {event_id: event_id in participated_threads for event_id in event_ids}
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index 18b1acd9e..87e9482c6 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -1077,6 +1077,37 @@ class RoomWorkerStore(CacheInvalidationWorkerStore):
             get_rooms_for_retention_period_in_range_txn,
         )
 
+    async def clear_partial_state_room(self, room_id: str) -> bool:
+        # this can race with incoming events, so we watch out for FK errors.
+        # TODO(faster_joins): this still doesn't completely fix the race, since the persist process
+        #   is not atomic. I fear we need an application-level lock.
+        try:
+            await self.db_pool.runInteraction(
+                "clear_partial_state_room", self._clear_partial_state_room_txn, room_id
+            )
+            return True
+        except self.db_pool.engine.module.DatabaseError as e:
+            # TODO(faster_joins): how do we distinguish between FK errors and other errors?
+            logger.warning(
+                "Exception while clearing lazy partial-state-room %s, retrying: %s",
+                room_id,
+                e,
+            )
+            return False
+
+    @staticmethod
+    def _clear_partial_state_room_txn(txn: LoggingTransaction, room_id: str) -> None:
+        DatabasePool.simple_delete_txn(
+            txn,
+            table="partial_state_rooms_servers",
+            keyvalues={"room_id": room_id},
+        )
+        DatabasePool.simple_delete_one_txn(
+            txn,
+            table="partial_state_rooms",
+            keyvalues={"room_id": room_id},
+        )
+
 
 class _BackgroundUpdates:
     REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory"
diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py
index ecdc1fdc4..e653841fe 100644
--- a/synapse/storage/databases/main/state.py
+++ b/synapse/storage/databases/main/state.py
@@ -21,6 +21,7 @@ from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
 from synapse.events import EventBase
+from synapse.events.snapshot import EventContext
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.database import (
     DatabasePool,
@@ -129,7 +130,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         )
 
         if room_version is None:
-            raise NotFoundError("Could not room_version for %s" % (room_id,))
+            raise NotFoundError("Could not find room_version for %s" % (room_id,))
 
         return room_version
 
@@ -354,6 +355,54 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         return {row["state_group"] for row in rows}
 
+    async def update_state_for_partial_state_event(
+        self,
+        event: EventBase,
+        context: EventContext,
+    ) -> None:
+        """Update the state group for a partial state event"""
+        await self.db_pool.runInteraction(
+            "update_state_for_partial_state_event",
+            self._update_state_for_partial_state_event_txn,
+            event,
+            context,
+        )
+
+    def _update_state_for_partial_state_event_txn(
+        self,
+        txn,
+        event: EventBase,
+        context: EventContext,
+    ):
+        # we shouldn't have any outliers here
+        assert not event.internal_metadata.is_outlier()
+
+        # anything that was rejected should have the same state as its
+        # predecessor.
+        if context.rejected:
+            assert context.state_group == context.state_group_before_event
+
+        self.db_pool.simple_update_txn(
+            txn,
+            table="event_to_state_groups",
+            keyvalues={"event_id": event.event_id},
+            updatevalues={"state_group": context.state_group},
+        )
+
+        self.db_pool.simple_delete_one_txn(
+            txn,
+            table="partial_state_events",
+            keyvalues={"event_id": event.event_id},
+        )
+
+        # TODO(faster_joins): need to do something about workers here
+        txn.call_after(self.is_partial_state_event.invalidate, (event.event_id,))
+        txn.call_after(
+            self._get_state_group_for_event.prefill,
+            (event.event_id,),
+            context.state_group,
+        )
+
 
 class MainStateBackgroundUpdateStore(RoomMemberWorkerStore):
 
diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py
index 6d45a8a9f..793e90663 100644
--- a/synapse/storage/databases/main/stream.py
+++ b/synapse/storage/databases/main/stream.py
@@ -758,6 +758,32 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             "get_room_event_before_stream_ordering", _f
         )
 
+    async def get_last_event_in_room_before_stream_ordering(
+        self,
+        room_id: str,
+        end_token: RoomStreamToken,
+    ) -> Optional[EventBase]:
+        """Returns the last event in a room at or before a stream ordering
+
+        Args:
+            room_id
+            end_token: The token used to stream from
+
+        Returns:
+            The most recent event.
+        """
+
+        last_row = await self.get_room_event_before_stream_ordering(
+            room_id=room_id,
+            stream_ordering=end_token.stream,
+        )
+        if last_row:
+            _, _, event_id = last_row
+            event = await self.get_event(event_id, get_prev_content=True)
+            return event
+
+        return None
+
     async def get_current_room_stream_token_for_room_id(
         self, room_id: Optional[str] = None
     ) -> RoomStreamToken:
diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py
index b40292281..e496ba7be 100644
--- a/synapse/storage/persist_events.py
+++ b/synapse/storage/persist_events.py
@@ -376,6 +376,62 @@ class EventsPersistenceStorage:
         pos = PersistedEventPosition(self._instance_name, event_stream_id)
         return event, pos, self.main_store.get_room_max_token()
 
+    async def update_current_state(self, room_id: str) -> None:
+        """Recalculate the current state for a room, and persist it"""
+        state = await self._calculate_current_state(room_id)
+        delta = await self._calculate_state_delta(room_id, state)
+
+        # TODO(faster_joins): get a real stream ordering, to make this work correctly
+        #    across workers.
+        #
+        # TODO(faster_joins): this can race against event persistence, in which case we
+        #    will end up with incorrect state. Perhaps we should make this a job we
+        #    farm out to the event persister, somehow.
+        stream_id = self.main_store.get_room_max_stream_ordering()
+        await self.persist_events_store.update_current_state(room_id, delta, stream_id)
+
+    async def _calculate_current_state(self, room_id: str) -> StateMap[str]:
+        """Calculate the current state of a room, based on the forward extremities
+
+        Args:
+            room_id: room for which to calculate current state
+
+        Returns:
+            map from (type, state_key) to event id for the  current state in the room
+        """
+        latest_event_ids = await self.main_store.get_latest_event_ids_in_room(room_id)
+        state_groups = set(
+            (
+                await self.main_store._get_state_group_for_events(latest_event_ids)
+            ).values()
+        )
+
+        state_maps_by_state_group = await self.state_store._get_state_for_groups(
+            state_groups
+        )
+
+        if len(state_groups) == 1:
+            # If there is only one state group, then we know what the current
+            # state is.
+            return state_maps_by_state_group[state_groups.pop()]
+
+        # Ok, we need to defer to the state handler to resolve our state sets.
+        logger.debug("calling resolve_state_groups from preserve_events")
+
+        # Avoid a circular import.
+        from synapse.state import StateResolutionStore
+
+        room_version = await self.main_store.get_room_version_id(room_id)
+        res = await self._state_resolution_handler.resolve_state_groups(
+            room_id,
+            room_version,
+            state_maps_by_state_group,
+            event_map=None,
+            state_res_store=StateResolutionStore(self.main_store),
+        )
+
+        return res.state
+
     async def _persist_event_batch(
         self,
         events_and_contexts: List[Tuple[EventBase, EventContext]],
diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py
index 151f2aa9b..871d4ace1 100644
--- a/synapse/storage/schema/__init__.py
+++ b/synapse/storage/schema/__init__.py
@@ -66,9 +66,9 @@ Changes in SCHEMA_VERSION = 69:
 
 
 SCHEMA_COMPAT_VERSION = (
-    # we now have `state_key` columns in both `events` and `state_events`, so
-    # now incompatible with synapses wth SCHEMA_VERSION < 66.
-    66
+    # We now assume that `device_lists_changes_in_room` has been filled out for
+    # recent device_list_updates.
+    69
 )
 """Limit on how far the synapse codebase can be rolled back without breaking db compat
 
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index cda194e8c..d1d585921 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -31,6 +31,7 @@ from frozendict import frozendict
 
 from synapse.api.constants import EventTypes
 from synapse.events import EventBase
+from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker
 from synapse.types import MutableStateMap, StateKey, StateMap
 
 if TYPE_CHECKING:
@@ -542,6 +543,10 @@ class StateGroupStorage:
 
     def __init__(self, hs: "HomeServer", stores: "Databases"):
         self.stores = stores
+        self._partial_state_events_tracker = PartialStateEventsTracker(stores.main)
+
+    def notify_event_un_partial_stated(self, event_id: str) -> None:
+        self._partial_state_events_tracker.notify_un_partial_stated(event_id)
 
     async def get_state_group_delta(
         self, state_group: int
@@ -579,7 +584,7 @@ class StateGroupStorage:
         if not event_ids:
             return {}
 
-        event_to_groups = await self.stores.main._get_state_group_for_events(event_ids)
+        event_to_groups = await self._get_state_group_for_events(event_ids)
 
         groups = set(event_to_groups.values())
         group_to_state = await self.stores.state._get_state_for_groups(groups)
@@ -668,7 +673,7 @@ class StateGroupStorage:
             RuntimeError if we don't have a state group for one or more of the events
                (ie they are outliers or unknown)
         """
-        event_to_groups = await self.stores.main._get_state_group_for_events(event_ids)
+        event_to_groups = await self._get_state_group_for_events(event_ids)
 
         groups = set(event_to_groups.values())
         group_to_state = await self.stores.state._get_state_for_groups(
@@ -709,7 +714,7 @@ class StateGroupStorage:
             RuntimeError if we don't have a state group for one or more of the events
                 (ie they are outliers or unknown)
         """
-        event_to_groups = await self.stores.main._get_state_group_for_events(event_ids)
+        event_to_groups = await self._get_state_group_for_events(event_ids)
 
         groups = set(event_to_groups.values())
         group_to_state = await self.stores.state._get_state_for_groups(
@@ -785,6 +790,23 @@ class StateGroupStorage:
             groups, state_filter or StateFilter.all()
         )
 
+    async def _get_state_group_for_events(
+        self,
+        event_ids: Collection[str],
+        await_full_state: bool = True,
+    ) -> Mapping[str, int]:
+        """Returns mapping event_id -> state_group
+
+        Args:
+            event_ids: events to get state groups for
+            await_full_state: if true, will block if we do not yet have complete
+               state at this event.
+        """
+        if await_full_state:
+            await self._partial_state_events_tracker.await_full_state(event_ids)
+
+        return await self.stores.main._get_state_group_for_events(event_ids)
+
     async def store_state_group(
         self,
         event_id: str,
diff --git a/synapse/storage/util/partial_state_events_tracker.py b/synapse/storage/util/partial_state_events_tracker.py
new file mode 100644
index 000000000..a61a951ef
--- /dev/null
+++ b/synapse/storage/util/partial_state_events_tracker.py
@@ -0,0 +1,120 @@
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from collections import defaultdict
+from typing import Collection, Dict, Set
+
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+
+from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
+from synapse.storage.databases.main.events_worker import EventsWorkerStore
+from synapse.util import unwrapFirstError
+
+logger = logging.getLogger(__name__)
+
+
+class PartialStateEventsTracker:
+    """Keeps track of which events have partial state, after a partial-state join"""
+
+    def __init__(self, store: EventsWorkerStore):
+        self._store = store
+        # a map from event id to a set of Deferreds which are waiting for that event to be
+        # un-partial-stated.
+        self._observers: Dict[str, Set[Deferred[None]]] = defaultdict(set)
+
+    def notify_un_partial_stated(self, event_id: str) -> None:
+        """Notify that we now have full state for a given event
+
+        Called by the state-resynchronization loop whenever we resynchronize the state
+        for a particular event. Unblocks any callers to await_full_state() for that
+        event.
+
+        Args:
+            event_id: the event that now has full state.
+        """
+        observers = self._observers.pop(event_id, None)
+        if not observers:
+            return
+        logger.info(
+            "Notifying %i things waiting for un-partial-stating of event %s",
+            len(observers),
+            event_id,
+        )
+        with PreserveLoggingContext():
+            for o in observers:
+                o.callback(None)
+
+    async def await_full_state(self, event_ids: Collection[str]) -> None:
+        """Wait for all the given events to have full state.
+
+        Args:
+            event_ids: the list of event ids that we want full state for
+        """
+        # first try the happy path: if there are no partial-state events, we can return
+        # quickly
+        partial_state_event_ids = [
+            ev
+            for ev, p in (await self._store.get_partial_state_events(event_ids)).items()
+            if p
+        ]
+
+        if not partial_state_event_ids:
+            return
+
+        logger.info(
+            "Awaiting un-partial-stating of events %s",
+            partial_state_event_ids,
+            stack_info=True,
+        )
+
+        # create an observer for each lazy-joined event
+        observers: Dict[str, Deferred[None]] = {
+            event_id: Deferred() for event_id in partial_state_event_ids
+        }
+        for event_id, observer in observers.items():
+            self._observers[event_id].add(observer)
+
+        try:
+            # some of them may have been un-lazy-joined between us checking the db and
+            # registering the observer, in which case we'd wait forever for the
+            # notification. Call back the observers now.
+            for event_id, partial in (
+                await self._store.get_partial_state_events(observers.keys())
+            ).items():
+                # there may have been a call to notify_un_partial_stated during the
+                # db query, so the observers may already have been called.
+                if not partial and not observers[event_id].called:
+                    observers[event_id].callback(None)
+
+            await make_deferred_yieldable(
+                defer.gatherResults(
+                    observers.values(),
+                    consumeErrors=True,
+                )
+            ).addErrback(unwrapFirstError)
+            logger.info("Events %s all un-partial-stated", observers.keys())
+        finally:
+            # remove any observers we created. This should happen when the notification
+            # is received, but that might not happen for two reasons:
+            #   (a) we're bailing out early on an exception (including us being
+            #       cancelled during the await)
+            #   (b) the event got de-lazy-joined before we set up the observer.
+            for event_id, observer in observers.items():
+                observer_set = self._observers.get(event_id)
+                if observer_set:
+                    observer_set.discard(observer)
+                    if not observer_set:
+                        del self._observers[event_id]
diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index 650e44de2..e27c5d298 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import abc
+import asyncio
 import collections
 import inspect
 import itertools
@@ -25,6 +26,7 @@ from typing import (
     Awaitable,
     Callable,
     Collection,
+    Coroutine,
     Dict,
     Generic,
     Hashable,
@@ -701,27 +703,57 @@ def stop_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]":
     return new_deferred
 
 
-def delay_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]":
-    """Delay cancellation of a `Deferred` until it resolves.
+@overload
+def delay_cancellation(awaitable: "defer.Deferred[T]") -> "defer.Deferred[T]":
+    ...
+
+
+@overload
+def delay_cancellation(awaitable: Coroutine[Any, Any, T]) -> "defer.Deferred[T]":
+    ...
+
+
+@overload
+def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]:
+    ...
+
+
+def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]:
+    """Delay cancellation of a coroutine or `Deferred` awaitable until it resolves.
 
     Has the same effect as `stop_cancellation`, but the returned `Deferred` will not
-    resolve with a `CancelledError` until the original `Deferred` resolves.
+    resolve with a `CancelledError` until the original awaitable resolves.
 
     Args:
-        deferred: The `Deferred` to protect against cancellation. May optionally follow
-            the Synapse logcontext rules.
+        deferred: The coroutine or `Deferred` to protect against cancellation. May
+            optionally follow the Synapse logcontext rules.
 
     Returns:
-        A new `Deferred`, which will contain the result of the original `Deferred`.
-        The new `Deferred` will not propagate cancellation through to the original.
-        When cancelled, the new `Deferred` will wait until the original `Deferred`
-        resolves before failing with a `CancelledError`.
+        A new `Deferred`, which will contain the result of the original coroutine or
+        `Deferred`. The new `Deferred` will not propagate cancellation through to the
+        original coroutine or `Deferred`.
 
-        The new `Deferred` will follow the Synapse logcontext rules if `deferred`
+        When cancelled, the new `Deferred` will wait until the original coroutine or
+        `Deferred` resolves before failing with a `CancelledError`.
+
+        The new `Deferred` will follow the Synapse logcontext rules if `awaitable`
         follows the Synapse logcontext rules. Otherwise the new `Deferred` should be
         wrapped with `make_deferred_yieldable`.
     """
 
+    # First, convert the awaitable into a `Deferred`.
+    if isinstance(awaitable, defer.Deferred):
+        deferred = awaitable
+    elif asyncio.iscoroutine(awaitable):
+        # Ideally we'd use `Deferred.fromCoroutine()` here, to save on redundant
+        # type-checking, but we'd need Twisted >= 21.2.
+        deferred = defer.ensureDeferred(awaitable)
+    else:
+        # We have no idea what to do with this awaitable.
+        # We assume it's already resolved, such as `DoneAwaitable`s or `Future`s from
+        # `make_awaitable`, and let the caller `await` it normally.
+        return awaitable
+
     def handle_cancel(new_deferred: "defer.Deferred[T]") -> None:
         # before the new deferred is cancelled, we `pause` it to stop the cancellation
         # propagating. we then `unpause` it once the wrapped deferred completes, to
diff --git a/synapse/visibility.py b/synapse/visibility.py
index 47bda9e38..598d2f8ba 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -427,6 +427,13 @@ async def _event_to_memberships(
         return {}
 
     # for each event, get the event_ids of the membership state at those events.
+    #
+    # TODO: this means that we request the entire membership list. If there  are only
+    #   one or two users on this server, and the room is huge, this is very wasteful
+    #   (it means more db work, and churns the *stateGroupMembersCache*).
+    #   It might be that we could extend StateFilter to specify "give me keys matching
+    #   *:<server_name>", to avoid this.
+
     event_to_state_ids = await storage.state.get_state_ids_for_events(
         frozenset(e.event_id for e in events),
         state_filter=StateFilter.from_types(types=((EventTypes.Member, None),)),
diff --git a/sytest-blacklist b/sytest-blacklist
index 57e603a4a..d5fa36cec 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -21,10 +21,6 @@ Newly created users see their own presence in /initialSync (SYT-34)
 # Blacklisted due to https://github.com/matrix-org/synapse/issues/1396
 Should reject keys claiming to belong to a different user
 
-# Blacklisted due to https://github.com/matrix-org/matrix-doc/pull/2314 removing
-# this requirement from the spec
-Inbound federation of state requires event_id as a mandatory paramater
-
 # Blacklisted until MSC2753 is implemented
 Local users can peek into world_readable rooms by room ID
 We can't peek into rooms with shared history_visibility
diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py
index 8c3354ce3..985d6e397 100644
--- a/tests/api/test_filtering.py
+++ b/tests/api/test_filtering.py
@@ -481,9 +481,7 @@ class FilteringTestCase(unittest.HomeserverTestCase):
         # events). This is a bit cheeky, but tests the logic of _check_event_relations.
 
         # Filter for a particular sender.
-        definition = {
-            "io.element.relation_senders": ["@foo:bar"],
-        }
+        definition = {"related_by_senders": ["@foo:bar"]}
 
         async def events_have_relations(*args, **kwargs):
             return ["$with_relation"]
diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py
index a6e91956a..91f982518 100644
--- a/tests/federation/test_federation_sender.py
+++ b/tests/federation/test_federation_sender.py
@@ -14,7 +14,6 @@
 from typing import Optional
 from unittest.mock import Mock
 
-from parameterized import parameterized_class
 from signedjson import key, sign
 from signedjson.types import BaseKey, SigningKey
 
@@ -155,12 +154,6 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase):
         )
 
 
-@parameterized_class(
-    [
-        {"enable_room_poke_code_path": False},
-        {"enable_room_poke_code_path": True},
-    ]
-)
 class FederationSenderDevicesTestCases(HomeserverTestCase):
     servlets = [
         admin.register_servlets,
@@ -169,13 +162,14 @@ class FederationSenderDevicesTestCases(HomeserverTestCase):
 
     def make_homeserver(self, reactor, clock):
         return self.setup_test_homeserver(
-            federation_transport_client=Mock(spec=["send_transaction"]),
+            federation_transport_client=Mock(
+                spec=["send_transaction", "query_user_devices"]
+            ),
         )
 
     def default_config(self):
         c = super().default_config()
         c["send_federation"] = True
-        c["use_new_device_lists_changes_in_room"] = self.enable_room_poke_code_path
         return c
 
     def prepare(self, reactor, clock, hs):
@@ -226,6 +220,45 @@ class FederationSenderDevicesTestCases(HomeserverTestCase):
         self.assertEqual(len(self.edus), 1)
         self.check_device_update_edu(self.edus.pop(0), u1, "D2", stream_id)
 
+    def test_dont_send_device_updates_for_remote_users(self):
+        """Check that we don't send device updates for remote users"""
+
+        # Send the server a device list EDU for the other user, this will cause
+        # it to try and resync the device lists.
+        self.hs.get_federation_transport_client().query_user_devices.return_value = (
+            defer.succeed(
+                {
+                    "stream_id": "1",
+                    "user_id": "@user2:host2",
+                    "devices": [{"device_id": "D1"}],
+                }
+            )
+        )
+
+        self.get_success(
+            self.hs.get_device_handler().device_list_updater.incoming_device_list_update(
+                "host2",
+                {
+                    "user_id": "@user2:host2",
+                    "device_id": "D1",
+                    "stream_id": "1",
+                    "prev_ids": [],
+                },
+            )
+        )
+
+        self.reactor.advance(1)
+
+        # We shouldn't see an EDU for that update
+        self.assertEqual(self.edus, [])
+
+        # Check that we did successfully process the inbound EDU (otherwise this
+        # test would pass if we failed to process the EDU)
+        devices = self.get_success(
+            self.hs.get_datastores().main.get_cached_devices_for_user("@user2:host2")
+        )
+        self.assertIn("D1", devices)
+
     def test_upload_signatures(self):
         """Uploading signatures on some devices should produce updates for that user"""
 
diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py
index 30e7e5093..b19365b81 100644
--- a/tests/federation/test_federation_server.py
+++ b/tests/federation/test_federation_server.py
@@ -104,58 +104,21 @@ class ServerACLsTestCase(unittest.TestCase):
 
 
 class StateQueryTests(unittest.FederatingHomeserverTestCase):
-
     servlets = [
         admin.register_servlets,
         room.register_servlets,
         login.register_servlets,
     ]
 
-    def test_without_event_id(self):
-        """
-        Querying v1/state/<room_id> without an event ID will return the current
-        known state.
-        """
-        u1 = self.register_user("u1", "pass")
-        u1_token = self.login("u1", "pass")
-
-        room_1 = self.helper.create_room_as(u1, tok=u1_token)
-        self.inject_room_member(room_1, "@user:other.example.com", "join")
-
-        channel = self.make_signed_federation_request(
-            "GET", "/_matrix/federation/v1/state/%s" % (room_1,)
-        )
-        self.assertEqual(200, channel.code, channel.result)
-
-        self.assertEqual(
-            channel.json_body["room_version"],
-            self.hs.config.server.default_room_version.identifier,
-        )
-
-        members = set(
-            map(
-                lambda x: x["state_key"],
-                filter(
-                    lambda x: x["type"] == "m.room.member", channel.json_body["pdus"]
-                ),
-            )
-        )
-
-        self.assertEqual(members, {"@user:other.example.com", u1})
-        self.assertEqual(len(channel.json_body["pdus"]), 6)
-
     def test_needs_to_be_in_room(self):
-        """
-        Querying v1/state/<room_id> requires the server
-        be in the room to provide data.
-        """
+        """/v1/state/<room_id> requires the server to be in the room"""
         u1 = self.register_user("u1", "pass")
         u1_token = self.login("u1", "pass")
 
         room_1 = self.helper.create_room_as(u1, tok=u1_token)
 
         channel = self.make_signed_federation_request(
-            "GET", "/_matrix/federation/v1/state/%s" % (room_1,)
+            "GET", "/_matrix/federation/v1/state/%s?event_id=xyz" % (room_1,)
         )
         self.assertEqual(403, channel.code, channel.result)
         self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN")
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index b2ed9cbe3..c96dc6caf 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -657,6 +657,85 @@ class PresenceHandlerTestCase(unittest.HomeserverTestCase):
         # Mark user as online and `status_msg = None`
         self._set_presencestate_with_status_msg(user_id, PresenceState.ONLINE, None)
 
+    def test_set_presence_from_syncing_not_set(self):
+        """Test that presence is not set by syncing if affect_presence is false"""
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.UNAVAILABLE, status_msg
+        )
+
+        self.get_success(
+            self.presence_handler.user_syncing(user_id, False, PresenceState.ONLINE)
+        )
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        # we should still be unavailable
+        self.assertEqual(state.state, PresenceState.UNAVAILABLE)
+        # and status message should still be the same
+        self.assertEqual(state.status_msg, status_msg)
+
+    def test_set_presence_from_syncing_is_set(self):
+        """Test that presence is set by syncing if affect_presence is true"""
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.UNAVAILABLE, status_msg
+        )
+
+        self.get_success(
+            self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE)
+        )
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        # we should now be online
+        self.assertEqual(state.state, PresenceState.ONLINE)
+
+    def test_set_presence_from_syncing_keeps_status(self):
+        """Test that presence set by syncing retains status message"""
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.UNAVAILABLE, status_msg
+        )
+
+        self.get_success(
+            self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE)
+        )
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        # our status message should be the same as it was before
+        self.assertEqual(state.status_msg, status_msg)
+
+    def test_set_presence_from_syncing_keeps_busy(self):
+        """Test that presence set by syncing doesn't affect busy status"""
+        # while this isn't the default
+        self.presence_handler._busy_presence_enabled = True
+
+        user_id = "@test:server"
+        status_msg = "I'm busy!"
+
+        self._set_presencestate_with_status_msg(user_id, PresenceState.BUSY, status_msg)
+
+        self.get_success(
+            self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE)
+        )
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        # we should still be busy
+        self.assertEqual(state.state, PresenceState.BUSY)
+
     def _set_presencestate_with_status_msg(
         self, user_id: str, state: str, status_msg: Optional[str]
     ):
diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py
index 849d00ab4..40571b753 100644
--- a/tests/rest/admin/test_admin.py
+++ b/tests/rest/admin/test_admin.py
@@ -63,6 +63,7 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase):
         self.other_user = self.register_user("user", "pass")
         self.other_user_token = self.login("user", "pass")
 
+    @unittest.override_config({"experimental_features": {"groups_enabled": True}})
     def test_delete_group(self) -> None:
         # Create a new group
         channel = self.make_request(
diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py
index 090d2d0a2..0a3d017dc 100644
--- a/tests/rest/client/test_login.py
+++ b/tests/rest/client/test_login.py
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import json
 import time
 import urllib.parse
 from typing import Any, Dict, List, Optional, Union
@@ -384,6 +384,31 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase):
         channel = self.make_request(b"POST", "/logout/all", access_token=access_token)
         self.assertEqual(channel.result["code"], b"200", channel.result)
 
+    def test_login_with_overly_long_device_id_fails(self) -> None:
+        self.register_user("mickey", "cheese")
+
+        # create a device_id longer than 512 characters
+        device_id = "yolo" * 512
+
+        body = {
+            "type": "m.login.password",
+            "user": "mickey",
+            "password": "cheese",
+            "device_id": device_id,
+        }
+
+        # make a login request with the bad device_id
+        channel = self.make_request(
+            "POST",
+            "/_matrix/client/v3/login",
+            json.dumps(body).encode("utf8"),
+            custom_headers=None,
+        )
+
+        # test that the login fails with the correct error code
+        self.assertEqual(channel.code, 400)
+        self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM")
+
 
 @skip_unless(has_saml2 and HAS_OIDC, "Requires SAML2 and OIDC")
 class MultiSSOTestCase(unittest.HomeserverTestCase):
diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py
index 6fabada8b..39667e322 100644
--- a/tests/rest/client/test_relations.py
+++ b/tests/rest/client/test_relations.py
@@ -355,7 +355,6 @@ class RelationsTestCase(BaseRelationsTestCase):
         self.assertEqual(200, channel.code, channel.json_body)
         self.assertNotIn("m.relations", channel.json_body["unsigned"])
 
-    @unittest.override_config({"experimental_features": {"msc3666_enabled": True}})
     def test_edit(self) -> None:
         """Test that a simple edit works."""
 
@@ -380,13 +379,16 @@ class RelationsTestCase(BaseRelationsTestCase):
                 {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict
             )
 
+        # /event should return the *original* event
         channel = self.make_request(
             "GET",
             f"/rooms/{self.room}/event/{self.parent_id}",
             access_token=self.user_token,
         )
         self.assertEqual(200, channel.code, channel.json_body)
-        self.assertEqual(channel.json_body["content"], new_body)
+        self.assertEqual(
+            channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"}
+        )
         assert_bundle(channel.json_body)
 
         # Request the room messages.
@@ -399,6 +401,7 @@ class RelationsTestCase(BaseRelationsTestCase):
         assert_bundle(self._find_event_in_chunk(channel.json_body["chunk"]))
 
         # Request the room context.
+        # /context should return the edited event.
         channel = self.make_request(
             "GET",
             f"/rooms/{self.room}/context/{self.parent_id}",
@@ -406,6 +409,7 @@ class RelationsTestCase(BaseRelationsTestCase):
         )
         self.assertEqual(200, channel.code, channel.json_body)
         assert_bundle(channel.json_body["event"])
+        self.assertEqual(channel.json_body["event"]["content"], new_body)
 
         # Request sync, but limit the timeline so it becomes limited (and includes
         # bundled aggregations).
@@ -470,14 +474,14 @@ class RelationsTestCase(BaseRelationsTestCase):
 
         channel = self.make_request(
             "GET",
-            f"/rooms/{self.room}/event/{self.parent_id}",
+            f"/rooms/{self.room}/context/{self.parent_id}",
             access_token=self.user_token,
         )
         self.assertEqual(200, channel.code, channel.json_body)
 
-        self.assertEqual(channel.json_body["content"], new_body)
+        self.assertEqual(channel.json_body["event"]["content"], new_body)
 
-        relations_dict = channel.json_body["unsigned"].get("m.relations")
+        relations_dict = channel.json_body["event"]["unsigned"].get("m.relations")
         self.assertIn(RelationTypes.REPLACE, relations_dict)
 
         m_replace_dict = relations_dict[RelationTypes.REPLACE]
@@ -492,10 +496,9 @@ class RelationsTestCase(BaseRelationsTestCase):
         """Test that editing a reply works."""
 
         # Create a reply to edit.
+        original_body = {"msgtype": "m.text", "body": "A reply!"}
         channel = self._send_relation(
-            RelationTypes.REFERENCE,
-            "m.room.message",
-            content={"msgtype": "m.text", "body": "A reply!"},
+            RelationTypes.REFERENCE, "m.room.message", content=original_body
         )
         reply = channel.json_body["event_id"]
 
@@ -508,75 +511,54 @@ class RelationsTestCase(BaseRelationsTestCase):
         )
         edit_event_id = channel.json_body["event_id"]
 
+        # /event returns the original event
         channel = self.make_request(
             "GET",
             f"/rooms/{self.room}/event/{reply}",
             access_token=self.user_token,
         )
         self.assertEqual(200, channel.code, channel.json_body)
+        event_result = channel.json_body
+        self.assertDictContainsSubset(original_body, event_result["content"])
 
-        # We expect to see the new body in the dict, as well as the reference
-        # metadata sill intact.
-        self.assertDictContainsSubset(new_body, channel.json_body["content"])
-        self.assertDictContainsSubset(
-            {
-                "m.relates_to": {
-                    "event_id": self.parent_id,
-                    "rel_type": "m.reference",
-                }
-            },
-            channel.json_body["content"],
-        )
-
-        # We expect that the edit relation appears in the unsigned relations
-        # section.
-        relations_dict = channel.json_body["unsigned"].get("m.relations")
-        self.assertIn(RelationTypes.REPLACE, relations_dict)
-
-        m_replace_dict = relations_dict[RelationTypes.REPLACE]
-        for key in ["event_id", "sender", "origin_server_ts"]:
-            self.assertIn(key, m_replace_dict)
-
-        self.assert_dict(
-            {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict
-        )
-
-    def test_edit_thread(self) -> None:
-        """Test that editing a thread works."""
-
-        # Create a thread and edit the last event.
-        channel = self._send_relation(
-            RelationTypes.THREAD,
-            "m.room.message",
-            content={"msgtype": "m.text", "body": "A threaded reply!"},
-        )
-        threaded_event_id = channel.json_body["event_id"]
-
-        new_body = {"msgtype": "m.text", "body": "I've been edited!"}
-        self._send_relation(
-            RelationTypes.REPLACE,
-            "m.room.message",
-            content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body},
-            parent_id=threaded_event_id,
-        )
-
-        # Fetch the thread root, to get the bundled aggregation for the thread.
+        # also check /context, which returns the *edited* event
         channel = self.make_request(
             "GET",
-            f"/rooms/{self.room}/event/{self.parent_id}",
+            f"/rooms/{self.room}/context/{reply}",
             access_token=self.user_token,
         )
         self.assertEqual(200, channel.code, channel.json_body)
+        context_result = channel.json_body["event"]
 
-        # We expect that the edit message appears in the thread summary in the
-        # unsigned relations section.
-        relations_dict = channel.json_body["unsigned"].get("m.relations")
-        self.assertIn(RelationTypes.THREAD, relations_dict)
+        # check that the relations are correct for both APIs
+        for result_event_dict, desc in (
+            (event_result, "/event"),
+            (context_result, "/context"),
+        ):
+            # The reference metadata should still be intact.
+            self.assertDictContainsSubset(
+                {
+                    "m.relates_to": {
+                        "event_id": self.parent_id,
+                        "rel_type": "m.reference",
+                    }
+                },
+                result_event_dict["content"],
+                desc,
+            )
 
-        thread_summary = relations_dict[RelationTypes.THREAD]
-        self.assertIn("latest_event", thread_summary)
-        latest_event_in_thread = thread_summary["latest_event"]
-        self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!")
+            # We expect that the edit relation appears in the unsigned relations
+            # section.
+            relations_dict = result_event_dict["unsigned"].get("m.relations")
+            self.assertIn(RelationTypes.REPLACE, relations_dict, desc)
+
+            m_replace_dict = relations_dict[RelationTypes.REPLACE]
+            for key in ["event_id", "sender", "origin_server_ts"]:
+                self.assertIn(key, m_replace_dict, desc)
+
+            self.assert_dict(
+                {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict
+            )
 
     def test_edit_edit(self) -> None:
         """Test that an edit cannot be edited."""
@@ -605,19 +587,31 @@ class RelationsTestCase(BaseRelationsTestCase):
         )
 
         # Request the original event.
+        # /event should return the original event.
         channel = self.make_request(
             "GET",
             f"/rooms/{self.room}/event/{self.parent_id}",
             access_token=self.user_token,
         )
         self.assertEqual(200, channel.code, channel.json_body)
-        # The edit to the edit should be ignored.
-        self.assertEqual(channel.json_body["content"], new_body)
+        self.assertEqual(
+            channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"}
+        )
 
         # The relations information should not include the edit to the edit.
         relations_dict = channel.json_body["unsigned"].get("m.relations")
         self.assertIn(RelationTypes.REPLACE, relations_dict)
 
+        # /context should return the event updated for the *first* edit
+        # (The edit to the edit should be ignored.)
+        channel = self.make_request(
+            "GET",
+            f"/rooms/{self.room}/context/{self.parent_id}",
+            access_token=self.user_token,
+        )
+        self.assertEqual(200, channel.code, channel.json_body)
+        self.assertEqual(channel.json_body["event"]["content"], new_body)
+
         m_replace_dict = relations_dict[RelationTypes.REPLACE]
         for key in ["event_id", "sender", "origin_server_ts"]:
             self.assertIn(key, m_replace_dict)
@@ -966,7 +960,6 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
         ]
         assert_bundle(self._find_event_in_chunk(chunk))
 
-    @unittest.override_config({"experimental_features": {"msc3666_enabled": True}})
     def test_annotation(self) -> None:
         """
         Test that annotations get correctly bundled.
@@ -991,7 +984,6 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
 
         self._test_bundled_aggregations(RelationTypes.ANNOTATION, assert_annotations, 7)
 
-    @unittest.override_config({"experimental_features": {"msc3666_enabled": True}})
     def test_reference(self) -> None:
         """
         Test that references get correctly bundled.
@@ -1010,7 +1002,6 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
 
         self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 7)
 
-    @unittest.override_config({"experimental_features": {"msc3666_enabled": True}})
     def test_thread(self) -> None:
         """
         Test that threads get correctly bundled.
@@ -1019,7 +1010,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
         channel = self._send_relation(RelationTypes.THREAD, "m.room.test")
         thread_2 = channel.json_body["event_id"]
 
-        def assert_annotations(bundled_aggregations: JsonDict) -> None:
+        def assert_thread(bundled_aggregations: JsonDict) -> None:
             self.assertEqual(2, bundled_aggregations.get("count"))
             self.assertTrue(bundled_aggregations.get("current_user_participated"))
             # The latest thread event has some fields that don't matter.
@@ -1038,7 +1029,38 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
                 bundled_aggregations.get("latest_event"),
             )
 
-        self._test_bundled_aggregations(RelationTypes.THREAD, assert_annotations, 9)
+        self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9)
+
+    def test_thread_edit_latest_event(self) -> None:
+        """Test that editing the latest event in a thread works."""
+
+        # Create a thread and edit the last event.
+        channel = self._send_relation(
+            RelationTypes.THREAD,
+            "m.room.message",
+            content={"msgtype": "m.text", "body": "A threaded reply!"},
+        )
+        threaded_event_id = channel.json_body["event_id"]
+
+        new_body = {"msgtype": "m.text", "body": "I've been edited!"}
+        channel = self._send_relation(
+            RelationTypes.REPLACE,
+            "m.room.message",
+            content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body},
+            parent_id=threaded_event_id,
+        )
+
+        # Fetch the thread root, to get the bundled aggregation for the thread.
+        relations_dict = self._get_bundled_aggregations()
+
+        # We expect that the edit message appears in the thread summary in the
+        # unsigned relations section.
+        self.assertIn(RelationTypes.THREAD, relations_dict)
+
+        thread_summary = relations_dict[RelationTypes.THREAD]
+        self.assertIn("latest_event", thread_summary)
+        latest_event_in_thread = thread_summary["latest_event"]
+        self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!")
 
     def test_aggregation_get_event_for_annotation(self) -> None:
         """Test that annotations do not get bundled aggregations included
@@ -1065,7 +1087,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
         channel = self._send_relation(RelationTypes.THREAD, "m.room.test")
         thread_id = channel.json_body["event_id"]
 
-        # Annotate the annotation.
+        # Annotate the thread.
         self._send_relation(
             RelationTypes.ANNOTATION, "m.reaction", "a", parent_id=thread_id
         )
diff --git a/tests/rest/client/test_room_batch.py b/tests/rest/client/test_room_batch.py
index 44f333a0e..41a1bf6d8 100644
--- a/tests/rest/client/test_room_batch.py
+++ b/tests/rest/client/test_room_batch.py
@@ -7,9 +7,9 @@ from twisted.test.proto_helpers import MemoryReactor
 from synapse.api.constants import EventContentFields, EventTypes
 from synapse.appservice import ApplicationService
 from synapse.rest import admin
-from synapse.rest.client import login, register, room, room_batch
+from synapse.rest.client import login, register, room, room_batch, sync
 from synapse.server import HomeServer
-from synapse.types import JsonDict
+from synapse.types import JsonDict, RoomStreamToken
 from synapse.util import Clock
 
 from tests import unittest
@@ -63,6 +63,7 @@ class RoomBatchTestCase(unittest.HomeserverTestCase):
         room.register_servlets,
         register.register_servlets,
         login.register_servlets,
+        sync.register_servlets,
     ]
 
     def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
@@ -178,3 +179,123 @@ class RoomBatchTestCase(unittest.HomeserverTestCase):
             "Expected a single state_group to be returned by saw state_groups=%s"
             % (state_group_map.keys(),),
         )
+
+    @unittest.override_config({"experimental_features": {"msc2716_enabled": True}})
+    def test_sync_while_batch_importing(self) -> None:
+        """
+        Make sure that /sync correctly returns full room state when a user joins
+        during ongoing batch backfilling.
+        See: https://github.com/matrix-org/synapse/issues/12281
+        """
+        # Create user who will be invited & join room
+        user_id = self.register_user("beep", "test")
+        user_tok = self.login("beep", "test")
+
+        time_before_room = int(self.clock.time_msec())
+
+        # Create a room with some events
+        room_id, _, _, _ = self._create_test_room()
+        # Invite the user
+        self.helper.invite(
+            room_id, src=self.appservice.sender, tok=self.appservice.token, targ=user_id
+        )
+
+        # Create another room, send a bunch of events to advance the stream token
+        other_room_id = self.helper.create_room_as(
+            self.appservice.sender, tok=self.appservice.token
+        )
+        for _ in range(5):
+            self.helper.send_event(
+                room_id=other_room_id,
+                type=EventTypes.Message,
+                content={"msgtype": "m.text", "body": "C"},
+                tok=self.appservice.token,
+            )
+
+        # Join the room as the normal user
+        self.helper.join(room_id, user_id, tok=user_tok)
+
+        # Create an event to hang the historical batch from - In order to see
+        # the failure case originally reported in #12281, the historical batch
+        # must be hung from the most recent event in the room so the base
+        # insertion event ends up with the highest `topogological_ordering`
+        # (`depth`) in the room but will have a negative `stream_ordering`
+        # because it's a `historical` event. Previously, when assembling the
+        # `state` for the `/sync` response, the bugged logic would sort by
+        # `topological_ordering` descending and pick up the base insertion
+        # event because it has a negative `stream_ordering` below the given
+        # pagination token. Now we properly sort by `stream_ordering`
+        # descending which puts `historical` events with a negative
+        # `stream_ordering` way at the bottom and aren't selected as expected.
+        response = self.helper.send_event(
+            room_id=room_id,
+            type=EventTypes.Message,
+            content={
+                "msgtype": "m.text",
+                "body": "C",
+            },
+            tok=self.appservice.token,
+        )
+        event_to_hang_id = response["event_id"]
+
+        channel = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2716/rooms/%s/batch_send?prev_event_id=%s"
+            % (room_id, event_to_hang_id),
+            content={
+                "events": _create_message_events_for_batch_send_request(
+                    self.virtual_user_id, time_before_room, 3
+                ),
+                "state_events_at_start": _create_join_state_events_for_batch_send_request(
+                    [self.virtual_user_id], time_before_room
+                ),
+            },
+            access_token=self.appservice.token,
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+
+        # Now we need to find the invite + join events stream tokens so we can sync between
+        main_store = self.hs.get_datastores().main
+        events, next_key = self.get_success(
+            main_store.get_recent_events_for_room(
+                room_id,
+                50,
+                end_token=main_store.get_room_max_token(),
+            ),
+        )
+        invite_event_position = None
+        for event in events:
+            if (
+                event.type == "m.room.member"
+                and event.content["membership"] == "invite"
+            ):
+                invite_event_position = self.get_success(
+                    main_store.get_topological_token_for_event(event.event_id)
+                )
+                break
+
+        assert invite_event_position is not None, "No invite event found"
+
+        # Remove the topological order from the token by re-creating w/stream only
+        invite_event_position = RoomStreamToken(None, invite_event_position.stream)
+
+        # Sync everything after this token
+        since_token = self.get_success(invite_event_position.to_string(main_store))
+        sync_response = self.make_request(
+            "GET",
+            f"/sync?since={since_token}",
+            access_token=user_tok,
+        )
+
+        # Assert that, for this room, the user was considered to have joined and thus
+        # receives the full state history
+        state_event_types = [
+            event["type"]
+            for event in sync_response.json_body["rooms"]["join"][room_id]["state"][
+                "events"
+            ]
+        ]
+
+        assert (
+            "m.room.create" in state_event_types
+        ), "Missing room full state in sync response"
diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py
index 1f6a9eb07..bf6374f93 100644
--- a/tests/storage/databases/main/test_events_worker.py
+++ b/tests/storage/databases/main/test_events_worker.py
@@ -13,10 +13,11 @@
 # limitations under the License.
 import json
 from contextlib import contextmanager
-from typing import Generator
+from typing import Generator, Tuple
+from unittest import mock
 
 from twisted.enterprise.adbapi import ConnectionPool
-from twisted.internet.defer import ensureDeferred
+from twisted.internet.defer import CancelledError, Deferred, ensureDeferred
 from twisted.test.proto_helpers import MemoryReactor
 
 from synapse.api.room_versions import EventFormatVersions, RoomVersions
@@ -281,3 +282,119 @@ class DatabaseOutageTestCase(unittest.HomeserverTestCase):
 
         # This next event fetch should succeed
         self.get_success(self.store.get_event(self.event_ids[0]))
+
+
+class GetEventCancellationTestCase(unittest.HomeserverTestCase):
+    """Test cancellation of `get_event` calls."""
+
+    servlets = [
+        admin.register_servlets,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer):
+        self.store: EventsWorkerStore = hs.get_datastores().main
+
+        self.user = self.register_user("user", "pass")
+        self.token = self.login(self.user, "pass")
+
+        self.room = self.helper.create_room_as(self.user, tok=self.token)
+
+        res = self.helper.send(self.room, tok=self.token)
+        self.event_id = res["event_id"]
+
+        # Reset the event cache so the tests start with it empty
+        self.store._get_event_cache.clear()
+
+    @contextmanager
+    def blocking_get_event_calls(
+        self,
+    ) -> Generator[
+        Tuple["Deferred[None]", "Deferred[None]", "Deferred[None]"], None, None
+    ]:
+        """Starts two concurrent `get_event` calls for the same event.
+
+        Both `get_event` calls will use the same database fetch, which will be blocked
+        at the time this function returns.
+
+        Returns:
+            A tuple containing:
+             * A `Deferred` that unblocks the database fetch.
+             * A cancellable `Deferred` for the first `get_event` call.
+             * A cancellable `Deferred` for the second `get_event` call.
+        """
+        # Patch `DatabasePool.runWithConnection` to block.
+        unblock: "Deferred[None]" = Deferred()
+        original_runWithConnection = self.store.db_pool.runWithConnection
+
+        async def runWithConnection(*args, **kwargs):
+            await unblock
+            return await original_runWithConnection(*args, **kwargs)
+
+        with mock.patch.object(
+            self.store.db_pool,
+            "runWithConnection",
+            new=runWithConnection,
+        ):
+            ctx1 = LoggingContext("get_event1")
+            ctx2 = LoggingContext("get_event2")
+
+            async def get_event(ctx: LoggingContext) -> None:
+                with ctx:
+                    await self.store.get_event(self.event_id)
+
+            get_event1 = ensureDeferred(get_event(ctx1))
+            get_event2 = ensureDeferred(get_event(ctx2))
+
+            # Both `get_event` calls ought to be blocked.
+            self.assertNoResult(get_event1)
+            self.assertNoResult(get_event2)
+
+            yield unblock, get_event1, get_event2
+
+        # Confirm that the two `get_event` calls shared the same database fetch.
+        self.assertEqual(ctx1.get_resource_usage().evt_db_fetch_count, 1)
+        self.assertEqual(ctx2.get_resource_usage().evt_db_fetch_count, 0)
+
+    def test_first_get_event_cancelled(self):
+        """Test cancellation of the first `get_event` call sharing a database fetch.
+
+        The first `get_event` call is the one which initiates the fetch. We expect the
+        fetch to complete despite the cancellation. Furthermore, the first `get_event`
+        call must not abort before the fetch is complete, otherwise the fetch will be
+        using a finished logging context.
+        """
+        with self.blocking_get_event_calls() as (unblock, get_event1, get_event2):
+            # Cancel the first `get_event` call.
+            get_event1.cancel()
+            # The first `get_event` call must not abort immediately, otherwise its
+            # logging context will be finished while it is still in use by the database
+            # fetch.
+            self.assertNoResult(get_event1)
+            # The second `get_event` call must not be cancelled.
+            self.assertNoResult(get_event2)
+
+            # Unblock the database fetch.
+            unblock.callback(None)
+            # A `CancelledError` should be raised out of the first `get_event` call.
+            exc = self.get_failure(get_event1, CancelledError).value
+            self.assertIsInstance(exc, CancelledError)
+            # The second `get_event` call should complete successfully.
+            self.get_success(get_event2)
+
+    def test_second_get_event_cancelled(self):
+        """Test cancellation of the second `get_event` call sharing a database fetch."""
+        with self.blocking_get_event_calls() as (unblock, get_event1, get_event2):
+            # Cancel the second `get_event` call.
+            get_event2.cancel()
+            # The first `get_event` call must not be cancelled.
+            self.assertNoResult(get_event1)
+            # The second `get_event` call gets cancelled immediately.
+            exc = self.get_failure(get_event2, CancelledError).value
+            self.assertIsInstance(exc, CancelledError)
+
+            # Unblock the database fetch.
+            unblock.callback(None)
+            # The first `get_event` call should complete successfully.
+            self.get_success(get_event1)
diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py
index d1227dd4a..ccc389386 100644
--- a/tests/storage/test_devices.py
+++ b/tests/storage/test_devices.py
@@ -21,6 +21,29 @@ class DeviceStoreTestCase(HomeserverTestCase):
     def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastores().main
 
+    def add_device_change(self, user_id, device_ids, host):
+        """Add a device list change for the given device to
+        `device_lists_outbound_pokes` table.
+        """
+
+        for device_id in device_ids:
+            stream_id = self.get_success(
+                self.store.add_device_change_to_streams(
+                    "user_id", [device_id], ["!some:room"]
+                )
+            )
+
+            self.get_success(
+                self.store.add_device_list_outbound_pokes(
+                    user_id=user_id,
+                    device_id=device_id,
+                    room_id="!some:room",
+                    stream_id=stream_id,
+                    hosts=[host],
+                    context={},
+                )
+            )
+
     def test_store_new_device(self):
         self.get_success(
             self.store.store_device("user_id", "device_id", "display_name")
@@ -95,11 +118,7 @@ class DeviceStoreTestCase(HomeserverTestCase):
         device_ids = ["device_id1", "device_id2"]
 
         # Add two device updates with sequential `stream_id`s
-        self.get_success(
-            self.store.add_device_change_to_streams(
-                "user_id", device_ids, ["somehost"], ["!some:room"]
-            )
-        )
+        self.add_device_change("@user_id:test", device_ids, "somehost")
 
         # Get all device updates ever meant for this remote
         now_stream_id, device_updates = self.get_success(
@@ -123,11 +142,7 @@ class DeviceStoreTestCase(HomeserverTestCase):
             "device_id4",
             "device_id5",
         ]
-        self.get_success(
-            self.store.add_device_change_to_streams(
-                "user_id", device_ids, ["somehost"], ["!some:room"]
-            )
-        )
+        self.add_device_change("@user_id:test", device_ids, "somehost")
 
         # Get device updates meant for this remote
         next_stream_id, device_updates = self.get_success(
@@ -147,11 +162,7 @@ class DeviceStoreTestCase(HomeserverTestCase):
 
         # Add some more device updates to ensure it still resumes properly
         device_ids = ["device_id6", "device_id7"]
-        self.get_success(
-            self.store.add_device_change_to_streams(
-                "user_id", device_ids, ["somehost"], ["!some:room"]
-            )
-        )
+        self.add_device_change("@user_id:test", device_ids, "somehost")
 
         # Get the next batch of device updates
         next_stream_id, device_updates = self.get_success(
@@ -224,11 +235,7 @@ class DeviceStoreTestCase(HomeserverTestCase):
             "fakeSelfSigning",
         ]
 
-        self.get_success(
-            self.store.add_device_change_to_streams(
-                "@user_id:test", device_ids, ["somehost"], ["!some:room"]
-            )
-        )
+        self.add_device_change("@user_id:test", device_ids, "somehost")
 
         # Get device updates meant for this remote
         next_stream_id, device_updates = self.get_success(
diff --git a/tests/storage/util/__init__.py b/tests/storage/util/__init__.py
new file mode 100644
index 000000000..3a5f22c02
--- /dev/null
+++ b/tests/storage/util/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/storage/util/test_partial_state_events_tracker.py b/tests/storage/util/test_partial_state_events_tracker.py
new file mode 100644
index 000000000..303e190b6
--- /dev/null
+++ b/tests/storage/util/test_partial_state_events_tracker.py
@@ -0,0 +1,117 @@
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict
+from unittest import mock
+
+from twisted.internet.defer import CancelledError, ensureDeferred
+
+from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker
+
+from tests.unittest import TestCase
+
+
+class PartialStateEventsTrackerTestCase(TestCase):
+    def setUp(self) -> None:
+        # the results to be returned by the mocked get_partial_state_events
+        self._events_dict: Dict[str, bool] = {}
+
+        async def get_partial_state_events(events):
+            return {e: self._events_dict[e] for e in events}
+
+        self.mock_store = mock.Mock(spec_set=["get_partial_state_events"])
+        self.mock_store.get_partial_state_events.side_effect = get_partial_state_events
+
+        self.tracker = PartialStateEventsTracker(self.mock_store)
+
+    def test_does_not_block_for_full_state_events(self):
+        self._events_dict = {"event1": False, "event2": False}
+
+        self.successResultOf(
+            ensureDeferred(self.tracker.await_full_state(["event1", "event2"]))
+        )
+
+        self.mock_store.get_partial_state_events.assert_called_once_with(
+            ["event1", "event2"]
+        )
+
+    def test_blocks_for_partial_state_events(self):
+        self._events_dict = {"event1": True, "event2": False}
+
+        d = ensureDeferred(self.tracker.await_full_state(["event1", "event2"]))
+
+        # there should be no result yet
+        self.assertNoResult(d)
+
+        # notifying that the event has been de-partial-stated should unblock
+        self.tracker.notify_un_partial_stated("event1")
+        self.successResultOf(d)
+
+    def test_un_partial_state_race(self):
+        # if the event is un-partial-stated between the initial check and the
+        # registration of the listener, it should not block.
+        self._events_dict = {"event1": True, "event2": False}
+
+        async def get_partial_state_events(events):
+            res = {e: self._events_dict[e] for e in events}
+            # change the result for next time
+            self._events_dict = {"event1": False, "event2": False}
+            return res
+
+        self.mock_store.get_partial_state_events.side_effect = get_partial_state_events
+
+        self.successResultOf(
+            ensureDeferred(self.tracker.await_full_state(["event1", "event2"]))
+        )
+
+    def test_un_partial_state_during_get_partial_state_events(self):
+        # we should correctly handle a call to notify_un_partial_stated during the
+        # second call to get_partial_state_events.
+
+        self._events_dict = {"event1": True, "event2": False}
+
+        async def get_partial_state_events1(events):
+            self.mock_store.get_partial_state_events.side_effect = (
+                get_partial_state_events2
+            )
+            return {e: self._events_dict[e] for e in events}
+
+        async def get_partial_state_events2(events):
+            self.tracker.notify_un_partial_stated("event1")
+            self._events_dict["event1"] = False
+            return {e: self._events_dict[e] for e in events}
+
+        self.mock_store.get_partial_state_events.side_effect = get_partial_state_events1
+
+        self.successResultOf(
+            ensureDeferred(self.tracker.await_full_state(["event1", "event2"]))
+        )
+
+    def test_cancellation(self):
+        self._events_dict = {"event1": True, "event2": False}
+
+        d1 = ensureDeferred(self.tracker.await_full_state(["event1", "event2"]))
+        self.assertNoResult(d1)
+
+        d2 = ensureDeferred(self.tracker.await_full_state(["event1"]))
+        self.assertNoResult(d2)
+
+        d1.cancel()
+        self.assertFailure(d1, CancelledError)
+
+        # d2 should still be waiting!
+        self.assertNoResult(d2)
+
+        self.tracker.notify_un_partial_stated("event1")
+        self.successResultOf(d2)
diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py
index e5bc416de..daacc54c7 100644
--- a/tests/util/test_async_helpers.py
+++ b/tests/util/test_async_helpers.py
@@ -382,7 +382,7 @@ class StopCancellationTests(TestCase):
 class DelayCancellationTests(TestCase):
     """Tests for the `delay_cancellation` function."""
 
-    def test_cancellation(self):
+    def test_deferred_cancellation(self):
         """Test that cancellation of the new `Deferred` waits for the original."""
         deferred: "Deferred[str]" = Deferred()
         wrapper_deferred = delay_cancellation(deferred)
@@ -403,6 +403,35 @@ class DelayCancellationTests(TestCase):
         # Now that the original `Deferred` has failed, we should get a `CancelledError`.
         self.failureResultOf(wrapper_deferred, CancelledError)
 
+    def test_coroutine_cancellation(self):
+        """Test that cancellation of the new `Deferred` waits for the original."""
+        blocking_deferred: "Deferred[None]" = Deferred()
+        completion_deferred: "Deferred[None]" = Deferred()
+
+        async def task():
+            await blocking_deferred
+            completion_deferred.callback(None)
+            # Raise an exception. Twisted should consume it, otherwise unwanted
+            # tracebacks will be printed in logs.
+            raise ValueError("abc")
+
+        wrapper_deferred = delay_cancellation(task())
+
+        # Cancel the new `Deferred`.
+        wrapper_deferred.cancel()
+        self.assertNoResult(wrapper_deferred)
+        self.assertFalse(
+            blocking_deferred.called, "Cancellation was propagated too deep"
+        )
+        self.assertFalse(completion_deferred.called)
+
+        # Unblock the task.
+        blocking_deferred.callback(None)
+        self.assertTrue(completion_deferred.called)
+
+        # Now that the original coroutine has failed, we should get a `CancelledError`.
+        self.failureResultOf(wrapper_deferred, CancelledError)
+
     def test_suppresses_second_cancellation(self):
         """Test that a second cancellation is suppressed.
 
@@ -451,7 +480,7 @@ class DelayCancellationTests(TestCase):
         async def outer():
             with LoggingContext("c") as c:
                 try:
-                    await delay_cancellation(defer.ensureDeferred(inner()))
+                    await delay_cancellation(inner())
                     self.fail("`CancelledError` was not raised")
                 except CancelledError:
                     self.assertEqual(c, current_context())
diff --git a/tox.ini b/tox.ini
index b4ce400ed..4cd9dfb96 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py37, py38, py39, py310, check_codestyle, check_isort
+envlist = py37, py38, py39, py310
 
 # we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208
 minversion = 2.3.2
@@ -32,20 +32,6 @@ deps =
     # install the "enum34" dependency of cryptography.
     pip>=10
 
-# directories/files we run the linters on.
-# TODO: this is now out of date; we will remove as part of poetry migration.
-lint_targets =
-    setup.py
-    synapse
-    tests
-    # annoyingly, black doesn't find these so we have to list them
-    scripts-dev
-    stubs
-    contrib
-    synmark
-    .ci
-    docker
-
 # default settings for all tox environments
 [testenv]
 deps =
@@ -116,18 +102,3 @@ setenv =
 commands =
     python -m synmark {posargs:}
 
-[testenv:check_codestyle]
-extras = lint
-commands =
-    python -m black --check --diff {[base]lint_targets}
-    flake8 {[base]lint_targets} {env:PEP8SUFFIX:}
-
-[testenv:check_isort]
-extras = lint
-commands = isort -c --df {[base]lint_targets}
-
-[testenv:mypy]
-deps =
-    {[base]deps}
-extras = all,mypy
-commands = mypy