maxmustermann/synapse
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'develop' into eric/jump-to-date

Browse source
This commit is contained in:
Eric Eastwood 2021-07-26 16:32:12 -05:00
parent 8e5fa114ed b7186c6e8d
commit 96c48ba96c
297 changed files with 4416 additions and 2203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
.circleci/config.yml
View file

@ -1,78 +0,0 @@
version: 2.1
jobs:
dockerhubuploadrelease:
docker:
- image: docker:git
steps:
- checkout
- docker_prepare
- run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
# for release builds, we want to get the amd64 image out asap, so first
# we do an amd64-only build, before following up with a multiarch build.
- docker_build:
tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
platforms: linux/amd64
- docker_build:
tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
platforms: linux/amd64,linux/arm64
dockerhubuploadlatest:
docker:
- image: docker:git
steps:
- checkout
- docker_prepare
- run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
# for `latest`, we don't want the arm images to disappear, so don't update the tag
# until all of the platforms are built.
- docker_build:
tag: -t matrixdotorg/synapse:latest
platforms: linux/amd64,linux/arm64
workflows:
build:
jobs:
- dockerhubuploadrelease:
filters:
tags:
only: /v[0-9].[0-9]+.[0-9]+.*/
branches:
ignore: /.*/
- dockerhubuploadlatest:
filters:
branches:
only: [ master, main ]
commands:
docker_prepare:
description: Sets up a remote docker server, downloads the buildx cli plugin, and enables multiarch images
parameters:
buildx_version:
type: string
default: "v0.4.1"
steps:
- setup_remote_docker:
# 19.03.13 was the most recent available on circleci at the time of
# writing.
version: 19.03.13
- run: apk add --no-cache curl
- run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
- run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
- run: chmod a+x ~/.docker/cli-plugins/docker-buildx
# install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
- run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
# create a context named `builder` for the builds
- run: docker context create builder
# create a buildx builder using the new context, and set it as the default
- run: docker buildx create builder --use
docker_build:
description: Builds and pushed images to dockerhub using buildx
parameters:
platforms:
type: string
default: linux/amd64
tag:
type: string
steps:
- run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .

72
.github/workflows/docker.yml vendored Normal file
View file

@ -0,0 +1,72 @@
# GitHub actions workflow which builds and publishes the docker images.
name: Build docker images
on:
push:
tags: ["v*"]
branches: [ master, main ]
workflow_dispatch:
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v1
with:
platforms: arm64
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
- name: Inspect builder
run: docker buildx inspect
- name: Log in to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Calculate docker image tag
id: set-tag
run: |
case "${GITHUB_REF}" in
refs/heads/master|refs/heads/main)
tag=latest
;;
refs/tags/*)
tag=${GITHUB_REF#refs/tags/}
;;
*)
tag=${GITHUB_SHA}
;;
esac
echo "::set-output name=tag::$tag"
# for release builds, we want to get the amd64 image out asap, so first
# we do an amd64-only build, before following up with a multiarch build.
- name: Build and push amd64
uses: docker/build-push-action@v2
if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
with:
push: true
labels: "gitsha1=${{ github.sha }}"
tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
file: "docker/Dockerfile"
platforms: linux/amd64
- name: Build and push all platforms
uses: docker/build-push-action@v2
with:
push: true
labels: "gitsha1=${{ github.sha }}"
tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
file: "docker/Dockerfile"
platforms: linux/amd64,linux/arm64

43
.github/workflows/release-artifacts.yml vendored
View file

@ -12,6 +12,10 @@ on:
# we do the full build on tags.
tags: ["v*"]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: write
@ -44,12 +48,43 @@ jobs:
distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
steps:
- uses: actions/checkout@v2
- name: Checkout
uses: actions/checkout@v2
with:
path: src
- uses: actions/setup-python@v2
- run: ./src/scripts-dev/build_debian_packages "${{ matrix.distro }}"
- uses: actions/upload-artifact@v2
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
with:
install: true
- name: Set up docker layer caching
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Set up python
uses: actions/setup-python@v2
- name: Build the packages
# see https://github.com/docker/build-push-action/issues/252
# for the cache magic here
run: |
./src/scripts-dev/build_debian_packages \
--docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
--docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
--docker-build-arg=--progress=plain \
--docker-build-arg=--load \
"${{ matrix.distro }}"
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Upload debs as artifacts
uses: actions/upload-artifact@v2
with:
name: debs
path: debs/*

33
.github/workflows/tests.yml vendored
View file

@ -5,6 +5,10 @@ on:
branches: ["develop", "release-*"]
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
runs-on: ubuntu-latest
@ -344,3 +348,32 @@ jobs:
env:
COMPLEMENT_BASE_IMAGE: complement-synapse:latest
working-directory: complement
# a job which marks all the other jobs as complete, thus allowing PRs to be merged.
tests-done:
if: ${{ always() }}
needs:
- lint
- lint-crlf
- lint-newsfile
- lint-sdist
- trial
- trial-olddeps
- sytest
- portdb
- complement
runs-on: ubuntu-latest
steps:
- name: Set build result
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
# the `jq` incantation dumps out a series of "<job> <result>" lines
run: |
set -o pipefail
jq -r 'to_entries[] | [.key,.value.result] | join(" ")' \
<<< $NEEDS_CONTEXT |
while read job result; do
if [ "$result" != "success" ]; then
echo "::set-failed ::Job $job returned $result"
fi
done

93
CHANGES.md
View file

@ -1,3 +1,96 @@
Synapse 1.39.0rc2 (2021-07-22)
==============================
Bugfixes
--------
- Always include `device_one_time_keys_count` key in `/sync` response to work around a bug in Element Android that broke encryption for new devices. ([\#10457](https://github.com/matrix-org/synapse/issues/10457))
Internal Changes
----------------
- Move docker image build to Github Actions. ([\#10416](https://github.com/matrix-org/synapse/issues/10416))
Synapse 1.38.1 (2021-07-22)
===========================
Bugfixes
--------
- Always include `device_one_time_keys_count` key in `/sync` response to work around a bug in Element Android that broke encryption for new devices. ([\#10457](https://github.com/matrix-org/synapse/issues/10457))
Synapse 1.39.0rc1 (2021-07-20)
==============================
The Third-Party Event Rules module interface has been deprecated in favour of the generic module interface introduced in Synapse v1.37.0. Support for the old interface is planned to be removed in September 2021. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information.
Features
--------
- Add the ability to override the account validity feature with a module. ([\#9884](https://github.com/matrix-org/synapse/issues/9884))
- The spaces summary API now returns any joinable rooms, not only rooms which are world-readable. ([\#10298](https://github.com/matrix-org/synapse/issues/10298), [\#10305](https://github.com/matrix-org/synapse/issues/10305))
- Add a new version of the R30 phone-home metric, which removes a false impression of retention given by the old R30 metric. ([\#10332](https://github.com/matrix-org/synapse/issues/10332), [\#10427](https://github.com/matrix-org/synapse/issues/10427))
- Allow providing credentials to `http_proxy`. ([\#10360](https://github.com/matrix-org/synapse/issues/10360))
Bugfixes
--------
- Fix error while dropping locks on shutdown. Introduced in v1.38.0. ([\#10433](https://github.com/matrix-org/synapse/issues/10433))
- Add base starting insertion event when no chunk ID is specified in the historical batch send API. ([\#10250](https://github.com/matrix-org/synapse/issues/10250))
- Fix historical batch send endpoint (MSC2716) rejecting batches with messages from multiple senders. ([\#10276](https://github.com/matrix-org/synapse/issues/10276))
- Fix purging rooms that other homeservers are still sending events for. Contributed by @ilmari. ([\#10317](https://github.com/matrix-org/synapse/issues/10317))
- Fix errors during backfill caused by previously purged redaction events. Contributed by Andreas Rammhold (@andir). ([\#10343](https://github.com/matrix-org/synapse/issues/10343))
- Fix the user directory becoming broken (and noisy errors being logged) when knocking and room statistics are in use. ([\#10344](https://github.com/matrix-org/synapse/issues/10344))
- Fix newly added `synapse_federation_server_oldest_inbound_pdu_in_staging` prometheus metric to measure age rather than timestamp. ([\#10355](https://github.com/matrix-org/synapse/issues/10355))
- Fix PostgreSQL sometimes using table scans for queries against `state_groups_state` table, taking a long time and a large amount of IO. ([\#10359](https://github.com/matrix-org/synapse/issues/10359))
- Fix `make_room_admin` failing for users that have left a private room. ([\#10367](https://github.com/matrix-org/synapse/issues/10367))
- Fix a number of logged errors caused by remote servers being down. ([\#10400](https://github.com/matrix-org/synapse/issues/10400), [\#10414](https://github.com/matrix-org/synapse/issues/10414))
- Responses from `/make_{join,leave,knock}` no longer include signatures, which will turn out to be invalid after events are returned to `/send_{join,leave,knock}`. ([\#10404](https://github.com/matrix-org/synapse/issues/10404))
Improved Documentation
----------------------
- Updated installation dependencies for newer macOS versions and ARM Macs. Contributed by Luke Walsh. ([\#9971](https://github.com/matrix-org/synapse/issues/9971))
- Simplify structure of room admin API. ([\#10313](https://github.com/matrix-org/synapse/issues/10313))
- Refresh the logcontext dev documentation. ([\#10353](https://github.com/matrix-org/synapse/issues/10353)), ([\#10337](https://github.com/matrix-org/synapse/issues/10337))
- Add delegation example for caddy in the reverse proxy documentation. Contributed by @moritzdietz. ([\#10368](https://github.com/matrix-org/synapse/issues/10368))
- Fix and clarify some links in `docs` and `contrib`. ([\#10370](https://github.com/matrix-org/synapse/issues/10370)), ([\#10322](https://github.com/matrix-org/synapse/issues/10322)), ([\#10399](https://github.com/matrix-org/synapse/issues/10399))
- Make deprecation notice of the spam checker doc more obvious. ([\#10395](https://github.com/matrix-org/synapse/issues/10395))
- Add instructions on installing Debian packages for release candidates. ([\#10396](https://github.com/matrix-org/synapse/issues/10396))
Deprecations and Removals
-------------------------
- Remove functionality associated with the unused `room_stats_historical` and `user_stats_historical` tables. Contributed by @xmunoz. ([\#9721](https://github.com/matrix-org/synapse/issues/9721))
- The third-party event rules module interface is deprecated in favour of the generic module interface introduced in Synapse v1.37.0. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information. ([\#10386](https://github.com/matrix-org/synapse/issues/10386))
Internal Changes
----------------
- Convert `room_depth.min_depth` column to a `BIGINT`. ([\#10289](https://github.com/matrix-org/synapse/issues/10289))
- Add tests to characterise the current behaviour of R30 phone-home metrics. ([\#10315](https://github.com/matrix-org/synapse/issues/10315))
- Rebuild event context and auth when processing specific results from `ThirdPartyEventRules` modules. ([\#10316](https://github.com/matrix-org/synapse/issues/10316))
- Minor change to the code that populates `user_daily_visits`. ([\#10324](https://github.com/matrix-org/synapse/issues/10324))
- Re-enable Sytests that were disabled for the 1.37.1 release. ([\#10345](https://github.com/matrix-org/synapse/issues/10345), [\#10357](https://github.com/matrix-org/synapse/issues/10357))
- Run `pyupgrade` on the codebase. ([\#10347](https://github.com/matrix-org/synapse/issues/10347), [\#10348](https://github.com/matrix-org/synapse/issues/10348))
- Switch `application_services_txns.txn_id` database column to `BIGINT`. ([\#10349](https://github.com/matrix-org/synapse/issues/10349))
- Convert internal type variable syntax to reflect wider ecosystem use. ([\#10350](https://github.com/matrix-org/synapse/issues/10350), [\#10380](https://github.com/matrix-org/synapse/issues/10380), [\#10381](https://github.com/matrix-org/synapse/issues/10381), [\#10382](https://github.com/matrix-org/synapse/issues/10382), [\#10418](https://github.com/matrix-org/synapse/issues/10418))
- Make the Github Actions workflow configuration more efficient. ([\#10383](https://github.com/matrix-org/synapse/issues/10383))
- Add type hints to `get_{domain,localpart}_from_id`. ([\#10385](https://github.com/matrix-org/synapse/issues/10385))
- When building Debian packages for prerelease versions, set the Section accordingly. ([\#10391](https://github.com/matrix-org/synapse/issues/10391))
- Add type hints and comments to event auth code. ([\#10393](https://github.com/matrix-org/synapse/issues/10393))
- Stagger sending of presence update to remote servers, reducing CPU spikes caused by starting many connections to remote servers at once. ([\#10398](https://github.com/matrix-org/synapse/issues/10398))
- Remove unused `events_by_room` code (tech debt). ([\#10421](https://github.com/matrix-org/synapse/issues/10421))
- Add a github actions job which records success of other jobs. ([\#10430](https://github.com/matrix-org/synapse/issues/10430))
Synapse 1.38.0 (2021-07-13)
===========================

2
CONTRIBUTING.md
View file

@ -392,7 +392,7 @@ By now, you know the drill!
# Notes for maintainers on merging PRs etc
There are some notes for those with commit access to the project on how we
manage git [here](docs/dev/git.md).
manage git [here](docs/development/git.md).
# Conclusion

1
changelog.d/10250.bugfix
View file

@ -1 +0,0 @@
Add base starting insertion event when no chunk ID is specified in the historical batch send API.

1
changelog.d/10254.feature Normal file
View file

@ -0,0 +1 @@
Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events.

1
changelog.d/10276.bugfix
View file

@ -1 +0,0 @@
Fix historical batch send endpoint (MSC2716) rejecting batches with messages from multiple senders.

1
changelog.d/10283.feature Normal file
View file

@ -0,0 +1 @@
Initial support for MSC3244, Room version capabilities over the /capabilities API.

1
changelog.d/10289.misc
View file

@ -1 +0,0 @@
Convert `room_depth.min_depth` column to a `BIGINT`.

1
changelog.d/10298.feature
View file

@ -1 +0,0 @@
The spaces summary API now returns any joinable rooms, not only rooms which are world-readable.

1
changelog.d/10305.feature
View file

@ -1 +0,0 @@
The spaces summary API now returns any joinable rooms, not only rooms which are world-readable.

1
changelog.d/10313.doc
View file

@ -1 +0,0 @@
Simplify structure of room admin API.

1
changelog.d/10316.misc
View file

@ -1 +0,0 @@
Rebuild event context and auth when processing specific results from `ThirdPartyEventRules` modules.

1
changelog.d/10317.bugfix
View file

@ -1 +0,0 @@
Fix purging rooms that other homeservers are still sending events for. Contributed by @ilmari.

1
changelog.d/10322.doc
View file

@ -1 +0,0 @@
Fix a broken link in the admin api docs.

1
changelog.d/10324.misc
View file

@ -1 +0,0 @@
Minor change to the code that populates `user_daily_visits`.

1
changelog.d/10337.doc
View file

@ -1 +0,0 @@
Fix formatting in the logcontext documentation.

1
changelog.d/10343.bugfix
View file

@ -1 +0,0 @@
Fix errors during backfill caused by previously purged redaction events. Contributed by Andreas Rammhold (@andir).

1
changelog.d/10344.bugfix
View file

@ -1 +0,0 @@
Fix the user directory becoming broken (and noisy errors being logged) when knocking and room statistics are in use.

1
changelog.d/10345.misc
View file

@ -1 +0,0 @@
Re-enable Sytests that were disabled for the 1.37.1 release.

1
changelog.d/10347.misc
View file

@ -1 +0,0 @@
Run `pyupgrade` on the codebase.

1
changelog.d/10349.misc
View file

@ -1 +0,0 @@
Switch `application_services_txns.txn_id` database column to `BIGINT`.

1
changelog.d/10350.misc
View file

@ -1 +0,0 @@
Convert internal type variable syntax to reflect wider ecosystem use.

1
changelog.d/10355.bugfix
View file

@ -1 +0,0 @@
Fix newly added `synapse_federation_server_oldest_inbound_pdu_in_staging` prometheus metric to measure age rather than timestamp.

1
changelog.d/10357.misc
View file

@ -1 +0,0 @@
Re-enable Sytests that were disabled for the 1.37.1 release.

1
changelog.d/10367.bugfix
View file

@ -1 +0,0 @@
Bugfix `make_room_admin` fails for users that have left a private room.

1
changelog.d/10368.doc
View file

@ -1 +0,0 @@
Add delegation example for caddy in the reverse proxy documentation. Contributed by @moritzdietz.

1
changelog.d/10370.doc
View file

@ -1 +0,0 @@
Fix some links in `docs` and `contrib`.

1
changelog.d/10383.misc
View file

@ -1 +0,0 @@
Make the Github Actions workflow configuration more efficient.

1
changelog.d/10385.misc
View file

@ -1 +0,0 @@
Add type hints to `get_{domain,localpart}_from_id`.

1
changelog.d/10391.misc
View file

@ -1 +0,0 @@
When building Debian packages for prerelease versions, set the Section accordingly.

1
changelog.d/10393.misc
View file

@ -1 +0,0 @@
Add type hints and comments to event auth code.

1
changelog.d/10396.doc
View file

@ -1 +0,0 @@
Add instructructions on installing Debian packages for release candidates.

1
changelog.d/10408.misc Normal file
View file

@ -0,0 +1 @@
Add type hints to `synapse.federation.transport.client` module.

1
changelog.d/10426.feature Normal file
View file

@ -0,0 +1 @@
Email notifications now state whether an invitation is to a room or a space.

1
changelog.d/10429.misc Normal file
View file

@ -0,0 +1 @@
Drop backwards-compatibility code that was required to support Ubuntu Xenial.

1
changelog.d/10431.misc Normal file
View file

@ -0,0 +1 @@
Use a docker image cache for the prerequisites for the debian package build.

1
changelog.d/10432.misc Normal file
View file

@ -0,0 +1 @@
Connect historical chunks together with chunk events instead of a content field (MSC2716).

1
changelog.d/10437.misc Normal file
View file

@ -0,0 +1 @@
Improve servlet type hints.

1
changelog.d/10438.misc Normal file
View file

@ -0,0 +1 @@
Improve servlet type hints.

1
changelog.d/10442.misc Normal file
View file

@ -0,0 +1 @@
Replace usage of `or_ignore` in `simple_insert` with `simple_upsert` usage, to stop spamming postgres logs with spurious ERROR messages.

1
changelog.d/10444.misc Normal file
View file

@ -0,0 +1 @@
Update the `tests-done` Github Actions status.

1
changelog.d/10445.doc Normal file
View file

@ -0,0 +1 @@
Fix hierarchy of providers on the OpenID page.

1
changelog.d/10446.misc Normal file
View file

@ -0,0 +1 @@
Update type annotations to work with forthcoming Twisted 21.7.0 release.

1
changelog.d/10448.feature Normal file
View file

@ -0,0 +1 @@
Add `creation_ts` to list users admin API.

1
changelog.d/10451.misc Normal file
View file

@ -0,0 +1 @@
Cancel redundant GHA workflows when a new commit is pushed.

1
changelog.d/10453.doc Normal file
View file

@ -0,0 +1 @@
Consolidate development documentation to `docs/development/`.

1
changelog.d/10461.misc Normal file
View file

@ -0,0 +1 @@
Fix an error which prevented the Github Actions workflow to build the docker images from running.

1
changelog.d/10463.misc Normal file
View file

@ -0,0 +1 @@
Disable `msc2716` Complement tests until Complement updates are merged.

1
changelog.d/10482.misc Normal file
View file

@ -0,0 +1 @@
Additional type hints in the state handler.

1
changelog.d/9721.removal
View file

@ -1 +0,0 @@
Remove functionality associated with the unused `room_stats_historical` and `user_stats_historical` tables. Contributed by @xmunoz.

4
debian/build_virtualenv vendored
View file

@ -33,13 +33,11 @@ esac
# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
# than the 2/3 compatible `virtualenv`.
# Pin pip to 20.3.4 to fix breakage in 21.0 on py3.5 (xenial)
dh_virtualenv \
--install-suffix "matrix-synapse" \
--builtin-venv \
--python "$SNAKE" \
--upgrade-pip-to="20.3.4" \
--upgrade-pip \
--preinstall="lxml" \
--preinstall="mock" \
--extra-pip-arg="--no-cache-dir" \

18
debian/changelog vendored
View file

@ -1,3 +1,21 @@
matrix-synapse-py3 (1.39.0ubuntu1) UNRELEASED; urgency=medium
* Drop backwards-compatibility code that was required to support Ubuntu Xenial.
-- Richard van der Hoff <richard@matrix.org> Tue, 20 Jul 2021 00:10:03 +0100
matrix-synapse-py3 (1.38.1) stable; urgency=medium
* New synapse release 1.38.1.
-- Synapse Packaging team <packages@matrix.org> Thu, 22 Jul 2021 15:37:06 +0100
matrix-synapse-py3 (1.39.0~rc1) stable; urgency=medium
* New synapse release 1.39.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 20 Jul 2021 14:28:34 +0100
matrix-synapse-py3 (1.38.0) stable; urgency=medium
* New synapse release 1.38.0.

2
debian/compat vendored
View file

@ -1 +1 @@
9
10

5
debian/control vendored
View file

@ -3,11 +3,8 @@ Section: contrib/python
Priority: extra
Maintainer: Synapse Packaging team <packages@matrix.org>
# keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
# TODO: Remove the dependency on dh-systemd after dropping support for Ubuntu xenial
# On all other supported releases, it's merely a transitional package which
# does nothing but depends on debhelper (> 9.20160709)
Build-Depends:
debhelper (>= 9.20160709) | dh-systemd,
debhelper (>= 10),
dh-virtualenv (>= 1.1),
libsystemd-dev,
libpq-dev,

4
debian/rules vendored
View file

@ -51,7 +51,5 @@ override_dh_shlibdeps:
override_dh_virtualenv:
./debian/build_virtualenv
# We are restricted to compat level 9 (because xenial), so have to
# enable the systemd bits manually.
%:
dh $@ --with python-virtualenv --with systemd
dh $@ --with python-virtualenv

18
docker/Dockerfile-dhvirtualenv
View file

@ -15,6 +15,15 @@ ARG distro=""
###
### Stage 0: build a dh-virtualenv
###
# This is only really needed on bionic and focal, since other distributions we
# care about have a recent version of dh-virtualenv by default. Unfortunately,
# it looks like focal is going to be with us for a while.
#
# (focal doesn't have a dh-virtualenv package at all. There is a PPA at
# https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
# it's not obviously easier to use that than to build our own.)
FROM ${distro} as builder
RUN apt-get update -qq -o Acquire::Languages=none
@ -27,7 +36,7 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
wget
# fetch and unpack the package
# TODO: Upgrade to 1.2.2 once xenial is dropped
# TODO: Upgrade to 1.2.2 once bionic is dropped (1.2.2 requires debhelper 12; bionic has only 11)
RUN mkdir /dh-virtualenv
RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz
RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
@ -59,8 +68,6 @@ ENV LANG C.UTF-8
#
# NB: keep this list in sync with the list of build-deps in debian/control
# TODO: it would be nice to do that automatically.
# TODO: Remove the dh-systemd stanza after dropping support for Ubuntu xenial
# it's a transitional package on all other, more recent releases
RUN apt-get update -qq -o Acquire::Languages=none \
&& env DEBIAN_FRONTEND=noninteractive apt-get install \
-yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
@ -76,10 +83,7 @@ RUN apt-get update -qq -o Acquire::Languages=none \
python3-venv \
sqlite3 \
libpq-dev \
xmlsec1 \
&& ( env DEBIAN_FRONTEND=noninteractive apt-get install \
-yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
dh-systemd || true )
xmlsec1
COPY --from=builder /dh-virtualenv_1.2~dev-1_all.deb /

4
docs/MSC1711_certificates_FAQ.md
View file

@ -132,7 +132,7 @@ your domain, you can simply route all traffic through the reverse proxy by
updating the SRV record appropriately (or removing it, if the proxy listens on
8448).
See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
reverse proxy.
#### Option 3: add a .well-known file to delegate your matrix traffic
@ -303,7 +303,7 @@ We no longer actively recommend against using a reverse proxy. Many admins will
find it easier to direct federation traffic to a reverse proxy and manage their
own TLS certificates, and this is a supported configuration.
See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
reverse proxy.
### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?

6
docs/SUMMARY.md
View file

@ -67,7 +67,7 @@
# Development
- [Contributing Guide](development/contributing_guide.md)
- [Code Style](code_style.md)
- [Git Usage](dev/git.md)
- [Git Usage](development/git.md)
- [Testing]()
- [OpenTracing](opentracing.md)
- [Database Schemas](development/database_schema.md)
@ -77,8 +77,8 @@
- [TCP Replication](tcp_replication.md)
- [Internal Documentation](development/internal_documentation/README.md)
- [Single Sign-On]()
- [SAML](dev/saml.md)
- [CAS](dev/cas.md)
- [SAML](development/saml.md)
- [CAS](development/cas.md)
- [State Resolution]()
- [The Auth Chain Difference Algorithm](auth_chain_difference_algorithm.md)
- [Media Repository](media_repository.md)

2
docs/admin_api/server_notices.md
View file

@ -45,4 +45,4 @@ Once the notice has been sent, the API will return the following response:
```
Note that server notices must be enabled in `homeserver.yaml` before this API
can be used. See [server_notices.md](../server_notices.md) for more information.
can be used. See [the server notices documentation](../server_notices.md) for more information.

10
docs/admin_api/user_admin_api.md
View file

@ -144,7 +144,8 @@ A response body like the following is returned:
"deactivated": 0,
"shadow_banned": 0,
"displayname": "<User One>",
"avatar_url": null
"avatar_url": null,
"creation_ts": 1560432668000
}, {
"name": "<user_id2>",
"is_guest": 0,
@ -153,7 +154,8 @@ A response body like the following is returned:
"deactivated": 0,
"shadow_banned": 0,
"displayname": "<User Two>",
"avatar_url": "<avatar_url>"
"avatar_url": "<avatar_url>",
"creation_ts": 1561550621000
}
],
"next_token": "100",
@ -197,11 +199,12 @@ The following parameters should be set in the URL:
- `shadow_banned` - Users are ordered by `shadow_banned` status.
- `displayname` - Users are ordered alphabetically by `displayname`.
- `avatar_url` - Users are ordered alphabetically by avatar URL.
- `creation_ts` - Users are ordered by when the users was created in ms.
- `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
Setting this value to `b` will reverse the above sort order. Defaults to `f`.
Caution. The database only has indexes on the columns `name` and `created_ts`.
Caution. The database only has indexes on the columns `name` and `creation_ts`.
This means that if a different sort order is used (`is_guest`, `admin`,
`user_type`, `deactivated`, `shadow_banned`, `avatar_url` or `displayname`),
this can cause a large load on the database, especially for large environments.
@ -222,6 +225,7 @@ The following fields are returned in the JSON response body:
- `shadow_banned` - bool - Status if that user has been marked as shadow banned.
- `displayname` - string - The user's display name if they have set one.
- `avatar_url` - string - The user's avatar URL if they have set one.
- `creation_ts` - integer - The user's creation timestamp in ms.
- `next_token`: string representing a positive integer - Indication for pagination. See above.
- `total` - integer - Total number of media.

2
docs/consent_tracking.md
View file

@ -152,7 +152,7 @@ version of the policy. To do so:
* ensure that the consent resource is configured, as in the previous section
* ensure that server notices are configured, as in [server_notices.md](server_notices.md).
* ensure that server notices are configured, as in [the server notice documentation](server_notices.md).
* Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For
example:

2
docs/delegate.md
View file

@ -74,7 +74,7 @@ We no longer actively recommend against using a reverse proxy. Many admins will
find it easier to direct federation traffic to a reverse proxy and manage their
own TLS certificates, and this is a supported configuration.
See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
reverse proxy.
### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?

0
docs/dev/cas.md → docs/development/cas.md
View file

6
docs/dev/git.md → docs/development/git.md
View file

@ -9,7 +9,7 @@ commits each of which contains a single change building on what came
before. Here, by way of an arbitrary example, is the top of `git log --graph
b2dba0607`:
<img src="git/clean.png" alt="clean git graph" width="500px">
<img src="img/git/clean.png" alt="clean git graph" width="500px">
Note how the commit comment explains clearly what is changing and why. Also
note the *absence* of merge commits, as well as the absence of commits called
@ -61,7 +61,7 @@ Ok, so that's what we'd like to achieve. How do we achieve it?
The TL;DR is: when you come to merge a pull request, you *probably* want to
“squash and merge”:
![squash and merge](git/squash.png).
![squash and merge](img/git/squash.png).
(This applies whether you are merging your own PR, or that of another
contributor.)
@ -105,7 +105,7 @@ complicated. Here's how we do it.
Let's start with a picture:
![branching model](git/branches.jpg)
![branching model](img/git/branches.jpg)
It looks complicated, but it's really not. There's one basic rule: *anyone* is
free to merge from *any* more-stable branch to *any* less-stable branch at

0
docs/dev/git/branches.jpg → docs/development/img/git/branches.jpg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 70 KiB

After

Width:  |  Height:  |  Size: 70 KiB

0
docs/dev/git/clean.png → docs/development/img/git/clean.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 108 KiB

After

Width:  |  Height:  |  Size: 108 KiB

0
docs/dev/git/squash.png → docs/development/img/git/squash.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 29 KiB

After

Width:  |  Height:  |  Size: 29 KiB

0
docs/dev/saml.md → docs/development/saml.md
View file

8
docs/federate.md
View file

@ -14,7 +14,7 @@ you set the `server_name` to match your machine's public DNS hostname.
For this default configuration to work, you will need to listen for TLS
connections on port 8448. The preferred way to do that is by using a
reverse proxy: see [reverse_proxy.md](reverse_proxy.md) for instructions
reverse proxy: see [the reverse proxy documentation](reverse_proxy.md) for instructions
on how to correctly set one up.
In some cases you might not want to run Synapse on the machine that has
@ -23,7 +23,7 @@ traffic to use a different port than 8448. For example, you might want to
have your user names look like `@user:example.com`, but you want to run
Synapse on `synapse.example.com` on port 443. This can be done using
delegation, which allows an admin to control where federation traffic should
be sent. See [delegate.md](delegate.md) for instructions on how to set this up.
be sent. See [the delegation documentation](delegate.md) for instructions on how to set this up.
Once federation has been configured, you should be able to join a room over
federation. A good place to start is `#synapse:matrix.org` - a room for
@ -44,8 +44,8 @@ a complicated dance which requires connections in both directions).
Another common problem is that people on other servers can't join rooms that
you invite them to. This can be caused by an incorrectly-configured reverse
proxy: see [reverse_proxy.md](reverse_proxy.md) for instructions on how to correctly
configure a reverse proxy.
proxy: see [the reverse proxy documentation](reverse_proxy.md) for instructions on how
to correctly configure a reverse proxy.
### Known issues

329
docs/log_contexts.md
View file

@ -14,12 +14,16 @@ The `synapse.logging.context` module provides a facilities for managing
the current log context (as well as providing the `LoggingContextFilter`
class).
Deferreds make the whole thing complicated, so this document describes
Asynchronous functions make the whole thing complicated, so this document describes
how it all works, and how to write code which follows the rules.
## Logcontexts without Deferreds
In this document, "awaitable" refers to any object which can be `await`ed. In the context of
Synapse, that normally means either a coroutine or a Twisted
[`Deferred`](https://twistedmatrix.com/documents/current/api/twisted.internet.defer.Deferred.html).
In the absence of any Deferred voodoo, things are simple enough. As with
## Logcontexts without asynchronous code
In the absence of any asynchronous voodoo, things are simple enough. As with
any code of this nature, the rule is that our function should leave
things as it found them:
@ -55,126 +59,109 @@ def do_request_handling():
logger.debug("phew")
```
## Using logcontexts with Deferreds
## Using logcontexts with awaitables
Deferreds --- and in particular, `defer.inlineCallbacks` --- break the
linear flow of code so that there is no longer a single entry point
where we should set the logcontext and a single exit point where we
should remove it.
Awaitables break the linear flow of code so that there is no longer a single entry point
where we should set the logcontext and a single exit point where we should remove it.
Consider the example above, where `do_request_handling` needs to do some
blocking operation, and returns a deferred:
blocking operation, and returns an awaitable:
```python
@defer.inlineCallbacks
def handle_request(request_id):
async def handle_request(request_id):
with context.LoggingContext() as request_context:
request_context.request = request_id
yield do_request_handling()
await do_request_handling()
logger.debug("finished")
```
In the above flow:
- The logcontext is set
- `do_request_handling` is called, and returns a deferred
- `handle_request` yields the deferred
- The `inlineCallbacks` wrapper of `handle_request` returns a deferred
- `do_request_handling` is called, and returns an awaitable
- `handle_request` awaits the awaitable
- Execution of `handle_request` is suspended
So we have stopped processing the request (and will probably go on to
start processing the next), without clearing the logcontext.
To circumvent this problem, synapse code assumes that, wherever you have
a deferred, you will want to yield on it. To that end, whereever
functions return a deferred, we adopt the following conventions:
an awaitable, you will want to `await` it. To that end, whereever
functions return awaitables, we adopt the following conventions:
**Rules for functions returning deferreds:**
**Rules for functions returning awaitables:**
> - If the deferred is already complete, the function returns with the
> - If the awaitable is already complete, the function returns with the
> same logcontext it started with.
> - If the deferred is incomplete, the function clears the logcontext
> before returning; when the deferred completes, it restores the
> - If the awaitable is incomplete, the function clears the logcontext
> before returning; when the awaitable completes, it restores the
> logcontext before running any callbacks.
That sounds complicated, but actually it means a lot of code (including
the example above) "just works". There are two cases:
- If `do_request_handling` returns a completed deferred, then the
- If `do_request_handling` returns a completed awaitable, then the
logcontext will still be in place. In this case, execution will
continue immediately after the `yield`; the "finished" line will
continue immediately after the `await`; the "finished" line will
be logged against the right context, and the `with` block restores
the original context before we return to the caller.
- If the returned deferred is incomplete, `do_request_handling` clears
- If the returned awaitable is incomplete, `do_request_handling` clears
the logcontext before returning. The logcontext is therefore clear
when `handle_request` yields the deferred. At that point, the
`inlineCallbacks` wrapper adds a callback to the deferred, and
returns another (incomplete) deferred to the caller, and it is safe
to begin processing the next request.
when `handle_request` `await`s the awaitable.
Once `do_request_handling`'s deferred completes, it will reinstate
the logcontext, before running the callback added by the
`inlineCallbacks` wrapper. That callback runs the second half of
`handle_request`, so again the "finished" line will be logged
against the right context, and the `with` block restores the
original context.
Once `do_request_handling`'s awaitable completes, it will reinstate
the logcontext, before running the second half of `handle_request`,
so again the "finished" line will be logged against the right context,
and the `with` block restores the original context.
As an aside, it's worth noting that `handle_request` follows our rules
-though that only matters if the caller has its own logcontext which it
- though that only matters if the caller has its own logcontext which it
cares about.
The following sections describe pitfalls and helpful patterns when
implementing these rules.
Always yield your deferreds
---------------------------
Always await your awaitables
----------------------------
Whenever you get a deferred back from a function, you should `yield` on
it as soon as possible. (Returning it directly to your caller is ok too,
if you're not doing `inlineCallbacks`.) Do not pass go; do not do any
logging; do not call any other functions.
Whenever you get an awaitable back from a function, you should `await` on
it as soon as possible. Do not pass go; do not do any logging; do not
call any other functions.
```python
@defer.inlineCallbacks
def fun():
async def fun():
logger.debug("starting")
yield do_some_stuff() # just like this
await do_some_stuff() # just like this
d = more_stuff()
result = yield d # also fine, of course
coro = more_stuff()
result = await coro # also fine, of course
return result
def nonInlineCallbacksFun():
logger.debug("just a wrapper really")
return do_some_stuff() # this is ok too - the caller will yield on
# it anyway.
```
Provided this pattern is followed all the way back up to the callchain
to where the logcontext was set, this will make things work out ok:
provided `do_some_stuff` and `more_stuff` follow the rules above, then
so will `fun` (as wrapped by `inlineCallbacks`) and
`nonInlineCallbacksFun`.
so will `fun`.
It's all too easy to forget to `yield`: for instance if we forgot that
`do_some_stuff` returned a deferred, we might plough on regardless. This
It's all too easy to forget to `await`: for instance if we forgot that
`do_some_stuff` returned an awaitable, we might plough on regardless. This
leads to a mess; it will probably work itself out eventually, but not
before a load of stuff has been logged against the wrong context.
(Normally, other things will break, more obviously, if you forget to
`yield`, so this tends not to be a major problem in practice.)
`await`, so this tends not to be a major problem in practice.)
Of course sometimes you need to do something a bit fancier with your
Deferreds - not all code follows the linear A-then-B-then-C pattern.
awaitable - not all code follows the linear A-then-B-then-C pattern.
Notes on implementing more complex patterns are in later sections.
## Where you create a new Deferred, make it follow the rules
## Where you create a new awaitable, make it follow the rules
Most of the time, a Deferred comes from another synapse function.
Sometimes, though, we need to make up a new Deferred, or we get a
Deferred back from external code. We need to make it follow our rules.
Most of the time, an awaitable comes from another synapse function.
Sometimes, though, we need to make up a new awaitable, or we get an awaitable
back from external code. We need to make it follow our rules.
The easy way to do it is with a combination of `defer.inlineCallbacks`,
and `context.PreserveLoggingContext`. Suppose we want to implement
The easy way to do it is by using `context.make_deferred_yieldable`. Suppose we want to implement
`sleep`, which returns a deferred which will run its callbacks after a
given number of seconds. That might look like:
@ -186,25 +173,12 @@ def get_sleep_deferred(seconds):
return d
```
That doesn't follow the rules, but we can fix it by wrapping it with
`PreserveLoggingContext` and `yield` ing on it:
That doesn't follow the rules, but we can fix it by calling it through
`context.make_deferred_yieldable`:
```python
@defer.inlineCallbacks
def sleep(seconds):
with PreserveLoggingContext():
yield get_sleep_deferred(seconds)
```
This technique works equally for external functions which return
deferreds, or deferreds we have made ourselves.
You can also use `context.make_deferred_yieldable`, which just does the
boilerplate for you, so the above could be written:
```python
def sleep(seconds):
return context.make_deferred_yieldable(get_sleep_deferred(seconds))
async def sleep(seconds):
return await context.make_deferred_yieldable(get_sleep_deferred(seconds))
```
## Fire-and-forget
@ -213,20 +187,18 @@ Sometimes you want to fire off a chain of execution, but not wait for
its result. That might look a bit like this:
```python
@defer.inlineCallbacks
def do_request_handling():
yield foreground_operation()
async def do_request_handling():
await foreground_operation()
# *don't* do this
background_operation()
logger.debug("Request handling complete")
@defer.inlineCallbacks
def background_operation():
yield first_background_step()
async def background_operation():
await first_background_step()
logger.debug("Completed first step")
yield second_background_step()
await second_background_step()
logger.debug("Completed second step")
```
@ -235,13 +207,13 @@ The above code does a couple of steps in the background after
against the `request_context` logcontext, which may or may not be
desirable. There are two big problems with the above, however. The first
problem is that, if `background_operation` returns an incomplete
Deferred, it will expect its caller to `yield` immediately, so will have
awaitable, it will expect its caller to `await` immediately, so will have
cleared the logcontext. In this example, that means that 'Request
handling complete' will be logged without any context.
The second problem, which is potentially even worse, is that when the
Deferred returned by `background_operation` completes, it will restore
the original logcontext. There is nothing waiting on that Deferred, so
awaitable returned by `background_operation` completes, it will restore
the original logcontext. There is nothing waiting on that awaitable, so
the logcontext will leak into the reactor and possibly get attached to
some arbitrary future operation.
@ -254,9 +226,8 @@ deferred completes will be the empty logcontext), and will restore the
current logcontext before continuing the foreground process:
```python
@defer.inlineCallbacks
def do_request_handling():
yield foreground_operation()
async def do_request_handling():
await foreground_operation()
# start background_operation off in the empty logcontext, to
# avoid leaking the current context into the reactor.
@ -274,16 +245,15 @@ Obviously that option means that the operations done in
The second option is to use `context.run_in_background`, which wraps a
function so that it doesn't reset the logcontext even when it returns
an incomplete deferred, and adds a callback to the returned deferred to
an incomplete awaitable, and adds a callback to the returned awaitable to
reset the logcontext. In other words, it turns a function that follows
the Synapse rules about logcontexts and Deferreds into one which behaves
the Synapse rules about logcontexts and awaitables into one which behaves
more like an external function --- the opposite operation to that
described in the previous section. It can be used like this:
```python
@defer.inlineCallbacks
def do_request_handling():
yield foreground_operation()
async def do_request_handling():
await foreground_operation()
context.run_in_background(background_operation)
@ -294,152 +264,53 @@ def do_request_handling():
## Passing synapse deferreds into third-party functions
A typical example of this is where we want to collect together two or
more deferred via `defer.gatherResults`:
more awaitables via `defer.gatherResults`:
```python
d1 = operation1()
d2 = operation2()
d3 = defer.gatherResults([d1, d2])
a1 = operation1()
a2 = operation2()
a3 = defer.gatherResults([a1, a2])
```
This is really a variation of the fire-and-forget problem above, in that
we are firing off `d1` and `d2` without yielding on them. The difference
we are firing off `a1` and `a2` without awaiting on them. The difference
is that we now have third-party code attached to their callbacks. Anyway
either technique given in the [Fire-and-forget](#fire-and-forget)
section will work.
Of course, the new Deferred returned by `gatherResults` needs to be
Of course, the new awaitable returned by `gather` needs to be
wrapped in order to make it follow the logcontext rules before we can
yield it, as described in [Where you create a new Deferred, make it
yield it, as described in [Where you create a new awaitable, make it
follow the
rules](#where-you-create-a-new-deferred-make-it-follow-the-rules).
rules](#where-you-create-a-new-awaitable-make-it-follow-the-rules).
So, option one: reset the logcontext before starting the operations to
be gathered:
```python
@defer.inlineCallbacks
def do_request_handling():
async def do_request_handling():
with PreserveLoggingContext():
d1 = operation1()
d2 = operation2()
result = yield defer.gatherResults([d1, d2])
a1 = operation1()
a2 = operation2()
result = await defer.gatherResults([a1, a2])
```
In this case particularly, though, option two, of using
`context.preserve_fn` almost certainly makes more sense, so that
`context.run_in_background` almost certainly makes more sense, so that
`operation1` and `operation2` are both logged against the original
logcontext. This looks like:
```python
@defer.inlineCallbacks
def do_request_handling():
d1 = context.preserve_fn(operation1)()
d2 = context.preserve_fn(operation2)()
async def do_request_handling():
a1 = context.run_in_background(operation1)
a2 = context.run_in_background(operation2)
with PreserveLoggingContext():
result = yield defer.gatherResults([d1, d2])
result = await make_deferred_yieldable(defer.gatherResults([a1, a2]))
```
## Was all this really necessary?
## A note on garbage-collection of awaitable chains
The conventions used work fine for a linear flow where everything
happens in series via `defer.inlineCallbacks` and `yield`, but are
certainly tricky to follow for any more exotic flows. It's hard not to
wonder if we could have done something else.
We're not going to rewrite Synapse now, so the following is entirely of
academic interest, but I'd like to record some thoughts on an
alternative approach.
I briefly prototyped some code following an alternative set of rules. I
think it would work, but I certainly didn't get as far as thinking how
it would interact with concepts as complicated as the cache descriptors.
My alternative rules were:
- functions always preserve the logcontext of their caller, whether or
not they are returning a Deferred.
- Deferreds returned by synapse functions run their callbacks in the
same context as the function was orignally called in.
The main point of this scheme is that everywhere that sets the
logcontext is responsible for clearing it before returning control to
the reactor.
So, for example, if you were the function which started a
`with LoggingContext` block, you wouldn't `yield` within it --- instead
you'd start off the background process, and then leave the `with` block
to wait for it:
```python
def handle_request(request_id):
with context.LoggingContext() as request_context:
request_context.request = request_id
d = do_request_handling()
def cb(r):
logger.debug("finished")
d.addCallback(cb)
return d
```
(in general, mixing `with LoggingContext` blocks and
`defer.inlineCallbacks` in the same function leads to slighly
counter-intuitive code, under this scheme).
Because we leave the original `with` block as soon as the Deferred is
returned (as opposed to waiting for it to be resolved, as we do today),
the logcontext is cleared before control passes back to the reactor; so
if there is some code within `do_request_handling` which needs to wait
for a Deferred to complete, there is no need for it to worry about
clearing the logcontext before doing so:
```python
def handle_request():
r = do_some_stuff()
r.addCallback(do_some_more_stuff)
return r
```
--- and provided `do_some_stuff` follows the rules of returning a
Deferred which runs its callbacks in the original logcontext, all is
happy.
The business of a Deferred which runs its callbacks in the original
logcontext isn't hard to achieve --- we have it today, in the shape of
`context._PreservingContextDeferred`:
```python
def do_some_stuff():
deferred = do_some_io()
pcd = _PreservingContextDeferred(LoggingContext.current_context())
deferred.chainDeferred(pcd)
return pcd
```
It turns out that, thanks to the way that Deferreds chain together, we
automatically get the property of a context-preserving deferred with
`defer.inlineCallbacks`, provided the final Defered the function
`yields` on has that property. So we can just write:
```python
@defer.inlineCallbacks
def handle_request():
yield do_some_stuff()
yield do_some_more_stuff()
```
To conclude: I think this scheme would have worked equally well, with
less danger of messing it up, and probably made some more esoteric code
easier to write. But again --- changing the conventions of the entire
Synapse codebase is not a sensible option for the marginal improvement
offered.
## A note on garbage-collection of Deferred chains
It turns out that our logcontext rules do not play nicely with Deferred
It turns out that our logcontext rules do not play nicely with awaitable
chains which get orphaned and garbage-collected.
Imagine we have some code that looks like this:
@ -451,13 +322,12 @@ def on_something_interesting():
for d in listener_queue:
d.callback("foo")
@defer.inlineCallbacks
def await_something_interesting():
new_deferred = defer.Deferred()
listener_queue.append(new_deferred)
async def await_something_interesting():
new_awaitable = defer.Deferred()
listener_queue.append(new_awaitable)
with PreserveLoggingContext():
yield new_deferred
await new_awaitable
```
Obviously, the idea here is that we have a bunch of things which are
@ -476,18 +346,19 @@ def reset_listener_queue():
listener_queue.clear()
```
So, both ends of the deferred chain have now dropped their references,
and the deferred chain is now orphaned, and will be garbage-collected at
some point. Note that `await_something_interesting` is a generator
function, and when Python garbage-collects generator functions, it gives
them a chance to clean up by making the `yield` raise a `GeneratorExit`
So, both ends of the awaitable chain have now dropped their references,
and the awaitable chain is now orphaned, and will be garbage-collected at
some point. Note that `await_something_interesting` is a coroutine,
which Python implements as a generator function. When Python
garbage-collects generator functions, it gives them a chance to
clean up by making the `async` (or `yield`) raise a `GeneratorExit`
exception. In our case, that means that the `__exit__` handler of
`PreserveLoggingContext` will carefully restore the request context, but
there is now nothing waiting for its return, so the request context is
never cleared.
To reiterate, this problem only arises when *both* ends of a deferred
chain are dropped. Dropping the the reference to a deferred you're
supposed to be calling is probably bad practice, so this doesn't
To reiterate, this problem only arises when *both* ends of a awaitable
chain are dropped. Dropping the the reference to an awaitable you're
supposed to be awaiting is bad practice, so this doesn't
actually happen too much. Unfortunately, when it does happen, it will
lead to leaked logcontexts which are incredibly hard to track down.

109
docs/modules.md
View file

@ -63,7 +63,7 @@ Modules can register web resources onto Synapse's web server using the following
API method:
```python
def ModuleApi.register_web_resource(path: str, resource: IResource)
def ModuleApi.register_web_resource(path: str, resource: IResource) -> None
```
The path is the full absolute path to register the resource at. For example, if you
@ -91,12 +91,17 @@ are split in categories. A single module may implement callbacks from multiple c
and is under no obligation to implement all callbacks from the categories it registers
callbacks for.
Modules can register callbacks using one of the module API's `register_[...]_callbacks`
methods. The callback functions are passed to these methods as keyword arguments, with
the callback name as the argument name and the function as its value. This is demonstrated
in the example below. A `register_[...]_callbacks` method exists for each module type
documented in this section.
#### Spam checker callbacks
To register one of the callbacks described in this section, a module needs to use the
module API's `register_spam_checker_callbacks` method. The callback functions are passed
to `register_spam_checker_callbacks` as keyword arguments, with the callback name as the
argument name and the function as its value. This is demonstrated in the example below.
Spam checker callbacks allow module developers to implement spam mitigation actions for
Synapse instances. Spam checker callbacks can be registered using the module API's
`register_spam_checker_callbacks` method.
The available spam checker callbacks are:
@ -115,7 +120,7 @@ async def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool
Called when processing an invitation. The module must return a `bool` indicating whether
the inviter can invite the invitee to the given room. Both inviter and invitee are
represented by their Matrix user ID (i.e. `@alice:example.com`).
represented by their Matrix user ID (e.g. `@alice:example.com`).
```python
async def user_may_create_room(user: str) -> bool
@ -181,13 +186,103 @@ The arguments passed to this callback are:
```python
async def check_media_file_for_spam(
file_wrapper: "synapse.rest.media.v1.media_storage.ReadableFileWrapper",
file_info: "synapse.rest.media.v1._base.FileInfo"
file_info: "synapse.rest.media.v1._base.FileInfo",
) -> bool
```
Called when storing a local or remote file. The module must return a boolean indicating
whether the given file can be stored in the homeserver's media store.
#### Account validity callbacks
Account validity callbacks allow module developers to add extra steps to verify the
validity on an account, i.e. see if a user can be granted access to their account on the
Synapse instance. Account validity callbacks can be registered using the module API's
`register_account_validity_callbacks` method.
The available account validity callbacks are:
```python
async def is_user_expired(user: str) -> Optional[bool]
```
Called when processing any authenticated request (except for logout requests). The module
can return a `bool` to indicate whether the user has expired and should be locked out of
their account, or `None` if the module wasn't able to figure it out. The user is
represented by their Matrix user ID (e.g. `@alice:example.com`).
If the module returns `True`, the current request will be denied with the error code
`ORG_MATRIX_EXPIRED_ACCOUNT` and the HTTP status code 403. Note that this doesn't
invalidate the user's access token.
```python
async def on_user_registration(user: str) -> None
```
Called after successfully registering a user, in case the module needs to perform extra
operations to keep track of them. (e.g. add them to a database table). The user is
represented by their Matrix user ID.
#### Third party rules callbacks
Third party rules callbacks allow module developers to add extra checks to verify the
validity of incoming events. Third party event rules callbacks can be registered using
the module API's `register_third_party_rules_callbacks` method.
The available third party rules callbacks are:
```python
async def check_event_allowed(
event: "synapse.events.EventBase",
state_events: "synapse.types.StateMap",
) -> Tuple[bool, Optional[dict]]
```
**<span style="color:red">
This callback is very experimental and can and will break without notice. Module developers
are encouraged to implement `check_event_for_spam` from the spam checker category instead.
</span>**
Called when processing any incoming event, with the event and a `StateMap`
representing the current state of the room the event is being sent into. A `StateMap` is
a dictionary that maps tuples containing an event type and a state key to the
corresponding state event. For example retrieving the room's `m.room.create` event from
the `state_events` argument would look like this: `state_events.get(("m.room.create", ""))`.
The module must return a boolean indicating whether the event can be allowed.
Note that this callback function processes incoming events coming via federation
traffic (on top of client traffic). This means denying an event might cause the local
copy of the room's history to diverge from that of remote servers. This may cause
federation issues in the room. It is strongly recommended to only deny events using this
callback function if the sender is a local user, or in a private federation in which all
servers are using the same module, with the same configuration.
If the boolean returned by the module is `True`, it may also tell Synapse to replace the
event with new data by returning the new event's data as a dictionary. In order to do
that, it is recommended the module calls `event.get_dict()` to get the current event as a
dictionary, and modify the returned dictionary accordingly.
Note that replacing the event only works for events sent by local users, not for events
received over federation.
```python
async def on_create_room(
requester: "synapse.types.Requester",
request_content: dict,
is_requester_admin: bool,
) -> None
```
Called when processing a room creation request, with the `Requester` object for the user
performing the request, a dictionary representing the room creation request's JSON body
(see [the spec](https://matrix.org/docs/spec/client_server/latest#post-matrix-client-r0-createroom)
for a list of possible parameters), and a boolean indicating whether the user performing
the request is a server admin.
Modules can modify the `request_content` (by e.g. adding events to its `initial_state`),
or deny the room's creation by raising a `module_api.errors.SynapseError`.
### Porting an existing module that uses the old interface
In order to port a module that uses Synapse's old module interface, its author needs to:

2
docs/openid.md
View file

@ -410,7 +410,7 @@ oidc_providers:
display_name_template: "{{ user.name }}"
```
## Apple
### Apple
Configuring "Sign in with Apple" (SiWA) requires an Apple Developer account.

2
docs/replication.md
View file

@ -28,7 +28,7 @@ minimal.
### The Replication Protocol
See [tcp_replication.md](tcp_replication.md)
See [the TCP replication documentation](tcp_replication.md).
### The Slaved DataStore

2
docs/reverse_proxy.md
View file

@ -21,7 +21,7 @@ port 8448. Where these are different, we refer to the 'client port' and the
'federation port'. See [the Matrix
specification](https://matrix.org/docs/spec/server_server/latest#resolving-server-names)
for more details of the algorithm used for federation connections, and
[delegate.md](delegate.md) for instructions on setting up delegation.
[Delegation](delegate.md) for instructions on setting up delegation.
**NOTE**: Your reverse proxy must not `canonicalise` or `normalise`
the requested URI in any way (for example, by decoding `%xx` escapes).

98
docs/sample_config.yaml
View file

@ -1310,91 +1310,6 @@ account_threepid_delegates:
#auto_join_rooms_for_guests: false
## Account Validity ##
# Optional account validity configuration. This allows for accounts to be denied
# any request after a given period.
#
# Once this feature is enabled, Synapse will look for registered users without an
# expiration date at startup and will add one to every account it found using the
# current settings at that time.
# This means that, if a validity period is set, and Synapse is restarted (it will
# then derive an expiration date from the current validity period), and some time
# after that the validity period changes and Synapse is restarted, the users'
# expiration dates won't be updated unless their account is manually renewed. This
# date will be randomly selected within a range [now + period - d ; now + period],
# where d is equal to 10% of the validity period.
#
account_validity:
# The account validity feature is disabled by default. Uncomment the
# following line to enable it.
#
#enabled: true
# The period after which an account is valid after its registration. When
# renewing the account, its validity period will be extended by this amount
# of time. This parameter is required when using the account validity
# feature.
#
#period: 6w
# The amount of time before an account's expiry date at which Synapse will
# send an email to the account's email address with a renewal link. By
# default, no such emails are sent.
#
# If you enable this setting, you will also need to fill out the 'email' and
# 'public_baseurl' configuration sections.
#
#renew_at: 1w
# The subject of the email sent out with the renewal link. '%(app)s' can be
# used as a placeholder for the 'app_name' parameter from the 'email'
# section.
#
# Note that the placeholder must be written '%(app)s', including the
# trailing 's'.
#
# If this is not set, a default value is used.
#
#renew_email_subject: "Renew your %(app)s account"
# Directory in which Synapse will try to find templates for the HTML files to
# serve to the user when trying to renew an account. If not set, default
# templates from within the Synapse package will be used.
#
# The currently available templates are:
#
# * account_renewed.html: Displayed to the user after they have successfully
# renewed their account.
#
# * account_previously_renewed.html: Displayed to the user if they attempt to
# renew their account with a token that is valid, but that has already
# been used. In this case the account is not renewed again.
#
# * invalid_token.html: Displayed to the user when they try to renew an account
# with an unknown or invalid renewal token.
#
# See https://github.com/matrix-org/synapse/tree/master/synapse/res/templates for
# default template contents.
#
# The file name of some of these templates can be configured below for legacy
# reasons.
#
#template_dir: "res/templates"
# A custom file name for the 'account_renewed.html' template.
#
# If not set, the file is assumed to be named "account_renewed.html".
#
#account_renewed_html_path: "account_renewed.html"
# A custom file name for the 'invalid_token.html' template.
#
# If not set, the file is assumed to be named "invalid_token.html".
#
#invalid_token_html_path: "invalid_token.html"
## Metrics ###
# Enable collection and rendering of performance metrics
@ -2739,19 +2654,6 @@ stats:
# action: allow
# Server admins can define a Python module that implements extra rules for
# allowing or denying incoming events. In order to work, this module needs to
# override the methods defined in synapse/events/third_party_rules.py.
#
# This feature is designed to be used in closed federations only, where each
# participating server enforces the same rules.
#
#third_party_event_rules:
# module: "my_custom_project.SuperRulesSet"
# config:
# example_option: 'things'
## Opentracing ##
# These settings enable opentracing, which implements distributed tracing.

4
docs/server_notices.md
View file

@ -3,8 +3,8 @@
'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
channel whereby server administrators can send messages to users on the server.
They are used as part of communication of the server polices(see
[consent_tracking.md](consent_tracking.md)), however the intention is that
They are used as part of communication of the server polices (see
[Consent Tracking](consent_tracking.md)), however the intention is that
they may also find a use for features such as "Message of the day".
This is a feature specific to Synapse, but it uses standard Matrix

19
docs/setup/installation.md
View file

@ -166,13 +166,16 @@ sudo dnf groupinstall "Development Tools"
Installing prerequisites on macOS:
You may need to install the latest Xcode developer tools:
```sh
xcode-select --install
sudo easy_install pip
sudo pip install virtualenv
brew install pkg-config libffi
```
On ARM-based Macs you may need to explicitly install libjpeg which is a pillow dependency. You can use Homebrew (https://brew.sh):
```sh
brew install jpeg
```
On macOS Catalina (10.15) you may need to explicitly install OpenSSL
via brew and inform `pip` about it so that `psycopg2` builds:
@ -412,7 +415,7 @@ instead. Advantages include:
- allowing the DB to be run on separate hardware
For information on how to install and use PostgreSQL in Synapse, please see
[docs/postgres.md](../postgres.md)
[Using Postgres](../postgres.md)
SQLite is only acceptable for testing purposes. SQLite should not be used in
a production server. Synapse will perform poorly when using
@ -427,7 +430,7 @@ over HTTPS.
The recommended way to do so is to set up a reverse proxy on port
`8448`. You can find documentation on doing so in
[docs/reverse_proxy.md](../reverse_proxy.md).
[the reverse proxy documentation](../reverse_proxy.md).
Alternatively, you can configure Synapse to expose an HTTPS port. To do
so, you will need to edit `homeserver.yaml`, as follows:
@ -454,7 +457,7 @@ so, you will need to edit `homeserver.yaml`, as follows:
`cert.pem`).
For a more detailed guide to configuring your server for federation, see
[federate.md](../federate.md).
[Federation](../federate.md).
### Client Well-Known URI
@ -566,9 +569,7 @@ on your server even if `enable_registration` is `false`.
### Setting up a TURN server
For reliable VoIP calls to be routed via this homeserver, you MUST configure
a TURN server. See
[docs/turn-howto.md](../turn-howto.md)
for details.
a TURN server. See [TURN setup](../turn-howto.md) for details.
### URL previews

6
docs/spam_checker.md
View file

@ -1,6 +1,8 @@
**Note: this page of the Synapse documentation is now deprecated. For up to date
<h2 style="color:red">
This page of the Synapse documentation is now deprecated. For up to date
documentation on setting up or writing a spam checker module, please see
[this page](https://matrix-org.github.io/synapse/develop/modules.html).**
<a href="modules.md">this page</a>.
</h2>
# Handling spam in Synapse

2
docs/systemd-with-workers/README.md
View file

@ -14,7 +14,7 @@ contains an example configuration for the `federation_reader` worker.
## Synapse configuration files
See [workers.md](../workers.md) for information on how to set up the
See [the worker documentation](../workers.md) for information on how to set up the
configuration files and reverse-proxy correctly.
Below is a sample `federation_reader` worker configuration file.
```yaml

13
docs/upgrade.md
View file

@ -86,6 +86,19 @@ process, for example:
```
# Upgrading to v1.39.0
## Deprecation of the current third-party rules module interface
The current third-party rules module interface is deprecated in favour of the new generic
modules system introduced in Synapse v1.37.0. Authors of third-party rules modules can refer
to [this documentation](modules.md#porting-an-existing-module-that-uses-the-old-interface)
to update their modules. Synapse administrators can refer to [this documentation](modules.md#using-modules)
to update their configuration once the modules they are using have been updated.
We plan to remove support for the current third-party rules interface in September 2021.
# Upgrading to v1.38.0
## Re-indexing of `events` table on Postgres databases

6
docs/workers.md
View file

@ -73,7 +73,7 @@ https://hub.docker.com/r/matrixdotorg/synapse/.
To make effective use of the workers, you will need to configure an HTTP
reverse-proxy such as nginx or haproxy, which will direct incoming requests to
the correct worker, or to the main synapse instance. See
[reverse_proxy.md](reverse_proxy.md) for information on setting up a reverse
[the reverse proxy documentation](reverse_proxy.md) for information on setting up a reverse
proxy.
When using workers, each worker process has its own configuration file which
@ -170,8 +170,8 @@ Finally, you need to start your worker processes. This can be done with either
`synctl` or your distribution's preferred service manager such as `systemd`. We
recommend the use of `systemd` where available: for information on setting up
`systemd` to start synapse workers, see
[systemd-with-workers](systemd-with-workers). To use `synctl`, see
[synctl_workers.md](synctl_workers.md).
[Systemd with Workers](systemd-with-workers). To use `synctl`, see
[Using synctl with Workers](synctl_workers.md).
## Available worker applications

38
scripts-dev/build_debian_packages
View file

@ -17,6 +17,7 @@ import subprocess
import sys
import threading
from concurrent.futures import ThreadPoolExecutor
from typing import Optional, Sequence
DISTS = (
"debian:buster",
@ -39,8 +40,11 @@ projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
class Builder(object):
def __init__(self, redirect_stdout=False):
def __init__(
self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None
):
self.redirect_stdout = redirect_stdout
self._docker_build_args = tuple(docker_build_args or ())
self.active_containers = set()
self._lock = threading.Lock()
self._failed = False
@ -79,8 +83,8 @@ class Builder(object):
stdout = None
# first build a docker image for the build environment
subprocess.check_call(
[
build_args = (
(
"docker",
"build",
"--tag",
@ -89,8 +93,13 @@ class Builder(object):
"distro=" + dist,
"-f",
"docker/Dockerfile-dhvirtualenv",
"docker",
],
)
+ self._docker_build_args
+ ("docker",)
)
subprocess.check_call(
build_args,
stdout=stdout,
stderr=subprocess.STDOUT,
cwd=projdir,
@ -147,9 +156,7 @@ class Builder(object):
self.active_containers.remove(c)
def run_builds(dists, jobs=1, skip_tests=False):
builder = Builder(redirect_stdout=(jobs > 1))
def run_builds(builder, dists, jobs=1, skip_tests=False):
def sig(signum, _frame):
print("Caught SIGINT")
builder.kill_containers()
@ -180,6 +187,11 @@ if __name__ == "__main__":
action="store_true",
help="skip running tests after building",
)
parser.add_argument(
"--docker-build-arg",
action="append",
help="specify an argument to pass to docker build",
)
parser.add_argument(
"--show-dists-json",
action="store_true",
@ -195,4 +207,12 @@ if __name__ == "__main__":
if args.show_dists_json:
print(json.dumps(DISTS))
else:
run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check)
builder = Builder(
redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg
)
run_builds(
builder,
dists=args.dist,
jobs=args.jobs,
skip_tests=args.no_check,
)

2
scripts-dev/complement.sh
View file

@ -65,4 +65,4 @@ if [[ -n "$1" ]]; then
fi
# Run the tests!
go test -v -tags synapse_blacklist,msc2946,msc3083,msc2716,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests

2
synapse/__init__.py
View file

@ -47,7 +47,7 @@ try:
except ImportError:
pass
__version__ = "1.38.0"
__version__ = "1.39.0rc2"
if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
# We import here so that we don't have to install a bunch of deps when

21
synapse/api/auth.py
View file

@ -62,16 +62,14 @@ class Auth:
self.clock = hs.get_clock()
self.store = hs.get_datastore()
self.state = hs.get_state_handler()
self._account_validity_handler = hs.get_account_validity_handler()
self.token_cache = LruCache(
self.token_cache: LruCache[str, Tuple[str, bool]] = LruCache(
10000, "token_cache"
) # type: LruCache[str, Tuple[str, bool]]
)
self._auth_blocking = AuthBlocking(self.hs)
self._account_validity_enabled = (
hs.config.account_validity.account_validity_enabled
)
self._track_appservice_user_ips = hs.config.track_appservice_user_ips
self._macaroon_secret_key = hs.config.macaroon_secret_key
self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
@ -187,12 +185,17 @@ class Auth:
shadow_banned = user_info.shadow_banned
# Deny the request if the user account has expired.
if self._account_validity_enabled and not allow_expired:
if await self.store.is_account_expired(
user_info.user_id, self.clock.time_msec()
if not allow_expired:
if await self._account_validity_handler.is_user_expired(
user_info.user_id
):
# Raise the error if either an account validity module has determined
# the account has expired, or the legacy account validity
# implementation is enabled and determined the account has expired
raise AuthError(
403, "User account has expired", errcode=Codes.EXPIRED_ACCOUNT
403,
"User account has expired",
errcode=Codes.EXPIRED_ACCOUNT,
)
device_id = user_info.device_id

6
synapse/api/constants.py
View file

@ -120,6 +120,7 @@ class EventTypes:
SpaceParent = "m.space.parent"
MSC2716_INSERTION = "org.matrix.msc2716.insertion"
MSC2716_CHUNK = "org.matrix.msc2716.chunk"
MSC2716_MARKER = "org.matrix.msc2716.marker"
@ -190,9 +191,10 @@ class EventContentFields:
# Used on normal messages to indicate they were historically imported after the fact
MSC2716_HISTORICAL = "org.matrix.msc2716.historical"
# For "insertion" events
# For "insertion" events to indicate what the next chunk ID should be in
# order to connect to it
MSC2716_NEXT_CHUNK_ID = "org.matrix.msc2716.next_chunk_id"
# Used on normal message events to indicate where the chunk connects to
# Used on "chunk" events to indicate which insertion event it connects to
MSC2716_CHUNK_ID = "org.matrix.msc2716.chunk_id"
# For "marker" events
MSC2716_MARKER_INSERTION = "org.matrix.msc2716.marker.insertion"

7
synapse/api/errors.py
View file

@ -75,6 +75,9 @@ class Codes:
INVALID_SIGNATURE = "M_INVALID_SIGNATURE"
USER_DEACTIVATED = "M_USER_DEACTIVATED"
BAD_ALIAS = "M_BAD_ALIAS"
# For restricted join rules.
UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN"
UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN"
class CodeMessageException(RuntimeError):
@ -118,7 +121,7 @@ class RedirectException(CodeMessageException):
super().__init__(code=http_code, msg=msg)
self.location = location
self.cookies = [] # type: List[bytes]
self.cookies: List[bytes] = []
class SynapseError(CodeMessageException):
@ -160,7 +163,7 @@ class ProxiedRequestError(SynapseError):
):
super().__init__(code, msg, errcode)
if additional_fields is None:
self._additional_fields = {} # type: Dict
self._additional_fields: Dict = {}
else:
self._additional_fields = dict(additional_fields)

2
synapse/api/filtering.py
View file

@ -289,7 +289,7 @@ class Filter:
room_id = None
ev_type = "m.presence"
contains_url = False
labels = [] # type: List[str]
labels: List[str] = []
else:
sender = event.get("sender", None)
if not sender:

4
synapse/api/ratelimiting.py
View file

@ -46,9 +46,7 @@ class Ratelimiter:
# * How many times an action has occurred since a point in time
# * The point in time
# * The rate_hz of this particular entry. This can vary per request
self.actions = (
OrderedDict()
) # type: OrderedDict[Hashable, Tuple[float, int, float]]
self.actions: OrderedDict[Hashable, Tuple[float, int, float]] = OrderedDict()
async def can_do_action(
self,

44
synapse/api/room_versions.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Dict
from typing import Callable, Dict, Optional
import attr
@ -168,7 +168,7 @@ class RoomVersions:
msc2403_knocking=False,
)
MSC3083 = RoomVersion(
"org.matrix.msc3083",
"org.matrix.msc3083.v2",
RoomDisposition.UNSTABLE,
EventFormatVersions.V3,
StateResolutionVersions.V2,
@ -195,7 +195,7 @@ class RoomVersions:
)
KNOWN_ROOM_VERSIONS = {
KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
v.identifier: v
for v in (
RoomVersions.V1,
@ -208,5 +208,39 @@ KNOWN_ROOM_VERSIONS = {
RoomVersions.MSC3083,
RoomVersions.V7,
)
# Note that we do not include MSC2043 here unless it is enabled in the config.
} # type: Dict[str, RoomVersion]
}
@attr.s(slots=True, frozen=True, auto_attribs=True)
class RoomVersionCapability:
"""An object which describes the unique attributes of a room version."""
identifier: str # the identifier for this capability
preferred_version: Optional[RoomVersion]
support_check_lambda: Callable[[RoomVersion], bool]
MSC3244_CAPABILITIES = {
cap.identifier: {
"preferred": cap.preferred_version.identifier
if cap.preferred_version is not None
else None,
"support": [
v.identifier
for v in KNOWN_ROOM_VERSIONS.values()
if cap.support_check_lambda(v)
],
}
for cap in (
RoomVersionCapability(
"knock",
RoomVersions.V7,
lambda room_version: room_version.msc2403_knocking,
),
RoomVersionCapability(
"restricted",
None,
lambda room_version: room_version.msc3083_join_rules,
),
)
}

2
synapse/app/_base.py
View file

@ -38,6 +38,7 @@ from synapse.app.phone_stats_home import start_phone_stats_home
from synapse.config.homeserver import HomeServerConfig
from synapse.crypto import context_factory
from synapse.events.spamcheck import load_legacy_spam_checkers
from synapse.events.third_party_rules import load_legacy_third_party_event_rules
from synapse.logging.context import PreserveLoggingContext
from synapse.metrics.background_process_metrics import wrap_as_background_process
from synapse.metrics.jemalloc import setup_jemalloc_stats
@ -368,6 +369,7 @@ async def start(hs: "HomeServer"):
module(config=config, api=module_api)
load_legacy_spam_checkers(hs)
load_legacy_third_party_event_rules(hs)
# If we've configured an expiry time for caches, start the background job now.
setup_expire_lru_cache_entries(hs)

8
synapse/app/generic_worker.py
View file

@ -270,7 +270,7 @@ class GenericWorkerServer(HomeServer):
site_tag = port
# We always include a health resource.
resources = {"/health": HealthResource()} # type: Dict[str, IResource]
resources: Dict[str, IResource] = {"/health": HealthResource()}
for res in listener_config.http_options.resources:
for name in res.names:
@ -395,10 +395,8 @@ class GenericWorkerServer(HomeServer):
elif listener.type == "metrics":
if not self.config.enable_metrics:
logger.warning(
(
"Metrics listener configured, but "
"enable_metrics is not True!"
)
"Metrics listener configured, but "
"enable_metrics is not True!"
)
else:
_base.listen_metrics(listener.bind_addresses, listener.port)

6
synapse/app/homeserver.py
View file

@ -305,10 +305,8 @@ class SynapseHomeServer(HomeServer):
elif listener.type == "metrics":
if not self.config.enable_metrics:
logger.warning(
(
"Metrics listener configured, but "
"enable_metrics is not True!"
)
"Metrics listener configured, but "
"enable_metrics is not True!"
)
else:
_base.listen_metrics(listener.bind_addresses, listener.port)

36
synapse/app/phone_stats_home.py
View file

@ -71,6 +71,8 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
# General statistics
#
store = hs.get_datastore()
stats["homeserver"] = hs.config.server_name
stats["server_context"] = hs.config.server_context
stats["timestamp"] = now
@ -79,34 +81,38 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
stats["python_version"] = "{}.{}.{}".format(
version.major, version.minor, version.micro
)
stats["total_users"] = await hs.get_datastore().count_all_users()
stats["total_users"] = await store.count_all_users()
total_nonbridged_users = await hs.get_datastore().count_nonbridged_users()
total_nonbridged_users = await store.count_nonbridged_users()
stats["total_nonbridged_users"] = total_nonbridged_users
daily_user_type_results = await hs.get_datastore().count_daily_user_type()
daily_user_type_results = await store.count_daily_user_type()
for name, count in daily_user_type_results.items():
stats["daily_user_type_" + name] = count
room_count = await hs.get_datastore().get_room_count()
room_count = await store.get_room_count()
stats["total_room_count"] = room_count
stats["daily_active_users"] = await hs.get_datastore().count_daily_users()
stats["monthly_active_users"] = await hs.get_datastore().count_monthly_users()
daily_active_e2ee_rooms = await hs.get_datastore().count_daily_active_e2ee_rooms()
stats["daily_active_users"] = await store.count_daily_users()
stats["monthly_active_users"] = await store.count_monthly_users()
daily_active_e2ee_rooms = await store.count_daily_active_e2ee_rooms()
stats["daily_active_e2ee_rooms"] = daily_active_e2ee_rooms
stats["daily_e2ee_messages"] = await hs.get_datastore().count_daily_e2ee_messages()
daily_sent_e2ee_messages = await hs.get_datastore().count_daily_sent_e2ee_messages()
stats["daily_e2ee_messages"] = await store.count_daily_e2ee_messages()
daily_sent_e2ee_messages = await store.count_daily_sent_e2ee_messages()
stats["daily_sent_e2ee_messages"] = daily_sent_e2ee_messages
stats["daily_active_rooms"] = await hs.get_datastore().count_daily_active_rooms()
stats["daily_messages"] = await hs.get_datastore().count_daily_messages()
daily_sent_messages = await hs.get_datastore().count_daily_sent_messages()
stats["daily_active_rooms"] = await store.count_daily_active_rooms()
stats["daily_messages"] = await store.count_daily_messages()
daily_sent_messages = await store.count_daily_sent_messages()
stats["daily_sent_messages"] = daily_sent_messages
r30_results = await hs.get_datastore().count_r30_users()
r30_results = await store.count_r30_users()
for name, count in r30_results.items():
stats["r30_users_" + name] = count
r30v2_results = await store.count_r30_users()
for name, count in r30v2_results.items():
stats["r30v2_users_" + name] = count
stats["cache_factor"] = hs.config.caches.global_factor
stats["event_cache_size"] = hs.config.caches.event_cache_size
@ -115,8 +121,8 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
#
# This only reports info about the *main* database.
stats["database_engine"] = hs.get_datastore().db_pool.engine.module.__name__
stats["database_server_version"] = hs.get_datastore().db_pool.engine.server_version
stats["database_engine"] = store.db_pool.engine.module.__name__
stats["database_server_version"] = store.db_pool.engine.server_version
#
# Logging configuration

4
synapse/appservice/api.py
View file

@ -88,9 +88,9 @@ class ApplicationServiceApi(SimpleHttpClient):
super().__init__(hs)
self.clock = hs.get_clock()
self.protocol_meta_cache = ResponseCache(
self.protocol_meta_cache: ResponseCache[Tuple[str, str]] = ResponseCache(
hs.get_clock(), "as_protocol_meta", timeout_ms=HOUR_IN_MS
) # type: ResponseCache[Tuple[str, str]]
)
async def query_user(self, service, user_id):
if service.url is None:

102
synapse/config/account_validity.py
View file

@ -18,6 +18,21 @@ class AccountValidityConfig(Config):
section = "account_validity"
def read_config(self, config, **kwargs):
"""Parses the old account validity config. The config format looks like this:
account_validity:
enabled: true
period: 6w
renew_at: 1w
renew_email_subject: "Renew your %(app)s account"
template_dir: "res/templates"
account_renewed_html_path: "account_renewed.html"
invalid_token_html_path: "invalid_token.html"
We expect admins to use modules for this feature (which is why it doesn't appear
in the sample config file), but we want to keep support for it around for a bit
for backwards compatibility.
"""
account_validity_config = config.get("account_validity") or {}
self.account_validity_enabled = account_validity_config.get("enabled", False)
self.account_validity_renew_by_email_enabled = (
@ -75,90 +90,3 @@ class AccountValidityConfig(Config):
],
account_validity_template_dir,
)
def generate_config_section(self, **kwargs):
return """\
## Account Validity ##
# Optional account validity configuration. This allows for accounts to be denied
# any request after a given period.
#
# Once this feature is enabled, Synapse will look for registered users without an
# expiration date at startup and will add one to every account it found using the
# current settings at that time.
# This means that, if a validity period is set, and Synapse is restarted (it will
# then derive an expiration date from the current validity period), and some time
# after that the validity period changes and Synapse is restarted, the users'
# expiration dates won't be updated unless their account is manually renewed. This
# date will be randomly selected within a range [now + period - d ; now + period],
# where d is equal to 10% of the validity period.
#
account_validity:
# The account validity feature is disabled by default. Uncomment the
# following line to enable it.
#
#enabled: true
# The period after which an account is valid after its registration. When
# renewing the account, its validity period will be extended by this amount
# of time. This parameter is required when using the account validity
# feature.
#
#period: 6w
# The amount of time before an account's expiry date at which Synapse will
# send an email to the account's email address with a renewal link. By
# default, no such emails are sent.
#
# If you enable this setting, you will also need to fill out the 'email' and
# 'public_baseurl' configuration sections.
#
#renew_at: 1w
# The subject of the email sent out with the renewal link. '%(app)s' can be
# used as a placeholder for the 'app_name' parameter from the 'email'
# section.
#
# Note that the placeholder must be written '%(app)s', including the
# trailing 's'.
#
# If this is not set, a default value is used.
#
#renew_email_subject: "Renew your %(app)s account"
# Directory in which Synapse will try to find templates for the HTML files to
# serve to the user when trying to renew an account. If not set, default
# templates from within the Synapse package will be used.
#
# The currently available templates are:
#
# * account_renewed.html: Displayed to the user after they have successfully
# renewed their account.
#
# * account_previously_renewed.html: Displayed to the user if they attempt to
# renew their account with a token that is valid, but that has already
# been used. In this case the account is not renewed again.
#
# * invalid_token.html: Displayed to the user when they try to renew an account
# with an unknown or invalid renewal token.
#
# See https://github.com/matrix-org/synapse/tree/master/synapse/res/templates for
# default template contents.
#
# The file name of some of these templates can be configured below for legacy
# reasons.
#
#template_dir: "res/templates"
# A custom file name for the 'account_renewed.html' template.
#
# If not set, the file is assumed to be named "account_renewed.html".
#
#account_renewed_html_path: "account_renewed.html"
# A custom file name for the 'invalid_token.html' template.
#
# If not set, the file is assumed to be named "invalid_token.html".
#
#invalid_token_html_path: "invalid_token.html"
"""

6
synapse/config/appservice.py
View file

@ -57,14 +57,14 @@ def load_appservices(hostname, config_files):
return []
# Dicts of value -> filename
seen_as_tokens = {} # type: Dict[str, str]
seen_ids = {} # type: Dict[str, str]
seen_as_tokens: Dict[str, str] = {}
seen_ids: Dict[str, str] = {}
appservices = []
for config_file in config_files:
try:
with open(config_file, "r") as f:
with open(config_file) as f:
appservice = _load_appservice(hostname, yaml.safe_load(f), config_file)
if appservice.id in seen_ids:
raise ConfigError(

Some files were not shown because too many files have changed in this diff Show more