Merge branch 'develop' of github.com:matrix-org/synapse into matrix-org-hotfixes

This commit is contained in:
Erik Johnston 2019-03-05 14:41:13 +00:00
commit dc510e0e43
143 changed files with 3114 additions and 1423 deletions

13
.buildkite/.env Normal file
View file

@ -0,0 +1,13 @@
CI
BUILDKITE
BUILDKITE_BUILD_NUMBER
BUILDKITE_BRANCH
BUILDKITE_BUILD_NUMBER
BUILDKITE_JOB_ID
BUILDKITE_BUILD_URL
BUILDKITE_PROJECT_SLUG
BUILDKITE_COMMIT
BUILDKITE_PULL_REQUEST
BUILDKITE_TAG
CODECOV_TOKEN
TRIAL_FLAGS

View file

@ -0,0 +1,21 @@
version: '3.1'
services:
postgres:
image: postgres:9.4
environment:
POSTGRES_PASSWORD: postgres
testenv:
image: python:2.7
depends_on:
- postgres
env_file: .env
environment:
SYNAPSE_POSTGRES_HOST: postgres
SYNAPSE_POSTGRES_USER: postgres
SYNAPSE_POSTGRES_PASSWORD: postgres
working_dir: /app
volumes:
- ..:/app

View file

@ -0,0 +1,21 @@
version: '3.1'
services:
postgres:
image: postgres:9.5
environment:
POSTGRES_PASSWORD: postgres
testenv:
image: python:2.7
depends_on:
- postgres
env_file: .env
environment:
SYNAPSE_POSTGRES_HOST: postgres
SYNAPSE_POSTGRES_USER: postgres
SYNAPSE_POSTGRES_PASSWORD: postgres
working_dir: /app
volumes:
- ..:/app

View file

@ -0,0 +1,21 @@
version: '3.1'
services:
postgres:
image: postgres:9.4
environment:
POSTGRES_PASSWORD: postgres
testenv:
image: python:3.5
depends_on:
- postgres
env_file: .env
environment:
SYNAPSE_POSTGRES_HOST: postgres
SYNAPSE_POSTGRES_USER: postgres
SYNAPSE_POSTGRES_PASSWORD: postgres
working_dir: /app
volumes:
- ..:/app

View file

@ -0,0 +1,21 @@
version: '3.1'
services:
postgres:
image: postgres:9.5
environment:
POSTGRES_PASSWORD: postgres
testenv:
image: python:3.5
depends_on:
- postgres
env_file: .env
environment:
SYNAPSE_POSTGRES_HOST: postgres
SYNAPSE_POSTGRES_USER: postgres
SYNAPSE_POSTGRES_PASSWORD: postgres
working_dir: /app
volumes:
- ..:/app

View file

@ -0,0 +1,21 @@
version: '3.1'
services:
postgres:
image: postgres:11
environment:
POSTGRES_PASSWORD: postgres
testenv:
image: python:3.7
depends_on:
- postgres
env_file: .env
environment:
SYNAPSE_POSTGRES_HOST: postgres
SYNAPSE_POSTGRES_USER: postgres
SYNAPSE_POSTGRES_PASSWORD: postgres
working_dir: /app
volumes:
- ..:/app

View file

@ -0,0 +1,21 @@
version: '3.1'
services:
postgres:
image: postgres:9.5
environment:
POSTGRES_PASSWORD: postgres
testenv:
image: python:3.7
depends_on:
- postgres
env_file: .env
environment:
SYNAPSE_POSTGRES_HOST: postgres
SYNAPSE_POSTGRES_USER: postgres
SYNAPSE_POSTGRES_PASSWORD: postgres
working_dir: /app
volumes:
- ..:/app

157
.buildkite/pipeline.yml Normal file
View file

@ -0,0 +1,157 @@
env:
CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
steps:
- command:
- "python -m pip install tox"
- "tox -e pep8"
label: "\U0001F9F9 PEP-8"
plugins:
- docker#v3.0.1:
image: "python:3.6"
- command:
- "python -m pip install tox"
- "tox -e packaging"
label: "\U0001F9F9 packaging"
plugins:
- docker#v3.0.1:
image: "python:3.6"
- command:
- "python -m pip install tox"
- "tox -e check_isort"
label: "\U0001F9F9 isort"
plugins:
- docker#v3.0.1:
image: "python:3.6"
- command:
- "python -m pip install tox"
- "scripts-dev/check-newsfragment"
label: ":newspaper: Newsfile"
branches: "!master !develop !release-*"
plugins:
- docker#v3.0.1:
image: "python:3.6"
propagate-environment: true
- wait
- command:
- "python -m pip install tox"
- "tox -e check-sampleconfig"
label: "\U0001F9F9 check-sample-config"
plugins:
- docker#v3.0.1:
image: "python:3.6"
- command:
- "python -m pip install tox"
- "tox -e py27,codecov"
label: ":python: 2.7 / SQLite"
env:
TRIAL_FLAGS: "-j 2"
plugins:
- docker#v3.0.1:
image: "python:2.7"
propagate-environment: true
- command:
- "python -m pip install tox"
- "tox -e py35,codecov"
label: ":python: 3.5 / SQLite"
env:
TRIAL_FLAGS: "-j 2"
plugins:
- docker#v3.0.1:
image: "python:3.5"
propagate-environment: true
- command:
- "python -m pip install tox"
- "tox -e py36,codecov"
label: ":python: 3.6 / SQLite"
env:
TRIAL_FLAGS: "-j 2"
plugins:
- docker#v3.0.1:
image: "python:3.6"
propagate-environment: true
- command:
- "python -m pip install tox"
- "tox -e py37,codecov"
label: ":python: 3.7 / SQLite"
env:
TRIAL_FLAGS: "-j 2"
plugins:
- docker#v3.0.1:
image: "python:3.7"
propagate-environment: true
- label: ":python: 2.7 / :postgres: 9.4"
env:
TRIAL_FLAGS: "-j 4"
command:
- "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
plugins:
- docker-compose#v2.1.0:
run: testenv
config:
- .buildkite/docker-compose.py27.pg94.yaml
- label: ":python: 2.7 / :postgres: 9.5"
env:
TRIAL_FLAGS: "-j 4"
command:
- "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
plugins:
- docker-compose#v2.1.0:
run: testenv
config:
- .buildkite/docker-compose.py27.pg95.yaml
- label: ":python: 3.5 / :postgres: 9.4"
env:
TRIAL_FLAGS: "-j 4"
command:
- "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
plugins:
- docker-compose#v2.1.0:
run: testenv
config:
- .buildkite/docker-compose.py35.pg94.yaml
- label: ":python: 3.5 / :postgres: 9.5"
env:
TRIAL_FLAGS: "-j 4"
command:
- "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
plugins:
- docker-compose#v2.1.0:
run: testenv
config:
- .buildkite/docker-compose.py35.pg95.yaml
- label: ":python: 3.7 / :postgres: 9.5"
env:
TRIAL_FLAGS: "-j 4"
command:
- "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
plugins:
- docker-compose#v2.1.0:
run: testenv
config:
- .buildkite/docker-compose.py37.pg95.yaml
- label: ":python: 3.7 / :postgres: 11"
env:
TRIAL_FLAGS: "-j 4"
command:
- "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
plugins:
- docker-compose#v2.1.0:
run: testenv
config:
- .buildkite/docker-compose.py37.pg11.yaml

View file

@ -1,97 +0,0 @@
dist: xenial
language: python
cache:
directories:
# we only bother to cache the wheels; parts of the http cache get
# invalidated every build (because they get served with a max-age of 600
# seconds), which means that we end up re-uploading the whole cache for
# every build, which is time-consuming In any case, it's not obvious that
# downloading the cache from S3 would be much faster than downloading the
# originals from pypi.
#
- $HOME/.cache/pip/wheels
# don't clone the whole repo history, one commit will do
git:
depth: 1
# only build branches we care about (PRs are built seperately)
branches:
only:
- master
- develop
- /^release-v/
- rav/pg95
# When running the tox environments that call Twisted Trial, we can pass the -j
# flag to run the tests concurrently. We set this to 2 for CPU bound tests
# (SQLite) and 4 for I/O bound tests (PostgreSQL).
matrix:
fast_finish: true
include:
- name: "pep8"
python: 3.6
env: TOX_ENV="pep8,check_isort,packaging"
- name: "py2.7 / sqlite"
python: 2.7
env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2"
- name: "py2.7 / sqlite / olddeps"
python: 2.7
env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
- name: "py2.7 / postgres9.5"
python: 2.7
addons:
postgresql: "9.5"
env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4"
services:
- postgresql
- name: "py3.5 / sqlite"
python: 3.5
env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2"
- name: "py3.7 / sqlite"
python: 3.7
env: TOX_ENV=py37,codecov TRIAL_FLAGS="-j 2"
- name: "py3.7 / postgres9.4"
python: 3.7
addons:
postgresql: "9.4"
env: TOX_ENV=py37-postgres TRIAL_FLAGS="-j 4"
services:
- postgresql
- name: "py3.7 / postgres9.5"
python: 3.7
addons:
postgresql: "9.5"
env: TOX_ENV=py37-postgres,codecov TRIAL_FLAGS="-j 4"
services:
- postgresql
- # we only need to check for the newsfragment if it's a PR build
if: type = pull_request
name: "check-newsfragment"
python: 3.6
script: scripts-dev/check-newsfragment
install:
# this just logs the postgres version we will be testing against (if any)
- psql -At -U postgres -c 'select version();' || true
- pip install tox
# if we don't have python3.6 in this environment, travis unhelpfully gives us
# a `python3.6` on our path which does nothing but spit out a warning. Tox
# tries to run it (even if we're not running a py36 env), so the build logs
# then have warnings which look like errors. To reduce the noise, remove the
# non-functional python3.6.
- ( ! command -v python3.6 || python3.6 --version ) &>/dev/null || rm -f $(command -v python3.6)
script:
- tox -e $TOX_ENV

View file

@ -1,3 +1,62 @@
Synapse 0.99.2 (2019-03-01)
===========================
Features
--------
- Added an HAProxy example in the reverse proxy documentation. Contributed by Benoît S. (“Benpro”). ([\#4541](https://github.com/matrix-org/synapse/issues/4541))
- Add basic optional sentry integration. ([\#4632](https://github.com/matrix-org/synapse/issues/4632), [\#4694](https://github.com/matrix-org/synapse/issues/4694))
- Transfer bans on room upgrade. ([\#4642](https://github.com/matrix-org/synapse/issues/4642))
- Add configurable room list publishing rules. ([\#4647](https://github.com/matrix-org/synapse/issues/4647))
- Support .well-known delegation when issuing certificates through ACME. ([\#4652](https://github.com/matrix-org/synapse/issues/4652))
- Allow registration and login to be handled by a worker instance. ([\#4666](https://github.com/matrix-org/synapse/issues/4666), [\#4670](https://github.com/matrix-org/synapse/issues/4670), [\#4682](https://github.com/matrix-org/synapse/issues/4682))
- Reduce the overhead of creating outbound federation connections over TLS by caching the TLS client options. ([\#4674](https://github.com/matrix-org/synapse/issues/4674))
- Add prometheus metrics for number of outgoing EDUs, by type. ([\#4695](https://github.com/matrix-org/synapse/issues/4695))
- Return correct error code when inviting a remote user to a room whose homeserver does not support the room version. ([\#4721](https://github.com/matrix-org/synapse/issues/4721))
- Prevent showing rooms to other servers that were set to not federate. ([\#4746](https://github.com/matrix-org/synapse/issues/4746))
Bugfixes
--------
- Fix possible exception when paginating. ([\#4263](https://github.com/matrix-org/synapse/issues/4263))
- The dependency checker now correctly reports a version mismatch for optional
dependencies, instead of reporting the dependency missing. ([\#4450](https://github.com/matrix-org/synapse/issues/4450))
- Set CORS headers on .well-known requests. ([\#4651](https://github.com/matrix-org/synapse/issues/4651))
- Fix kicking guest users on guest access revocation in worker mode. ([\#4667](https://github.com/matrix-org/synapse/issues/4667))
- Fix an issue in the database migration script where the
`e2e_room_keys.is_verified` column wasn't considered as
a boolean. ([\#4680](https://github.com/matrix-org/synapse/issues/4680))
- Fix TaskStopped exceptions in logs when outbound requests time out. ([\#4690](https://github.com/matrix-org/synapse/issues/4690))
- Fix ACME config for python 2. ([\#4717](https://github.com/matrix-org/synapse/issues/4717))
- Fix paginating over federation persisting incorrect state. ([\#4718](https://github.com/matrix-org/synapse/issues/4718))
Internal Changes
----------------
- Run `black` to reformat user directory code. ([\#4635](https://github.com/matrix-org/synapse/issues/4635))
- Reduce number of exceptions we log. ([\#4643](https://github.com/matrix-org/synapse/issues/4643), [\#4668](https://github.com/matrix-org/synapse/issues/4668))
- Introduce upsert batching functionality in the database layer. ([\#4644](https://github.com/matrix-org/synapse/issues/4644))
- Fix various spelling mistakes. ([\#4657](https://github.com/matrix-org/synapse/issues/4657))
- Cleanup request exception logging. ([\#4669](https://github.com/matrix-org/synapse/issues/4669), [\#4737](https://github.com/matrix-org/synapse/issues/4737), [\#4738](https://github.com/matrix-org/synapse/issues/4738))
- Improve replication performance by reducing cache invalidation traffic. ([\#4671](https://github.com/matrix-org/synapse/issues/4671), [\#4715](https://github.com/matrix-org/synapse/issues/4715), [\#4748](https://github.com/matrix-org/synapse/issues/4748))
- Test against Postgres 9.5 as well as 9.4. ([\#4676](https://github.com/matrix-org/synapse/issues/4676))
- Run unit tests against python 3.7. ([\#4677](https://github.com/matrix-org/synapse/issues/4677))
- Attempt to clarify installation instructions/config. ([\#4681](https://github.com/matrix-org/synapse/issues/4681))
- Clean up gitignores. ([\#4688](https://github.com/matrix-org/synapse/issues/4688))
- Minor tweaks to acme docs. ([\#4689](https://github.com/matrix-org/synapse/issues/4689))
- Improve the logging in the pusher process. ([\#4691](https://github.com/matrix-org/synapse/issues/4691))
- Better checks on newsfragments. ([\#4698](https://github.com/matrix-org/synapse/issues/4698), [\#4750](https://github.com/matrix-org/synapse/issues/4750))
- Avoid some redundant work when processing read receipts. ([\#4706](https://github.com/matrix-org/synapse/issues/4706))
- Run `push_receipts_to_remotes` as background job. ([\#4707](https://github.com/matrix-org/synapse/issues/4707))
- Add prometheus metrics for number of badge update pushes. ([\#4709](https://github.com/matrix-org/synapse/issues/4709))
- Reduce pusher logging on startup ([\#4716](https://github.com/matrix-org/synapse/issues/4716))
- Don't log exceptions when failing to fetch remote server keys. ([\#4722](https://github.com/matrix-org/synapse/issues/4722))
- Correctly proxy exception in frontend_proxy worker. ([\#4723](https://github.com/matrix-org/synapse/issues/4723))
- Add database version to phonehome stats. ([\#4753](https://github.com/matrix-org/synapse/issues/4753))
Synapse 0.99.1.1 (2019-02-14)
=============================

View file

@ -39,6 +39,7 @@ prune .circleci
prune .coveragerc
prune debian
prune .codecov.yml
prune .buildkite
exclude jenkins*
recursive-exclude jenkins *.sh

View file

@ -117,9 +117,9 @@ recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.rst>`_.)
Once ``enable_registration`` is set to ``true``, it is possible to register a
user via `riot.im <https://riot.im/app/#/register>`_ or other Matrix clients.
Your new user name will be formed partly from the ``server_name`` (see
`Configuring synapse`_), and partly from a localpart you specify when you
create the account. Your name will take the form of::
Your new user name will be formed partly from the ``server_name``, and partly
from a localpart you specify when you create the account. Your name will take
the form of::
@localpart:my.domain.name

View file

@ -1 +0,0 @@
Prevent crash on pagination.

View file

@ -1,2 +0,0 @@
The dependency checker now correctly reports a version mismatch for optional
dependencies, instead of reporting the dependency missing.

View file

@ -1 +0,0 @@
Added an HAProxy example in the reverse proxy documentation. Contributed by Benoît S. (Benpro).

View file

@ -1 +0,0 @@
Add basic optional sentry integration

View file

@ -1 +0,0 @@
Run `black` to reformat user directory code.

View file

@ -1 +0,0 @@
Transfer bans on room upgrade.

View file

@ -1 +0,0 @@
Reduce number of exceptions we log

View file

@ -1 +0,0 @@
Introduce upsert batching functionality in the database layer.

View file

@ -1 +0,0 @@
Add configurable room list publishing rules

View file

@ -1 +0,0 @@
Set CORS headers on .well-known requests

View file

@ -1 +0,0 @@
Support .well-known delegation when issuing certificates through ACME.

View file

@ -1 +0,0 @@
Fix various spelling mistakes.

View file

@ -1 +0,0 @@
Allow registration and login to be handled by a worker instance.

View file

@ -1 +0,0 @@
Fix kicking guest users on guest access revocation in worker mode.

View file

@ -1 +0,0 @@
Reduce number of exceptions we log

View file

@ -1 +0,0 @@
Cleanup request exception logging.

View file

@ -1 +0,0 @@
Allow registration and login to be handled by a worker instance.

View file

@ -1 +0,0 @@
Improve replication performance by reducing cache invalidation traffic.

View file

@ -1 +0,0 @@
Reduce the overhead of creating outbound federation connections over TLS by caching the TLS client options.

View file

@ -1 +0,0 @@
Test against Postgres 9.5 as well as 9.4

View file

@ -1 +0,0 @@
Run unit tests against python 3.7.

View file

@ -1,3 +0,0 @@
Fix an issue in the database migration script where the
`e2e_room_keys.is_verified` column wasn't considered as
a boolean

View file

@ -1 +0,0 @@
Attempt to clarify installation instructions/config

View file

@ -1 +0,0 @@
Allow registration and login to be handled by a worker instance.

View file

@ -1 +0,0 @@
Clean up gitignores

View file

@ -1 +0,0 @@
Minor tweaks to acme docs.

View file

@ -1 +0,0 @@
Fix TaskStopped exceptions in logs when outbound requests time out.

View file

@ -1 +0,0 @@
Improve the logging in the pusher process.

View file

@ -1 +0,0 @@
Add basic optional sentry integration

View file

@ -1 +0,0 @@
Add prometheus metrics for number of outgoing EDUs, by type.

View file

@ -1 +0,0 @@
Better checks on newsfragments.

1
changelog.d/4699.bugfix Normal file
View file

@ -0,0 +1 @@
Fix attempting to paginate in rooms where server cannot see any events, to avoid unnecessarily pulling in lots of redacted events.

View file

@ -1 +0,0 @@
Avoid some redundant work when processing read receipts

View file

@ -1 +0,0 @@
Run push_receipts_to_remotes as background job.

View file

@ -1 +0,0 @@
Add prometheus metrics for number of badge update pushes.

View file

@ -1 +0,0 @@
Improve replication performance by reducing cache invalidation traffic.

View file

@ -1 +0,0 @@
Reduce pusher logging on startup

View file

@ -1 +0,0 @@
Fix ACME config for python 2.

View file

@ -1 +0,0 @@
Fix paginating over federation persisting incorrect state.

View file

@ -1 +0,0 @@
Return correct error code when inviting a remote user to a room whose homeserver does not support the room version.

View file

@ -1 +0,0 @@
Don't log exceptions when failing to fetch remote server keys

View file

@ -1 +0,0 @@
Correctly proxy exception in frontend_proxy worker

1
changelog.d/4735.feature Normal file
View file

@ -0,0 +1 @@
Add configurable rate limiting to the /register endpoint.

View file

@ -1 +0,0 @@
Cleanup request exception logging.

View file

@ -1 +0,0 @@
Cleanup request exception logging.

1
changelog.d/4740.bugfix Normal file
View file

@ -0,0 +1 @@
'event_id' is now a required parameter in federated state requests, as per the matrix spec.

View file

@ -1 +0,0 @@
Prevent showing rooms to other servers that were set to not federate.

View file

@ -1 +0,0 @@
Improve replication performance by reducing cache invalidation traffic.

1
changelog.d/4749.bugfix Normal file
View file

@ -0,0 +1 @@
Fix tightloop over connecting to replication server.

View file

@ -1 +0,0 @@
Better checks on newsfragments.

1
changelog.d/4752.misc Normal file
View file

@ -0,0 +1 @@
Change from TravisCI to Buildkite for CI.

View file

@ -1 +0,0 @@
Add database version to phonehome stats.

1
changelog.d/4757.feature Normal file
View file

@ -0,0 +1 @@
Move server key queries to federation reader.

1
changelog.d/4757.misc Normal file
View file

@ -0,0 +1 @@
When presence is disabled don't send over replication.

1
changelog.d/4759.feature Normal file
View file

@ -0,0 +1 @@
Add support for /account/3pid REST endpoint to client_reader worker.

1
changelog.d/4763.bugfix Normal file
View file

@ -0,0 +1 @@
Fix parsing of Content-Disposition headers on remote media requests and URL previews.

1
changelog.d/4765.misc Normal file
View file

@ -0,0 +1 @@
Minor docstring fixes for MatrixFederationAgent.

1
changelog.d/4770.misc Normal file
View file

@ -0,0 +1 @@
Optimise EDU transmission for the federation_sender worker.

1
changelog.d/4771.misc Normal file
View file

@ -0,0 +1 @@
Update test_typing to use HomeserverTestCase.

1
changelog.d/4776.bugfix Normal file
View file

@ -0,0 +1 @@
Fix incorrect log about not persisting duplicate state event.

1
changelog.d/4790.bugfix Normal file
View file

@ -0,0 +1 @@
Fix v4v6 option in HAProxy example config. Contributed by Flakebi.

1
changelog.d/4791.feature Normal file
View file

@ -0,0 +1 @@
Include a default configuration file in the 'docs' directory.

1
changelog.d/4794.misc Normal file
View file

@ -0,0 +1 @@
Removed unnecessary $ from some federation endpoint path regexes.

1
changelog.d/4795.misc Normal file
View file

@ -0,0 +1 @@
Remove link to deleted title in README.

1
changelog.d/4796.feature Normal file
View file

@ -0,0 +1 @@
Add support for /keys/query and /keys/changes REST endpoints to client_reader worker.

1
changelog.d/4797.misc Normal file
View file

@ -0,0 +1 @@
Clean up read-receipt handling.

1
changelog.d/4798.misc Normal file
View file

@ -0,0 +1 @@
Add some debug about processing read receipts.

1
changelog.d/4799.misc Normal file
View file

@ -0,0 +1 @@
Clean up some replication code.

5
debian/changelog vendored
View file

@ -1,8 +1,9 @@
matrix-synapse-py3 (0.99.2) UNRELEASED; urgency=medium
matrix-synapse-py3 (0.99.2) stable; urgency=medium
* Fix overwriting of config settings on upgrade.
* New synapse release 0.99.2.
-- Synapse Packaging team <packages@matrix.org> Wed, 20 Feb 2019 17:11:25 +0000
-- Synapse Packaging team <packages@matrix.org> Fri, 01 Mar 2019 10:55:08 +0000
matrix-synapse-py3 (0.99.1.1) stable; urgency=medium

View file

@ -0,0 +1,7 @@
# This file is a reference to the configuration options which can be set in
# homeserver.yaml.
#
# Note that it is not quite ready to be used as-is. If you are starting from
# scratch, it is easier to generate the config files following the instructions
# in INSTALL.md.

View file

@ -88,18 +88,16 @@ Let's assume that we expect clients to connect to our server at
* HAProxy::
frontend https
bind 0.0.0.0:443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
bind :::443 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
bind :::443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
# Matrix client traffic
acl matrix hdr(host) -i matrix.example.com
use_backend matrix if matrix
frontend matrix-federation
bind 0.0.0.0:8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
bind :::8448 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
bind :::8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
default_backend matrix
backend matrix
server matrix 127.0.0.1:8008

1052
docs/sample_config.yaml Normal file

File diff suppressed because it is too large Load diff

View file

@ -188,7 +188,9 @@ RDATA (S)
A single update in a stream
POSITION (S)
The position of the stream has been updated
The position of the stream has been updated. Sent to the client after all
missing updates for a stream have been sent to the client and they're now
up to date.
ERROR (S, C)
There was an error

View file

@ -182,6 +182,7 @@ endpoints matching the following regular expressions::
^/_matrix/federation/v1/event_auth/
^/_matrix/federation/v1/exchange_third_party_invite/
^/_matrix/federation/v1/send/
^/_matrix/key/v2/query
The above endpoints should all be routed to the federation_reader worker by the
reverse-proxy configuration.
@ -223,6 +224,9 @@ following regular expressions::
^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$
^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$
^/_matrix/client/(api/v1|r0|unstable)/login$
^/_matrix/client/(api/v1|r0|unstable)/account/3pid$
^/_matrix/client/(api/v1|r0|unstable)/keys/query$
^/_matrix/client/(api/v1|r0|unstable)/keys/changes$
Additionally, the following REST endpoints can be handled, but all requests must
be routed to the same instance::

View file

@ -0,0 +1,18 @@
#!/bin/bash
#
# Update/check the docs/sample_config.yaml
set -e
cd `dirname $0`/..
SAMPLE_CONFIG="docs/sample_config.yaml"
if [ "$1" == "--check" ]; then
diff -u "$SAMPLE_CONFIG" <(./scripts/generate_config --header-file docs/.sample_config_header.yaml) >/dev/null || {
echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
exit 1
}
else
./scripts/generate_config --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG"
fi

View file

@ -1,6 +1,7 @@
#!/usr/bin/env python
import argparse
import shutil
import sys
from synapse.config.homeserver import HomeServerConfig
@ -50,6 +51,13 @@ if __name__ == "__main__":
help="File to write the configuration to. Default: stdout",
)
parser.add_argument(
"--header-file",
type=argparse.FileType('r'),
help="File from which to read a header, which will be printed before the "
"generated config.",
)
args = parser.parse_args()
report_stats = args.report_stats
@ -64,4 +72,7 @@ if __name__ == "__main__":
report_stats=report_stats,
)
if args.header_file:
shutil.copyfileobj(args.header_file, args.output_file)
args.output_file.write(conf)

View file

@ -27,4 +27,4 @@ try:
except ImportError:
pass
__version__ = "0.99.1.1"
__version__ = "0.99.2"

View file

@ -23,12 +23,13 @@ class Ratelimiter(object):
def __init__(self):
self.message_counts = collections.OrderedDict()
def send_message(self, user_id, time_now_s, msg_rate_hz, burst_count, update=True):
"""Can the user send a message?
def can_do_action(self, key, time_now_s, rate_hz, burst_count, update=True):
"""Can the entity (e.g. user or IP address) perform the action?
Args:
user_id: The user sending a message.
key: The key we should use when rate limiting. Can be a user ID
(when sending events), an IP address, etc.
time_now_s: The time now.
msg_rate_hz: The long term number of messages a user can send in a
rate_hz: The long term number of messages a user can send in a
second.
burst_count: How many messages the user can send before being
limited.
@ -41,10 +42,10 @@ class Ratelimiter(object):
"""
self.prune_message_counts(time_now_s)
message_count, time_start, _ignored = self.message_counts.get(
user_id, (0., time_now_s, None),
key, (0., time_now_s, None),
)
time_delta = time_now_s - time_start
sent_count = message_count - time_delta * msg_rate_hz
sent_count = message_count - time_delta * rate_hz
if sent_count < 0:
allowed = True
time_start = time_now_s
@ -56,13 +57,13 @@ class Ratelimiter(object):
message_count += 1
if update:
self.message_counts[user_id] = (
message_count, time_start, msg_rate_hz
self.message_counts[key] = (
message_count, time_start, rate_hz
)
if msg_rate_hz > 0:
if rate_hz > 0:
time_allowed = (
time_start + (message_count - burst_count + 1) / msg_rate_hz
time_start + (message_count - burst_count + 1) / rate_hz
)
if time_allowed < time_now_s:
time_allowed = time_now_s
@ -72,12 +73,12 @@ class Ratelimiter(object):
return allowed, time_allowed
def prune_message_counts(self, time_now_s):
for user_id in list(self.message_counts.keys()):
message_count, time_start, msg_rate_hz = (
self.message_counts[user_id]
for key in list(self.message_counts.keys()):
message_count, time_start, rate_hz = (
self.message_counts[key]
)
time_delta = time_now_s - time_start
if message_count - time_delta * msg_rate_hz > 0:
if message_count - time_delta * rate_hz > 0:
break
else:
del self.message_counts[user_id]
del self.message_counts[key]

View file

@ -33,9 +33,13 @@ from synapse.replication.slave.storage._base import BaseSlavedStore
from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
from synapse.replication.slave.storage.devices import SlavedDeviceStore
from synapse.replication.slave.storage.directory import DirectoryStore
from synapse.replication.slave.storage.events import SlavedEventStore
from synapse.replication.slave.storage.keys import SlavedKeyStore
from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
from synapse.replication.slave.storage.registration import SlavedRegistrationStore
from synapse.replication.slave.storage.room import RoomStore
from synapse.replication.slave.storage.transactions import SlavedTransactionStore
@ -48,6 +52,8 @@ from synapse.rest.client.v1.room import (
RoomMemberListRestServlet,
RoomStateRestServlet,
)
from synapse.rest.client.v2_alpha.account import ThreepidRestServlet
from synapse.rest.client.v2_alpha.keys import KeyChangesServlet, KeyQueryServlet
from synapse.rest.client.v2_alpha.register import RegisterRestServlet
from synapse.server import HomeServer
from synapse.storage.engines import create_engine
@ -60,6 +66,10 @@ logger = logging.getLogger("synapse.app.client_reader")
class ClientReaderSlavedStore(
SlavedDeviceInboxStore,
SlavedDeviceStore,
SlavedReceiptsStore,
SlavedPushRuleStore,
SlavedAccountDataStore,
SlavedEventStore,
SlavedKeyStore,
@ -96,6 +106,9 @@ class ClientReaderServer(HomeServer):
RoomEventContextServlet(self).register(resource)
RegisterRestServlet(self).register(resource)
LoginRestServlet(self).register(resource)
ThreepidRestServlet(self).register(resource)
KeyQueryServlet(self).register(resource)
KeyChangesServlet(self).register(resource)
resources.update({
"/_matrix/client/r0": resource,

View file

@ -180,9 +180,7 @@ class Config(object):
Returns:
str: the yaml config file
"""
default_config = "# vim:ft=yaml\n"
default_config += "\n\n".join(
default_config = "\n\n".join(
dedent(conf)
for conf in self.invoke_all(
"default_config",
@ -297,19 +295,26 @@ class Config(object):
"Must specify a server_name to a generate config for."
" Pass -H server.name."
)
config_str = obj.generate_config(
config_dir_path=config_dir_path,
data_dir_path=os.getcwd(),
server_name=server_name,
report_stats=(config_args.report_stats == "yes"),
generate_secrets=True,
)
if not cls.path_exists(config_dir_path):
os.makedirs(config_dir_path)
with open(config_path, "w") as config_file:
config_str = obj.generate_config(
config_dir_path=config_dir_path,
data_dir_path=os.getcwd(),
server_name=server_name,
report_stats=(config_args.report_stats == "yes"),
generate_secrets=True,
config_file.write(
"# vim:ft=yaml\n\n"
)
config = yaml.load(config_str)
obj.invoke_all("generate_files", config)
config_file.write(config_str)
config = yaml.load(config_str)
obj.invoke_all("generate_files", config)
print(
(
"A config file has been generated in %r for server name"

View file

@ -49,7 +49,8 @@ class DatabaseConfig(Config):
def default_config(self, data_dir_path, **kwargs):
database_path = os.path.join(data_dir_path, "homeserver.db")
return """\
# Database configuration
## Database ##
database:
# The database engine name
name: "sqlite3"

View file

@ -81,7 +81,9 @@ class LoggingConfig(Config):
def default_config(self, config_dir_path, server_name, **kwargs):
log_config = os.path.join(config_dir_path, server_name + ".log.config")
return """
return """\
## Logging ##
# A yaml python logging config file
#
log_config: "%(log_config)s"

View file

@ -54,6 +54,13 @@ class RegistrationConfig(Config):
config.get("disable_msisdn_registration", False)
)
self.rc_registration_requests_per_second = config.get(
"rc_registration_requests_per_second", 0.17,
)
self.rc_registration_request_burst_count = config.get(
"rc_registration_request_burst_count", 3,
)
def default_config(self, generate_secrets=False, **kwargs):
if generate_secrets:
registration_shared_secret = 'registration_shared_secret: "%s"' % (
@ -140,6 +147,17 @@ class RegistrationConfig(Config):
# users cannot be auto-joined since they do not exist.
#
autocreate_auto_join_rooms: true
# Number of registration requests a client can send per second.
# Defaults to 1/minute (0.17).
#
#rc_registration_requests_per_second: 0.17
# Number of registration requests a client can send before being
# throttled.
# Defaults to 3.
#
#rc_registration_request_burst_count: 3.0
""" % locals()
def add_arguments(self, parser):

View file

@ -260,9 +260,11 @@ class ServerConfig(Config):
# This is used by remote servers to connect to this server,
# e.g. matrix.org, localhost:8080, etc.
# This is also the last part of your UserID.
#
server_name: "%(server_name)s"
# When running as a daemon, the file to store the pid in
#
pid_file: %(pid_file)s
# CPU affinity mask. Setting this restricts the CPUs on which the
@ -304,9 +306,11 @@ class ServerConfig(Config):
# Set the soft limit on the number of file descriptors synapse can use
# Zero is used to indicate synapse should set the soft limit to the
# hard limit.
#
soft_file_limit: 0
# Set to false to disable presence tracking on this homeserver.
#
use_presence: true
# The GC threshold parameters to pass to `gc.set_threshold`, if defined

View file

@ -886,7 +886,7 @@ class ReplicationFederationHandlerRegistry(FederationHandlerRegistry):
def on_edu(self, edu_type, origin, content):
"""Overrides FederationHandlerRegistry
"""
if edu_type == "m.presence":
if not self.config.use_presence and edu_type == "m.presence":
return
handler = self.edu_handlers.get(edu_type)

View file

@ -159,8 +159,12 @@ class FederationRemoteSendQueue(object):
# stream.
pass
def send_edu(self, destination, edu_type, content, key=None):
def build_and_send_edu(self, destination, edu_type, content, key=None):
"""As per TransactionQueue"""
if destination == self.server_name:
logger.info("Not sending EDU to ourselves")
return
pos = self._next_pos()
edu = Edu(
@ -465,15 +469,11 @@ def process_rows_for_federation(transaction_queue, rows):
for destination, edu_map in iteritems(buff.keyed_edus):
for key, edu in edu_map.items():
transaction_queue.send_edu(
edu.destination, edu.edu_type, edu.content, key=key,
)
transaction_queue.send_edu(edu, key)
for destination, edu_list in iteritems(buff.edus):
for edu in edu_list:
transaction_queue.send_edu(
edu.destination, edu.edu_type, edu.content, key=None,
)
transaction_queue.send_edu(edu, None)
for destination in buff.device_destinations:
transaction_queue.send_device_messages(destination)

View file

@ -370,7 +370,19 @@ class TransactionQueue(object):
self._attempt_new_transaction(destination)
def send_edu(self, destination, edu_type, content, key=None):
def build_and_send_edu(self, destination, edu_type, content, key=None):
"""Construct an Edu object, and queue it for sending
Args:
destination (str): name of server to send to
edu_type (str): type of EDU to send
content (dict): content of EDU
key (Any|None): clobbering key for this edu
"""
if destination == self.server_name:
logger.info("Not sending EDU to ourselves")
return
edu = Edu(
origin=self.server_name,
destination=destination,
@ -378,16 +390,21 @@ class TransactionQueue(object):
content=content,
)
if destination == self.server_name:
logger.info("Not sending EDU to ourselves")
return
self.send_edu(edu, key)
def send_edu(self, edu, key):
"""Queue an EDU for sending
Args:
edu (Edu): edu to send
key (Any|None): clobbering key for this edu
"""
if key:
self.pending_edus_keyed_by_dest.setdefault(
destination, {}
edu.destination, {}
)[(edu.edu_type, key)] = edu
else:
self.pending_edus_by_dest.setdefault(destination, []).append(edu)
self.pending_edus_by_dest.setdefault(edu.destination, []).append(edu)
if destination not in self.edu_tx_time_by_dest:
txtime = self.clock.time() + EDU_BATCH_TIME * 1000

View file

@ -393,7 +393,7 @@ class FederationStateServlet(BaseFederationServlet):
return self.handler.on_context_state_request(
origin,
context,
parse_string_from_args(query, "event_id", None),
parse_string_from_args(query, "event_id", None, required=True),
)
@ -404,7 +404,7 @@ class FederationStateIdsServlet(BaseFederationServlet):
return self.handler.on_state_ids_request(
origin,
room_id,
parse_string_from_args(query, "event_id", None),
parse_string_from_args(query, "event_id", None, required=True),
)
@ -759,7 +759,7 @@ class FederationVersionServlet(BaseFederationServlet):
class FederationGroupsProfileServlet(BaseFederationServlet):
"""Get/set the basic profile of a group on behalf of a user
"""
PATH = "/groups/(?P<group_id>[^/]*)/profile$"
PATH = "/groups/(?P<group_id>[^/]*)/profile"
@defer.inlineCallbacks
def on_GET(self, origin, content, query, group_id):
@ -787,7 +787,7 @@ class FederationGroupsProfileServlet(BaseFederationServlet):
class FederationGroupsSummaryServlet(BaseFederationServlet):
PATH = "/groups/(?P<group_id>[^/]*)/summary$"
PATH = "/groups/(?P<group_id>[^/]*)/summary"
@defer.inlineCallbacks
def on_GET(self, origin, content, query, group_id):
@ -805,7 +805,7 @@ class FederationGroupsSummaryServlet(BaseFederationServlet):
class FederationGroupsRoomsServlet(BaseFederationServlet):
"""Get the rooms in a group on behalf of a user
"""
PATH = "/groups/(?P<group_id>[^/]*)/rooms$"
PATH = "/groups/(?P<group_id>[^/]*)/rooms"
@defer.inlineCallbacks
def on_GET(self, origin, content, query, group_id):
@ -823,7 +823,7 @@ class FederationGroupsRoomsServlet(BaseFederationServlet):
class FederationGroupsAddRoomsServlet(BaseFederationServlet):
"""Add/remove room from group
"""
PATH = "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)$"
PATH = "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, room_id):
@ -855,7 +855,7 @@ class FederationGroupsAddRoomsConfigServlet(BaseFederationServlet):
"""
PATH = (
"/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
"/config/(?P<config_key>[^/]*)$"
"/config/(?P<config_key>[^/]*)"
)
@defer.inlineCallbacks
@ -874,7 +874,7 @@ class FederationGroupsAddRoomsConfigServlet(BaseFederationServlet):
class FederationGroupsUsersServlet(BaseFederationServlet):
"""Get the users in a group on behalf of a user
"""
PATH = "/groups/(?P<group_id>[^/]*)/users$"
PATH = "/groups/(?P<group_id>[^/]*)/users"
@defer.inlineCallbacks
def on_GET(self, origin, content, query, group_id):
@ -892,7 +892,7 @@ class FederationGroupsUsersServlet(BaseFederationServlet):
class FederationGroupsInvitedUsersServlet(BaseFederationServlet):
"""Get the users that have been invited to a group
"""
PATH = "/groups/(?P<group_id>[^/]*)/invited_users$"
PATH = "/groups/(?P<group_id>[^/]*)/invited_users"
@defer.inlineCallbacks
def on_GET(self, origin, content, query, group_id):
@ -910,7 +910,7 @@ class FederationGroupsInvitedUsersServlet(BaseFederationServlet):
class FederationGroupsInviteServlet(BaseFederationServlet):
"""Ask a group server to invite someone to the group
"""
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite$"
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, user_id):
@ -928,7 +928,7 @@ class FederationGroupsInviteServlet(BaseFederationServlet):
class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
"""Accept an invitation from the group server
"""
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/accept_invite$"
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/accept_invite"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, user_id):
@ -945,7 +945,7 @@ class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
class FederationGroupsJoinServlet(BaseFederationServlet):
"""Attempt to join a group
"""
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join$"
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, user_id):
@ -962,7 +962,7 @@ class FederationGroupsJoinServlet(BaseFederationServlet):
class FederationGroupsRemoveUserServlet(BaseFederationServlet):
"""Leave or kick a user from the group
"""
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove$"
PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, user_id):
@ -980,7 +980,7 @@ class FederationGroupsRemoveUserServlet(BaseFederationServlet):
class FederationGroupsLocalInviteServlet(BaseFederationServlet):
"""A group server has invited a local user
"""
PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite$"
PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, user_id):
@ -997,7 +997,7 @@ class FederationGroupsLocalInviteServlet(BaseFederationServlet):
class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet):
"""A group server has removed a local user
"""
PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove$"
PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, user_id):
@ -1014,7 +1014,7 @@ class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet):
class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):
"""A group or user's server renews their attestation
"""
PATH = "/groups/(?P<group_id>[^/]*)/renew_attestation/(?P<user_id>[^/]*)$"
PATH = "/groups/(?P<group_id>[^/]*)/renew_attestation/(?P<user_id>[^/]*)"
@defer.inlineCallbacks
def on_POST(self, origin, content, query, group_id, user_id):
@ -1037,7 +1037,7 @@ class FederationGroupsSummaryRoomsServlet(BaseFederationServlet):
PATH = (
"/groups/(?P<group_id>[^/]*)/summary"
"(/categories/(?P<category_id>[^/]+))?"
"/rooms/(?P<room_id>[^/]*)$"
"/rooms/(?P<room_id>[^/]*)"
)
@defer.inlineCallbacks
@ -1080,7 +1080,7 @@ class FederationGroupsCategoriesServlet(BaseFederationServlet):
"""Get all categories for a group
"""
PATH = (
"/groups/(?P<group_id>[^/]*)/categories/$"
"/groups/(?P<group_id>[^/]*)/categories/"
)
@defer.inlineCallbacks
@ -1100,7 +1100,7 @@ class FederationGroupsCategoryServlet(BaseFederationServlet):
"""Add/remove/get a category in a group
"""
PATH = (
"/groups/(?P<group_id>[^/]*)/categories/(?P<category_id>[^/]+)$"
"/groups/(?P<group_id>[^/]*)/categories/(?P<category_id>[^/]+)"
)
@defer.inlineCallbacks
@ -1150,7 +1150,7 @@ class FederationGroupsRolesServlet(BaseFederationServlet):
"""Get roles in a group
"""
PATH = (
"/groups/(?P<group_id>[^/]*)/roles/$"
"/groups/(?P<group_id>[^/]*)/roles/"
)
@defer.inlineCallbacks
@ -1170,7 +1170,7 @@ class FederationGroupsRoleServlet(BaseFederationServlet):
"""Add/remove/get a role in a group
"""
PATH = (
"/groups/(?P<group_id>[^/]*)/roles/(?P<role_id>[^/]+)$"
"/groups/(?P<group_id>[^/]*)/roles/(?P<role_id>[^/]+)"
)
@defer.inlineCallbacks
@ -1226,7 +1226,7 @@ class FederationGroupsSummaryUsersServlet(BaseFederationServlet):
PATH = (
"/groups/(?P<group_id>[^/]*)/summary"
"(/roles/(?P<role_id>[^/]+))?"
"/users/(?P<user_id>[^/]*)$"
"/users/(?P<user_id>[^/]*)"
)
@defer.inlineCallbacks
@ -1269,7 +1269,7 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
"""Get roles in a group
"""
PATH = (
"/get_groups_publicised$"
"/get_groups_publicised"
)
@defer.inlineCallbacks
@ -1284,7 +1284,7 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
"""Sets whether a group is joinable without an invite or knock
"""
PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy$"
PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy"
@defer.inlineCallbacks
def on_PUT(self, origin, content, query, group_id):

Some files were not shown because too many files have changed in this diff Show more