forked from MirrorHub/synapse
Merge branch 'develop' of github.com:matrix-org/synapse into erikj/split_out_persistence_store
This commit is contained in:
Erik Johnston
commit
ec6de1cc7d
146 changed files with 1245 additions and 722 deletions
21
.buildkite/postgres-config.yaml
Normal file
21
.buildkite/postgres-config.yaml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# Configuration file used for testing the 'synapse_port_db' script.
|
||||
# Tells the script to connect to the postgresql database that will be available in the
|
||||
# CI's Docker setup at the point where this file is considered.
|
||||
server_name: "test"
|
||||
|
||||
signing_key_path: "/src/.buildkite/test.signing.key"
|
||||
|
||||
report_stats: false
|
||||
|
||||
database:
|
||||
name: "psycopg2"
|
||||
args:
|
||||
user: postgres
|
||||
host: postgres
|
||||
password: postgres
|
||||
database: synapse
|
||||
|
||||
# Suppress the key server warning.
|
||||
trusted_key_servers:
|
||||
- server_name: "matrix.org"
|
||||
suppress_key_server_warning: true
|
||||
36
.buildkite/scripts/create_postgres_db.py
Executable file
36
.buildkite/scripts/create_postgres_db.py
Executable file
|
|
@ -0,0 +1,36 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2019 The Matrix.org Foundation C.I.C.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
from synapse.storage.engines import create_engine
|
||||
|
||||
logger = logging.getLogger("create_postgres_db")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Create a PostgresEngine.
|
||||
db_engine = create_engine({"name": "psycopg2", "args": {}})
|
||||
|
||||
# Connect to postgres to create the base database.
|
||||
# We use "postgres" as a database because it's bound to exist and the "synapse" one
|
||||
# doesn't exist yet.
|
||||
db_conn = db_engine.module.connect(
|
||||
user="postgres", host="postgres", password="postgres", dbname="postgres"
|
||||
)
|
||||
db_conn.autocommit = True
|
||||
cur = db_conn.cursor()
|
||||
cur.execute("CREATE DATABASE synapse;")
|
||||
cur.close()
|
||||
db_conn.close()
|
||||
36
.buildkite/scripts/test_synapse_port_db.sh
Executable file
36
.buildkite/scripts/test_synapse_port_db.sh
Executable file
|
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
|
||||
# with additional dependencies needed for the test (such as coverage or the PostgreSQL
|
||||
# driver), update the schema of the test SQLite database and run background updates on it,
|
||||
# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
|
||||
# test porting the SQLite database to the PostgreSQL database (with coverage).
|
||||
|
||||
set -xe
|
||||
cd `dirname $0`/../..
|
||||
|
||||
echo "--- Install dependencies"
|
||||
|
||||
# Install dependencies for this test.
|
||||
pip install psycopg2 coverage coverage-enable-subprocess
|
||||
|
||||
# Install Synapse itself. This won't update any libraries.
|
||||
pip install -e .
|
||||
|
||||
echo "--- Generate the signing key"
|
||||
|
||||
# Generate the server's signing key.
|
||||
python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
|
||||
|
||||
echo "--- Prepare the databases"
|
||||
|
||||
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
|
||||
scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
|
||||
|
||||
# Create the PostgreSQL database.
|
||||
./.buildkite/scripts/create_postgres_db.py
|
||||
|
||||
echo "+++ Run synapse_port_db"
|
||||
|
||||
# Run the script
|
||||
coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
|
||||
18
.buildkite/sqlite-config.yaml
Normal file
18
.buildkite/sqlite-config.yaml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# Configuration file used for testing the 'synapse_port_db' script.
|
||||
# Tells the 'update_database' script to connect to the test SQLite database to upgrade its
|
||||
# schema and run background updates on it.
|
||||
server_name: "test"
|
||||
|
||||
signing_key_path: "/src/.buildkite/test.signing.key"
|
||||
|
||||
report_stats: false
|
||||
|
||||
database:
|
||||
name: "sqlite3"
|
||||
args:
|
||||
database: ".buildkite/test_db.db"
|
||||
|
||||
# Suppress the key server warning.
|
||||
trusted_key_servers:
|
||||
- server_name: "matrix.org"
|
||||
suppress_key_server_warning: true
|
||||
BIN
.buildkite/test_db.db
Normal file
BIN
.buildkite/test_db.db
Normal file
Binary file not shown.
41
AUTHORS.rst
41
AUTHORS.rst
|
|
@ -1,34 +1,8 @@
|
|||
Erik Johnston <erik at matrix.org>
|
||||
* HS core
|
||||
* Federation API impl
|
||||
The following is an incomplete list of people outside the core team who have
|
||||
contributed to Synapse. It is no longer maintained: more recent contributions
|
||||
are listed in the `changelog <CHANGES.md>`_.
|
||||
|
||||
Mark Haines <mark at matrix.org>
|
||||
* HS core
|
||||
* Crypto
|
||||
* Content repository
|
||||
* CS v2 API impl
|
||||
|
||||
Kegan Dougal <kegan at matrix.org>
|
||||
* HS core
|
||||
* CS v1 API impl
|
||||
* AS API impl
|
||||
|
||||
Paul "LeoNerd" Evans <paul at matrix.org>
|
||||
* HS core
|
||||
* Presence
|
||||
* Typing Notifications
|
||||
* Performance metrics and caching layer
|
||||
|
||||
Dave Baker <dave at matrix.org>
|
||||
* Push notifications
|
||||
* Auth CS v2 impl
|
||||
|
||||
Matthew Hodgson <matthew at matrix.org>
|
||||
* General doc & housekeeping
|
||||
* Vertobot/vertobridge matrix<->verto PoC
|
||||
|
||||
Emmanuel Rohee <manu at matrix.org>
|
||||
* Supporting iOS clients (testability and fallback registration)
|
||||
----
|
||||
|
||||
Turned to Dust <dwinslow86 at gmail.com>
|
||||
* ArchLinux installation instructions
|
||||
|
|
@ -62,16 +36,13 @@ Christoph Witzany <christoph at web.crofting.com>
|
|||
* Add LDAP support for authentication
|
||||
|
||||
Pierre Jaury <pierre at jaury.eu>
|
||||
* Docker packaging
|
||||
* Docker packaging
|
||||
|
||||
Serban Constantin <serban.constantin at gmail dot com>
|
||||
* Small bug fix
|
||||
|
||||
Jason Robinson <jasonr at matrix.org>
|
||||
* Minor fixes
|
||||
|
||||
Joseph Weston <joseph at weston.cloud>
|
||||
+ Add admin API for querying HS version
|
||||
* Add admin API for querying HS version
|
||||
|
||||
Benjamin Saunders <ben.e.saunders at gmail dot com>
|
||||
* Documentation improvements
|
||||
|
|
|
|||
111
CHANGES.md
111
CHANGES.md
|
|
@ -1,3 +1,114 @@
|
|||
Synapse 1.5.0 (2019-10-29)
|
||||
==========================
|
||||
|
||||
Security updates
|
||||
----------------
|
||||
|
||||
This release includes a security fix ([\#6262](https://github.com/matrix-org/synapse/issues/6262), below). Administrators are encouraged to upgrade as soon as possible.
|
||||
|
||||
Bugfixes
|
||||
--------
|
||||
|
||||
- Fix bug where room directory search was case sensitive. ([\#6268](https://github.com/matrix-org/synapse/issues/6268))
|
||||
|
||||
|
||||
Synapse 1.5.0rc2 (2019-10-28)
|
||||
=============================
|
||||
|
||||
Bugfixes
|
||||
--------
|
||||
|
||||
- Update list of boolean columns in `synapse_port_db`. ([\#6247](https://github.com/matrix-org/synapse/issues/6247))
|
||||
- Fix /keys/query API on workers. ([\#6256](https://github.com/matrix-org/synapse/issues/6256))
|
||||
- Improve signature checking on some federation APIs. ([\#6262](https://github.com/matrix-org/synapse/issues/6262))
|
||||
|
||||
|
||||
Internal Changes
|
||||
----------------
|
||||
|
||||
- Move schema delta files to the correct data store. ([\#6248](https://github.com/matrix-org/synapse/issues/6248))
|
||||
- Small performance improvement by removing repeated config lookups in room stats calculation. ([\#6255](https://github.com/matrix-org/synapse/issues/6255))
|
||||
|
||||
|
||||
Synapse 1.5.0rc1 (2019-10-24)
|
||||
==========================
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
- Improve quality of thumbnails for 1-bit/8-bit color palette images. ([\#2142](https://github.com/matrix-org/synapse/issues/2142))
|
||||
- Add ability to upload cross-signing signatures. ([\#5726](https://github.com/matrix-org/synapse/issues/5726))
|
||||
- Allow uploading of cross-signing keys. ([\#5769](https://github.com/matrix-org/synapse/issues/5769))
|
||||
- CAS login now provides a default display name for users if a `displayname_attribute` is set in the configuration file. ([\#6114](https://github.com/matrix-org/synapse/issues/6114))
|
||||
- Reject all pending invites for a user during deactivation. ([\#6125](https://github.com/matrix-org/synapse/issues/6125))
|
||||
- Add config option to suppress client side resource limit alerting. ([\#6173](https://github.com/matrix-org/synapse/issues/6173))
|
||||
|
||||
|
||||
Bugfixes
|
||||
--------
|
||||
|
||||
- Return an HTTP 404 instead of 400 when requesting a filter by ID that is unknown to the server. Thanks to @krombel for contributing this! ([\#2380](https://github.com/matrix-org/synapse/issues/2380))
|
||||
- Fix a bug where users could be invited twice to the same group. ([\#3436](https://github.com/matrix-org/synapse/issues/3436))
|
||||
- Fix `/createRoom` failing with badly-formatted MXIDs in the invitee list. Thanks to @wener291! ([\#4088](https://github.com/matrix-org/synapse/issues/4088))
|
||||
- Make the `synapse_port_db` script create the right indexes on a new PostgreSQL database. ([\#6102](https://github.com/matrix-org/synapse/issues/6102), [\#6178](https://github.com/matrix-org/synapse/issues/6178), [\#6243](https://github.com/matrix-org/synapse/issues/6243))
|
||||
- Fix bug when uploading a large file: Synapse responds with `M_UNKNOWN` while it should be `M_TOO_LARGE` according to spec. Contributed by Anshul Angaria. ([\#6109](https://github.com/matrix-org/synapse/issues/6109))
|
||||
- Fix user push rules being deleted from a room when it is upgraded. ([\#6144](https://github.com/matrix-org/synapse/issues/6144))
|
||||
- Don't 500 when trying to exchange a revoked 3PID invite. ([\#6147](https://github.com/matrix-org/synapse/issues/6147))
|
||||
- Fix transferring notifications and tags when joining an upgraded room that is new to your server. ([\#6155](https://github.com/matrix-org/synapse/issues/6155))
|
||||
- Fix bug where guest account registration can wedge after restart. ([\#6161](https://github.com/matrix-org/synapse/issues/6161))
|
||||
- Fix monthly active user reaping when reserved users are specified. ([\#6168](https://github.com/matrix-org/synapse/issues/6168))
|
||||
- Fix `/federation/v1/state` endpoint not supporting newer room versions. ([\#6170](https://github.com/matrix-org/synapse/issues/6170))
|
||||
- Fix bug where we were updating censored events as bytes rather than text, occaisonally causing invalid JSON being inserted breaking APIs that attempted to fetch such events. ([\#6186](https://github.com/matrix-org/synapse/issues/6186))
|
||||
- Fix occasional missed updates in the room and user directories. ([\#6187](https://github.com/matrix-org/synapse/issues/6187))
|
||||
- Fix tracing of non-JSON APIs, `/media`, `/key` etc. ([\#6195](https://github.com/matrix-org/synapse/issues/6195))
|
||||
- Fix bug where presence would not get timed out correctly if a synchrotron worker is used and restarted. ([\#6212](https://github.com/matrix-org/synapse/issues/6212))
|
||||
- synapse_port_db: Add 2 additional BOOLEAN_COLUMNS to be able to convert from database schema v56. ([\#6216](https://github.com/matrix-org/synapse/issues/6216))
|
||||
- Fix a bug where the Synapse demo script blacklisted `::1` (ipv6 localhost) from receiving federation traffic. ([\#6229](https://github.com/matrix-org/synapse/issues/6229))
|
||||
|
||||
|
||||
Updates to the Docker image
|
||||
---------------------------
|
||||
|
||||
- Fix logging getting lost for the docker image. ([\#6197](https://github.com/matrix-org/synapse/issues/6197))
|
||||
|
||||
|
||||
Internal Changes
|
||||
----------------
|
||||
|
||||
- Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this. ([\#1172](https://github.com/matrix-org/synapse/issues/1172), [\#6175](https://github.com/matrix-org/synapse/issues/6175), [\#6184](https://github.com/matrix-org/synapse/issues/6184))
|
||||
- Allow devices to be marked as hidden, for use by features such as cross-signing.
|
||||
This adds a new field with a default value to the devices field in the database,
|
||||
and so the database upgrade may take a long time depending on how many devices
|
||||
are in the database. ([\#5759](https://github.com/matrix-org/synapse/issues/5759))
|
||||
- Move lookup-related functions from RoomMemberHandler to IdentityHandler. ([\#5978](https://github.com/matrix-org/synapse/issues/5978))
|
||||
- Improve performance of the public room list directory. ([\#6019](https://github.com/matrix-org/synapse/issues/6019), [\#6152](https://github.com/matrix-org/synapse/issues/6152), [\#6153](https://github.com/matrix-org/synapse/issues/6153), [\#6154](https://github.com/matrix-org/synapse/issues/6154))
|
||||
- Edit header dicts docstrings in `SimpleHttpClient` to note that `str` or `bytes` can be passed as header keys. ([\#6077](https://github.com/matrix-org/synapse/issues/6077))
|
||||
- Add snapcraft packaging information. Contributed by @devec0. ([\#6084](https://github.com/matrix-org/synapse/issues/6084), [\#6191](https://github.com/matrix-org/synapse/issues/6191))
|
||||
- Kill off half-implemented password-reset via sms. ([\#6101](https://github.com/matrix-org/synapse/issues/6101))
|
||||
- Remove `get_user_by_req` opentracing span and add some tags. ([\#6108](https://github.com/matrix-org/synapse/issues/6108))
|
||||
- Drop some unused database tables. ([\#6115](https://github.com/matrix-org/synapse/issues/6115))
|
||||
- Add env var to turn on tracking of log context changes. ([\#6127](https://github.com/matrix-org/synapse/issues/6127))
|
||||
- Refactor configuration loading to allow better typechecking. ([\#6137](https://github.com/matrix-org/synapse/issues/6137))
|
||||
- Log responder when responding to media request. ([\#6139](https://github.com/matrix-org/synapse/issues/6139))
|
||||
- Improve performance of `find_next_generated_user_id` DB query. ([\#6148](https://github.com/matrix-org/synapse/issues/6148))
|
||||
- Expand type-checking on modules imported by `synapse.config`. ([\#6150](https://github.com/matrix-org/synapse/issues/6150))
|
||||
- Use Postgres ANY for selecting many values. ([\#6156](https://github.com/matrix-org/synapse/issues/6156))
|
||||
- Add more caching to `_get_joined_users_from_context` DB query. ([\#6159](https://github.com/matrix-org/synapse/issues/6159))
|
||||
- Add some metrics on the federation sender. ([\#6160](https://github.com/matrix-org/synapse/issues/6160))
|
||||
- Add some logging to the rooms stats updates, to try to track down a flaky test. ([\#6167](https://github.com/matrix-org/synapse/issues/6167))
|
||||
- Remove unused `timeout` parameter from `_get_public_room_list`. ([\#6179](https://github.com/matrix-org/synapse/issues/6179))
|
||||
- Reject (accidental) attempts to insert bytes into postgres tables. ([\#6186](https://github.com/matrix-org/synapse/issues/6186))
|
||||
- Make `version` optional in body of `PUT /room_keys/version/{version}`, since it's redundant. ([\#6189](https://github.com/matrix-org/synapse/issues/6189))
|
||||
- Make storage layer responsible for adding device names to key, rather than the handler. ([\#6193](https://github.com/matrix-org/synapse/issues/6193))
|
||||
- Port `synapse.rest.admin` module to use async/await. ([\#6196](https://github.com/matrix-org/synapse/issues/6196))
|
||||
- Enforce that all boolean configuration values are lowercase in CI. ([\#6203](https://github.com/matrix-org/synapse/issues/6203))
|
||||
- Remove some unused event-auth code. ([\#6214](https://github.com/matrix-org/synapse/issues/6214))
|
||||
- Remove `Auth.check` method. ([\#6217](https://github.com/matrix-org/synapse/issues/6217))
|
||||
- Remove `format_tap.py` script in favour of a perl reimplementation in Sytest's repo. ([\#6219](https://github.com/matrix-org/synapse/issues/6219))
|
||||
- Refactor storage layer in preparation to support having multiple databases. ([\#6231](https://github.com/matrix-org/synapse/issues/6231))
|
||||
- Remove some extra quotation marks across the codebase. ([\#6236](https://github.com/matrix-org/synapse/issues/6236))
|
||||
|
||||
|
||||
Synapse 1.4.1 (2019-10-18)
|
||||
==========================
|
||||
|
||||
|
|
|
|||
|
|
@ -114,17 +114,6 @@ directory, you will need both a regular newsfragment *and* an entry in the
|
|||
debian changelog. (Though typically such changes should be submitted as two
|
||||
separate pull requests.)
|
||||
|
||||
Attribution
|
||||
~~~~~~~~~~~
|
||||
|
||||
Everyone who contributes anything to Matrix is welcome to be listed in the
|
||||
AUTHORS.rst file for the project in question. Please feel free to include a
|
||||
change to AUTHORS.rst in your pull request to list yourself and a short
|
||||
description of the area(s) you've worked on. Also, we sometimes have swag to
|
||||
give away to contributors - if you feel that Matrix-branded apparel is missing
|
||||
from your life, please mail us your shipping address to matrix at matrix.org and
|
||||
we'll try to fix it :)
|
||||
|
||||
Sign off
|
||||
~~~~~~~~
|
||||
|
||||
|
|
|
|||
12
UPGRADE.rst
12
UPGRADE.rst
|
|
@ -2,7 +2,7 @@ Upgrading Synapse
|
|||
=================
|
||||
|
||||
Before upgrading check if any special steps are required to upgrade from the
|
||||
what you currently have installed to current version of Synapse. The extra
|
||||
version you currently have installed to the current version of Synapse. The extra
|
||||
instructions that may be required are listed later in this document.
|
||||
|
||||
* If Synapse was installed using `prebuilt packages
|
||||
|
|
@ -75,6 +75,16 @@ for example:
|
|||
wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
|
||||
dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
|
||||
|
||||
|
||||
Upgrading to v1.5.0
|
||||
===================
|
||||
|
||||
This release includes a database migration which may take several minutes to
|
||||
complete if there are a large number (more than a million or so) of entries in
|
||||
the ``devices`` table. This is only likely to a be a problem on very large
|
||||
installations.
|
||||
|
||||
|
||||
Upgrading to v1.4.0
|
||||
===================
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve quality of thumbnails for 1-bit/8-bit color palette images.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Return an HTTP 404 instead of 400 when requesting a filter by ID that is unknown to the server. Thanks to @krombel for contributing this!
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix a problem where users could be invited twice to the same group.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Added domain validation when including a list of invitees upon room creation.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add ability to upload cross-signing signatures.
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
Allow devices to be marked as hidden, for use by features such as cross-signing.
|
||||
This adds a new field with a default value to the devices field in the database,
|
||||
and so the database upgrade may take a long time depending on how many devices
|
||||
are in the database.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Allow uploading of cross-signing keys.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Move lookup-related functions from RoomMemberHandler to IdentityHandler.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve performance of the public room list directory.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Edit header dicts docstrings in SimpleHttpClient to note that `str` or `bytes` can be passed as header keys.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add snapcraft packaging information. Contributed by @devec0.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Kill off half-implemented password-reset via sms.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove `get_user_by_req` opentracing span and add some tags.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix bug when uploading a large file: Synapse responds with `M_UNKNOWN` while it should be `M_TOO_LARGE` according to spec. Contributed by Anshul Angaria.
|
||||
|
|
@ -1 +0,0 @@
|
|||
CAS login now provides a default display name for users if a `displayname_attribute` is set in the configuration file.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Drop some unused database tables.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Reject all pending invites for a user during deactivation.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add env var to turn on tracking of log context changes.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Refactor configuration loading to allow better typechecking.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Log responder when responding to media request.
|
||||
1
changelog.d/6140.misc
Normal file
1
changelog.d/6140.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Add a CI job to test the `synapse_port_db` script.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Prevent user push rules being deleted from a room when it is upgraded.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Don't 500 when trying to exchange a revoked 3PID invite.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve performance of `find_next_generated_user_id` DB query.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Expand type-checking on modules imported by synapse.config.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve performance of the public room list directory.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve performance of the public room list directory.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve performance of the public room list directory.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix transferring notifications and tags when joining an upgraded room that is new to your server.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Use Postgres ANY for selecting many values.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add more caching to `_get_joined_users_from_context` DB query.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add some metrics on the federation sender.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix bug where guest account registration can wedge after restart.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add some logging to the rooms stats updates, to try to track down a flaky test.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix monthly active user reaping where reserved users are specified.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix /federation/v1/state endpoint for recent room versions.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Make the `synapse_port_db` script create the right indexes on a new PostgreSQL database.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove unused `timeout` parameter from `_get_public_room_list`.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix bug where we were updating censored events as bytes rather than text, occaisonally causing invalid JSON being inserted breaking APIs that attempted to fetch such events.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Reject (accidental) attempts to insert bytes into postgres tables.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix occasional missed updates in the room and user directories.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Make `version` optional in body of `PUT /room_keys/version/{version}`, since it's redundant.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add snapcraft packaging information. Contributed by @devec0.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Make storage layer responsible for adding device names to key, rather than the handler.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix tracing of non-JSON APIs, /media, /key etc.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Port synapse.rest.admin module to use async/await.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix logging getting lost for the docker image.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Enforce that all boolean configuration values are lowercase in CI.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix bug where presence would not get timed out correctly if a synchrotron worker is used and restarted.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove some unused event-auth code.
|
||||
|
|
@ -1 +0,0 @@
|
|||
synapse_port_db: Add 2 additional BOOLEAN_COLUMNS to be able to convert from database schema v56.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove Auth.check method.
|
||||
1
changelog.d/6218.misc
Normal file
1
changelog.d/6218.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Convert EventContext to an attrs.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove `format_tap.py` script in favour of a perl reimplementation in Sytest's repo.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Prevent the demo Synapse's from blacklisting `::1`.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Refactor storage layer in preparation to support having multiple databases.
|
||||
1
changelog.d/6250.misc
Normal file
1
changelog.d/6250.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Reduce verbosity of user/room stats.
|
||||
1
changelog.d/6251.misc
Normal file
1
changelog.d/6251.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Reduce impact of debug logging.
|
||||
1
changelog.d/6253.bugfix
Normal file
1
changelog.d/6253.bugfix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Delete keys from key backup when deleting backup versions.
|
||||
1
changelog.d/6257.doc
Normal file
1
changelog.d/6257.doc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Modify CAPTCHA_SETUP.md to update the terms `private key` and `public key` to `secret key` and `site key` respectively. Contributed by Yash Jipkate.
|
||||
1
changelog.d/6263.misc
Normal file
1
changelog.d/6263.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Change cache descriptors to always return deferreds.
|
||||
1
changelog.d/6269.misc
Normal file
1
changelog.d/6269.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix incorrect comment regarding the functionality of an `if` statement.
|
||||
1
changelog.d/6270.misc
Normal file
1
changelog.d/6270.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Update CI to run `isort` over the `scripts` and `scripts-dev` directories.
|
||||
1
changelog.d/6273.doc
Normal file
1
changelog.d/6273.doc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix a small typo in `account_threepid_delegates` configuration option.
|
||||
1
changelog.d/6274.misc
Normal file
1
changelog.d/6274.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Port replication http server endpoints to async/await.
|
||||
1
changelog.d/6275.misc
Normal file
1
changelog.d/6275.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Port room rest handlers to async/await.
|
||||
1
changelog.d/6276.misc
Normal file
1
changelog.d/6276.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Add a CI job to test the `synapse_port_db` script.
|
||||
1
changelog.d/6277.misc
Normal file
1
changelog.d/6277.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Remove redundant CLI parameters on CI's `flake8` step.
|
||||
1
changelog.d/6278.bugfix
Normal file
1
changelog.d/6278.bugfix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix exception when remote servers attempt to join a room that they're not allowed to join.
|
||||
1
changelog.d/6279.misc
Normal file
1
changelog.d/6279.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Port `federation_server.py` to async/await.
|
||||
1
changelog.d/6280.misc
Normal file
1
changelog.d/6280.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Port receipt and read markers to async/wait.
|
||||
1
changelog.d/6284.bugfix
Normal file
1
changelog.d/6284.bugfix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Prevent errors from appearing on Synapse startup if `git` is not installed.
|
||||
1
changelog.d/6291.misc
Normal file
1
changelog.d/6291.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Change cache descriptors to always return deferreds.
|
||||
|
|
@ -339,7 +339,7 @@ def main(stdscr):
|
|||
root_logger = logging.getLogger()
|
||||
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
|
||||
"%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
|
||||
)
|
||||
if not os.path.exists("logs"):
|
||||
os.makedirs("logs")
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
|
|||
args = [room_id]
|
||||
|
||||
if limit:
|
||||
sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?"
|
||||
sql += " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?"
|
||||
|
||||
args.append(limit)
|
||||
|
||||
|
|
@ -53,7 +53,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
|
|||
|
||||
for event in events:
|
||||
c = conn.execute(
|
||||
"SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?",
|
||||
"SELECT state_group FROM event_to_state_groups WHERE event_id = ?",
|
||||
(event.event_id,),
|
||||
)
|
||||
|
||||
|
|
|
|||
6
debian/changelog
vendored
6
debian/changelog
vendored
|
|
@ -1,3 +1,9 @@
|
|||
matrix-synapse-py3 (1.5.0) stable; urgency=medium
|
||||
|
||||
* New synapse release 1.5.0.
|
||||
|
||||
-- Synapse Packaging team <packages@matrix.org> Tue, 29 Oct 2019 14:28:41 +0000
|
||||
|
||||
matrix-synapse-py3 (1.4.1) stable; urgency=medium
|
||||
|
||||
* New synapse release 1.4.1.
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ is suitable for local testing, but for any practical use, you will either need
|
|||
to use a reverse proxy, or configure Synapse to expose an HTTPS port.
|
||||
|
||||
For documentation on using a reverse proxy, see
|
||||
https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst.
|
||||
https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
|
||||
|
||||
For more information on enabling TLS support in synapse itself, see
|
||||
https://github.com/matrix-org/synapse/blob/master/INSTALL.md#tls-certificates. Of
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ The captcha mechanism used is Google's ReCaptcha. This requires API keys from Go
|
|||
|
||||
## Getting keys
|
||||
|
||||
Requires a public/private key pair from:
|
||||
Requires a site/secret key pair from:
|
||||
|
||||
<https://developers.google.com/recaptcha/>
|
||||
|
||||
|
|
@ -15,8 +15,8 @@ Must be a reCAPTCHA v2 key using the "I'm not a robot" Checkbox option
|
|||
The keys are a config option on the home server config. If they are not
|
||||
visible, you can generate them via `--generate-config`. Set the following value:
|
||||
|
||||
recaptcha_public_key: YOUR_PUBLIC_KEY
|
||||
recaptcha_private_key: YOUR_PRIVATE_KEY
|
||||
recaptcha_public_key: YOUR_SITE_KEY
|
||||
recaptcha_private_key: YOUR_SECRET_KEY
|
||||
|
||||
In addition, you MUST enable captchas via:
|
||||
|
||||
|
|
|
|||
|
|
@ -241,7 +241,6 @@ listeners:
|
|||
#
|
||||
#hs_disabled: false
|
||||
#hs_disabled_message: 'Human readable reason for why the HS is blocked'
|
||||
#hs_disabled_limit_type: 'error code(str), to help clients decode reason'
|
||||
|
||||
# Monthly Active User Blocking
|
||||
#
|
||||
|
|
@ -261,9 +260,16 @@ listeners:
|
|||
# sign up in a short space of time never to return after their initial
|
||||
# session.
|
||||
#
|
||||
# 'mau_limit_alerting' is a means of limiting client side alerting
|
||||
# should the mau limit be reached. This is useful for small instances
|
||||
# where the admin has 5 mau seats (say) for 5 specific people and no
|
||||
# interest increasing the mau limit further. Defaults to True, which
|
||||
# means that alerting is enabled
|
||||
#
|
||||
#limit_usage_by_mau: false
|
||||
#max_mau_value: 50
|
||||
#mau_trial_days: 2
|
||||
#mau_limit_alerting: false
|
||||
|
||||
# If enabled, the metrics for the number of monthly active users will
|
||||
# be populated, however no one will be limited. If limit_usage_by_mau
|
||||
|
|
@ -949,7 +955,7 @@ uploads_path: "DATADIR/uploads"
|
|||
# If a delegate is specified, the config option public_baseurl must also be filled out.
|
||||
#
|
||||
account_threepid_delegates:
|
||||
#email: https://example.com # Delegate email sending to example.org
|
||||
#email: https://example.com # Delegate email sending to example.com
|
||||
#msisdn: http://localhost:8090 # Delegate SMS sending to this local process
|
||||
|
||||
# Users who register on this homeserver will automatically be joined
|
||||
|
|
|
|||
124
scripts-dev/update_database
Executable file
124
scripts-dev/update_database
Executable file
|
|
@ -0,0 +1,124 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2019 The Matrix.org Foundation C.I.C.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
from twisted.internet import defer, reactor
|
||||
|
||||
from synapse.config.homeserver import HomeServerConfig
|
||||
from synapse.metrics.background_process_metrics import run_as_background_process
|
||||
from synapse.server import HomeServer
|
||||
from synapse.storage import DataStore
|
||||
from synapse.storage.engines import create_engine
|
||||
from synapse.storage.prepare_database import prepare_database
|
||||
|
||||
logger = logging.getLogger("update_database")
|
||||
|
||||
|
||||
class MockHomeserver(HomeServer):
|
||||
DATASTORE_CLASS = DataStore
|
||||
|
||||
def __init__(self, config, database_engine, db_conn, **kwargs):
|
||||
super(MockHomeserver, self).__init__(
|
||||
config.server_name,
|
||||
reactor=reactor,
|
||||
config=config,
|
||||
database_engine=database_engine,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
self.database_engine = database_engine
|
||||
self.db_conn = db_conn
|
||||
|
||||
def get_db_conn(self):
|
||||
return self.db_conn
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Updates a synapse database to the latest schema and runs background updates"
|
||||
" on it."
|
||||
)
|
||||
)
|
||||
parser.add_argument("-v", action='store_true')
|
||||
parser.add_argument(
|
||||
"--database-config",
|
||||
type=argparse.FileType('r'),
|
||||
required=True,
|
||||
help="A database config file for either a SQLite3 database or a PostgreSQL one.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logging_config = {
|
||||
"level": logging.DEBUG if args.v else logging.INFO,
|
||||
"format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
|
||||
}
|
||||
|
||||
logging.basicConfig(**logging_config)
|
||||
|
||||
# Load, process and sanity-check the config.
|
||||
hs_config = yaml.safe_load(args.database_config)
|
||||
|
||||
if "database" not in hs_config:
|
||||
sys.stderr.write("The configuration file must have a 'database' section.\n")
|
||||
sys.exit(4)
|
||||
|
||||
config = HomeServerConfig()
|
||||
config.parse_config_dict(hs_config, "", "")
|
||||
|
||||
# Create the database engine and a connection to it.
|
||||
database_engine = create_engine(config.database_config)
|
||||
db_conn = database_engine.module.connect(
|
||||
**{
|
||||
k: v
|
||||
for k, v in config.database_config.get("args", {}).items()
|
||||
if not k.startswith("cp_")
|
||||
}
|
||||
)
|
||||
|
||||
# Update the database to the latest schema.
|
||||
prepare_database(db_conn, database_engine, config=config)
|
||||
db_conn.commit()
|
||||
|
||||
# Instantiate and initialise the homeserver object.
|
||||
hs = MockHomeserver(
|
||||
config,
|
||||
database_engine,
|
||||
db_conn,
|
||||
db_config=config.database_config,
|
||||
)
|
||||
# setup instantiates the store within the homeserver object.
|
||||
hs.setup()
|
||||
store = hs.get_datastore()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_background_updates():
|
||||
yield store.run_background_updates(sleep=False)
|
||||
# Stop the reactor to exit the script once every background update is run.
|
||||
reactor.stop()
|
||||
|
||||
# Apply all background updates on the database.
|
||||
reactor.callWhenRunning(lambda: run_as_background_process(
|
||||
"background_updates", run_background_updates
|
||||
))
|
||||
|
||||
reactor.run()
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2015, 2016 OpenMarket Ltd
|
||||
# Copyright 2018 New Vector Ltd
|
||||
# Copyright 2019 The Matrix.org Foundation C.I.C.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -29,9 +30,33 @@ import yaml
|
|||
from twisted.enterprise import adbapi
|
||||
from twisted.internet import defer, reactor
|
||||
|
||||
from synapse.storage._base import LoggingTransaction, SQLBaseStore
|
||||
from synapse.config.homeserver import HomeServerConfig
|
||||
from synapse.logging.context import PreserveLoggingContext
|
||||
from synapse.storage._base import LoggingTransaction
|
||||
from synapse.storage.data_stores.main.client_ips import ClientIpBackgroundUpdateStore
|
||||
from synapse.storage.data_stores.main.deviceinbox import (
|
||||
DeviceInboxBackgroundUpdateStore,
|
||||
)
|
||||
from synapse.storage.data_stores.main.devices import DeviceBackgroundUpdateStore
|
||||
from synapse.storage.data_stores.main.events_bg_updates import (
|
||||
EventsBackgroundUpdatesStore,
|
||||
)
|
||||
from synapse.storage.data_stores.main.media_repository import (
|
||||
MediaRepositoryBackgroundUpdateStore,
|
||||
)
|
||||
from synapse.storage.data_stores.main.registration import (
|
||||
RegistrationBackgroundUpdateStore,
|
||||
)
|
||||
from synapse.storage.data_stores.main.roommember import RoomMemberBackgroundUpdateStore
|
||||
from synapse.storage.data_stores.main.search import SearchBackgroundUpdateStore
|
||||
from synapse.storage.data_stores.main.state import StateBackgroundUpdateStore
|
||||
from synapse.storage.data_stores.main.stats import StatsStore
|
||||
from synapse.storage.data_stores.main.user_directory import (
|
||||
UserDirectoryBackgroundUpdateStore,
|
||||
)
|
||||
from synapse.storage.engines import create_engine
|
||||
from synapse.storage.prepare_database import prepare_database
|
||||
from synapse.util import Clock
|
||||
|
||||
logger = logging.getLogger("synapse_port_db")
|
||||
|
||||
|
|
@ -43,6 +68,7 @@ BOOLEAN_COLUMNS = {
|
|||
"presence_list": ["accepted"],
|
||||
"presence_stream": ["currently_active"],
|
||||
"public_room_list_stream": ["visibility"],
|
||||
"devices": ["hidden"],
|
||||
"device_lists_outbound_pokes": ["sent"],
|
||||
"users_who_share_rooms": ["share_private"],
|
||||
"groups": ["is_public"],
|
||||
|
|
@ -98,33 +124,24 @@ APPEND_ONLY_TABLES = [
|
|||
end_error_exec_info = None
|
||||
|
||||
|
||||
class Store(object):
|
||||
"""This object is used to pull out some of the convenience API from the
|
||||
Storage layer.
|
||||
|
||||
*All* database interactions should go through this object.
|
||||
"""
|
||||
|
||||
def __init__(self, db_pool, engine):
|
||||
self.db_pool = db_pool
|
||||
self.database_engine = engine
|
||||
|
||||
_simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]
|
||||
_simple_insert = SQLBaseStore.__dict__["_simple_insert"]
|
||||
|
||||
_simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"]
|
||||
_simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"]
|
||||
_simple_select_one = SQLBaseStore.__dict__["_simple_select_one"]
|
||||
_simple_select_one_txn = SQLBaseStore.__dict__["_simple_select_one_txn"]
|
||||
_simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"]
|
||||
_simple_select_one_onecol_txn = SQLBaseStore.__dict__[
|
||||
"_simple_select_one_onecol_txn"
|
||||
]
|
||||
|
||||
_simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
|
||||
_simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
|
||||
_simple_update_txn = SQLBaseStore.__dict__["_simple_update_txn"]
|
||||
class Store(
|
||||
ClientIpBackgroundUpdateStore,
|
||||
DeviceInboxBackgroundUpdateStore,
|
||||
DeviceBackgroundUpdateStore,
|
||||
EventsBackgroundUpdatesStore,
|
||||
MediaRepositoryBackgroundUpdateStore,
|
||||
RegistrationBackgroundUpdateStore,
|
||||
RoomMemberBackgroundUpdateStore,
|
||||
SearchBackgroundUpdateStore,
|
||||
StateBackgroundUpdateStore,
|
||||
UserDirectoryBackgroundUpdateStore,
|
||||
StatsStore,
|
||||
):
|
||||
def __init__(self, db_conn, hs):
|
||||
super().__init__(db_conn, hs)
|
||||
self.db_pool = hs.get_db_pool()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def runInteraction(self, desc, func, *args, **kwargs):
|
||||
def r(conn):
|
||||
try:
|
||||
|
|
@ -150,7 +167,8 @@ class Store(object):
|
|||
logger.debug("[TXN FAIL] {%s} %s", desc, e)
|
||||
raise
|
||||
|
||||
return self.db_pool.runWithConnection(r)
|
||||
with PreserveLoggingContext():
|
||||
return (yield self.db_pool.runWithConnection(r))
|
||||
|
||||
def execute(self, f, *args, **kwargs):
|
||||
return self.runInteraction(f.__name__, f, *args, **kwargs)
|
||||
|
|
@ -176,6 +194,25 @@ class Store(object):
|
|||
raise
|
||||
|
||||
|
||||
class MockHomeserver:
|
||||
def __init__(self, config, database_engine, db_conn, db_pool):
|
||||
self.database_engine = database_engine
|
||||
self.db_conn = db_conn
|
||||
self.db_pool = db_pool
|
||||
self.clock = Clock(reactor)
|
||||
self.config = config
|
||||
self.hostname = config.server_name
|
||||
|
||||
def get_db_conn(self):
|
||||
return self.db_conn
|
||||
|
||||
def get_db_pool(self):
|
||||
return self.db_pool
|
||||
|
||||
def get_clock(self):
|
||||
return self.clock
|
||||
|
||||
|
||||
class Porter(object):
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(kwargs)
|
||||
|
|
@ -447,31 +484,75 @@ class Porter(object):
|
|||
|
||||
db_conn.commit()
|
||||
|
||||
return db_conn
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def build_db_store(self, config):
|
||||
"""Builds and returns a database store using the provided configuration.
|
||||
|
||||
Args:
|
||||
config: The database configuration, i.e. a dict following the structure of
|
||||
the "database" section of Synapse's configuration file.
|
||||
|
||||
Returns:
|
||||
The built Store object.
|
||||
"""
|
||||
engine = create_engine(config)
|
||||
|
||||
self.progress.set_state("Preparing %s" % config["name"])
|
||||
conn = self.setup_db(config, engine)
|
||||
|
||||
db_pool = adbapi.ConnectionPool(
|
||||
config["name"], **config["args"]
|
||||
)
|
||||
|
||||
hs = MockHomeserver(self.hs_config, engine, conn, db_pool)
|
||||
|
||||
store = Store(conn, hs)
|
||||
|
||||
yield store.runInteraction(
|
||||
"%s_engine.check_database" % config["name"],
|
||||
engine.check_database,
|
||||
)
|
||||
|
||||
return store
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_background_updates_on_postgres(self):
|
||||
# Manually apply all background updates on the PostgreSQL database.
|
||||
postgres_ready = yield self.postgres_store.has_completed_background_updates()
|
||||
|
||||
if not postgres_ready:
|
||||
# Only say that we're running background updates when there are background
|
||||
# updates to run.
|
||||
self.progress.set_state("Running background updates on PostgreSQL")
|
||||
|
||||
while not postgres_ready:
|
||||
yield self.postgres_store.do_next_background_update(100)
|
||||
postgres_ready = yield (
|
||||
self.postgres_store.has_completed_background_updates()
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(self):
|
||||
try:
|
||||
sqlite_db_pool = adbapi.ConnectionPool(
|
||||
self.sqlite_config["name"], **self.sqlite_config["args"]
|
||||
self.sqlite_store = yield self.build_db_store(self.sqlite_config)
|
||||
|
||||
# Check if all background updates are done, abort if not.
|
||||
updates_complete = yield self.sqlite_store.has_completed_background_updates()
|
||||
if not updates_complete:
|
||||
sys.stderr.write(
|
||||
"Pending background updates exist in the SQLite3 database."
|
||||
" Please start Synapse again and wait until every update has finished"
|
||||
" before running this script.\n"
|
||||
)
|
||||
defer.returnValue(None)
|
||||
|
||||
self.postgres_store = yield self.build_db_store(
|
||||
self.hs_config.database_config
|
||||
)
|
||||
|
||||
postgres_db_pool = adbapi.ConnectionPool(
|
||||
self.postgres_config["name"], **self.postgres_config["args"]
|
||||
)
|
||||
|
||||
sqlite_engine = create_engine(sqlite_config)
|
||||
postgres_engine = create_engine(postgres_config)
|
||||
|
||||
self.sqlite_store = Store(sqlite_db_pool, sqlite_engine)
|
||||
self.postgres_store = Store(postgres_db_pool, postgres_engine)
|
||||
|
||||
yield self.postgres_store.execute(postgres_engine.check_database)
|
||||
|
||||
# Step 1. Set up databases.
|
||||
self.progress.set_state("Preparing SQLite3")
|
||||
self.setup_db(sqlite_config, sqlite_engine)
|
||||
|
||||
self.progress.set_state("Preparing PostgreSQL")
|
||||
self.setup_db(postgres_config, postgres_engine)
|
||||
yield self.run_background_updates_on_postgres()
|
||||
|
||||
self.progress.set_state("Creating port tables")
|
||||
|
||||
|
|
@ -563,6 +644,8 @@ class Porter(object):
|
|||
def conv(j, col):
|
||||
if j in bool_cols:
|
||||
return bool(col)
|
||||
if isinstance(col, bytes):
|
||||
return bytearray(col)
|
||||
elif isinstance(col, string_types) and "\0" in col:
|
||||
logger.warn(
|
||||
"DROPPING ROW: NUL value in table %s col %s: %r",
|
||||
|
|
@ -926,18 +1009,24 @@ if __name__ == "__main__":
|
|||
},
|
||||
}
|
||||
|
||||
postgres_config = yaml.safe_load(args.postgres_config)
|
||||
hs_config = yaml.safe_load(args.postgres_config)
|
||||
|
||||
if "database" in postgres_config:
|
||||
postgres_config = postgres_config["database"]
|
||||
if "database" not in hs_config:
|
||||
sys.stderr.write("The configuration file must have a 'database' section.\n")
|
||||
sys.exit(4)
|
||||
|
||||
postgres_config = hs_config["database"]
|
||||
|
||||
if "name" not in postgres_config:
|
||||
sys.stderr.write("Malformed database config: no 'name'")
|
||||
sys.stderr.write("Malformed database config: no 'name'\n")
|
||||
sys.exit(2)
|
||||
if postgres_config["name"] != "psycopg2":
|
||||
sys.stderr.write("Database must use 'psycopg2' connector.")
|
||||
sys.stderr.write("Database must use the 'psycopg2' connector.\n")
|
||||
sys.exit(3)
|
||||
|
||||
config = HomeServerConfig()
|
||||
config.parse_config_dict(hs_config, "", "")
|
||||
|
||||
def start(stdscr=None):
|
||||
if stdscr:
|
||||
progress = CursesProgress(stdscr)
|
||||
|
|
@ -946,9 +1035,9 @@ if __name__ == "__main__":
|
|||
|
||||
porter = Porter(
|
||||
sqlite_config=sqlite_config,
|
||||
postgres_config=postgres_config,
|
||||
progress=progress,
|
||||
batch_size=args.batch_size,
|
||||
hs_config=config,
|
||||
)
|
||||
|
||||
reactor.callWhenRunning(porter.run)
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ try:
|
|||
except ImportError:
|
||||
pass
|
||||
|
||||
__version__ = "1.4.1"
|
||||
__version__ = "1.5.0"
|
||||
|
||||
if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
|
||||
# We import here so that we don't have to install a bunch of deps when
|
||||
|
|
|
|||
|
|
@ -25,7 +25,13 @@ from twisted.internet import defer
|
|||
import synapse.logging.opentracing as opentracing
|
||||
import synapse.types
|
||||
from synapse import event_auth
|
||||
from synapse.api.constants import EventTypes, JoinRules, Membership, UserTypes
|
||||
from synapse.api.constants import (
|
||||
EventTypes,
|
||||
JoinRules,
|
||||
LimitBlockingTypes,
|
||||
Membership,
|
||||
UserTypes,
|
||||
)
|
||||
from synapse.api.errors import (
|
||||
AuthError,
|
||||
Codes,
|
||||
|
|
@ -726,7 +732,7 @@ class Auth(object):
|
|||
self.hs.config.hs_disabled_message,
|
||||
errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
|
||||
admin_contact=self.hs.config.admin_contact,
|
||||
limit_type=self.hs.config.hs_disabled_limit_type,
|
||||
limit_type=LimitBlockingTypes.HS_DISABLED,
|
||||
)
|
||||
if self.hs.config.limit_usage_by_mau is True:
|
||||
assert not (user_id and threepid)
|
||||
|
|
@ -759,5 +765,5 @@ class Auth(object):
|
|||
"Monthly Active User Limit Exceeded",
|
||||
admin_contact=self.hs.config.admin_contact,
|
||||
errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
|
||||
limit_type="monthly_active_user",
|
||||
limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -131,3 +131,10 @@ class RelationTypes(object):
|
|||
ANNOTATION = "m.annotation"
|
||||
REPLACE = "m.replace"
|
||||
REFERENCE = "m.reference"
|
||||
|
||||
|
||||
class LimitBlockingTypes(object):
|
||||
"""Reasons that a server may be blocked"""
|
||||
|
||||
MONTHLY_ACTIVE_USER = "monthly_active_user"
|
||||
HS_DISABLED = "hs_disabled"
|
||||
|
|
|
|||
|
|
@ -300,7 +300,7 @@ class RegistrationConfig(Config):
|
|||
# If a delegate is specified, the config option public_baseurl must also be filled out.
|
||||
#
|
||||
account_threepid_delegates:
|
||||
#email: https://example.com # Delegate email sending to example.org
|
||||
#email: https://example.com # Delegate email sending to example.com
|
||||
#msisdn: http://localhost:8090 # Delegate SMS sending to this local process
|
||||
|
||||
# Users who register on this homeserver will automatically be joined
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ class ServerConfig(Config):
|
|||
)
|
||||
|
||||
self.mau_trial_days = config.get("mau_trial_days", 0)
|
||||
self.mau_limit_alerting = config.get("mau_limit_alerting", True)
|
||||
|
||||
# How long to keep redacted events in the database in unredacted form
|
||||
# before redacting them.
|
||||
|
|
@ -192,7 +193,6 @@ class ServerConfig(Config):
|
|||
# Options to disable HS
|
||||
self.hs_disabled = config.get("hs_disabled", False)
|
||||
self.hs_disabled_message = config.get("hs_disabled_message", "")
|
||||
self.hs_disabled_limit_type = config.get("hs_disabled_limit_type", "")
|
||||
|
||||
# Admin uri to direct users at should their instance become blocked
|
||||
# due to resource constraints
|
||||
|
|
@ -675,7 +675,6 @@ class ServerConfig(Config):
|
|||
#
|
||||
#hs_disabled: false
|
||||
#hs_disabled_message: 'Human readable reason for why the HS is blocked'
|
||||
#hs_disabled_limit_type: 'error code(str), to help clients decode reason'
|
||||
|
||||
# Monthly Active User Blocking
|
||||
#
|
||||
|
|
@ -695,9 +694,16 @@ class ServerConfig(Config):
|
|||
# sign up in a short space of time never to return after their initial
|
||||
# session.
|
||||
#
|
||||
# 'mau_limit_alerting' is a means of limiting client side alerting
|
||||
# should the mau limit be reached. This is useful for small instances
|
||||
# where the admin has 5 mau seats (say) for 5 specific people and no
|
||||
# interest increasing the mau limit further. Defaults to True, which
|
||||
# means that alerting is enabled
|
||||
#
|
||||
#limit_usage_by_mau: false
|
||||
#max_mau_value: 50
|
||||
#mau_trial_days: 2
|
||||
#mau_limit_alerting: false
|
||||
|
||||
# If enabled, the metrics for the number of monthly active users will
|
||||
# be populated, however no one will be limited. If limit_usage_by_mau
|
||||
|
|
|
|||
|
|
@ -125,8 +125,10 @@ def compute_event_signature(event_dict, signature_name, signing_key):
|
|||
redact_json = prune_event_dict(event_dict)
|
||||
redact_json.pop("age_ts", None)
|
||||
redact_json.pop("unsigned", None)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug("Signing event: %s", encode_canonical_json(redact_json))
|
||||
redact_json = sign_json(redact_json, signature_name, signing_key)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug("Signed event: %s", encode_canonical_json(redact_json))
|
||||
return redact_json["signatures"]
|
||||
|
||||
|
|
|
|||
|
|
@ -493,8 +493,7 @@ def _check_power_levels(event, auth_events):
|
|||
new_level_too_big = new_level is not None and new_level > user_level
|
||||
if old_level_too_big or new_level_too_big:
|
||||
raise AuthError(
|
||||
403,
|
||||
"You don't have permission to add ops level greater " "than your own",
|
||||
403, "You don't have permission to add ops level greater than your own"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue