forked from MirrorHub/synapse
Merge remote-tracking branch 'upstream/release-v1.85'
This commit is contained in:
Tulir Asokan
commit
a7d2979971
|
|
@ -1,49 +0,0 @@
|
|||
name: Write changelog for dependabot PR
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- reopened # For debugging!
|
||||
|
||||
permissions:
|
||||
# Needed to be able to push the commit. See
|
||||
# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
|
||||
# for a similar example
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
add-changelog:
|
||||
runs-on: 'ubuntu-latest'
|
||||
if: ${{ github.actor == 'dependabot[bot]' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
- name: Write, commit and push changelog
|
||||
env:
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
echo "${PR_TITLE}." > "changelog.d/${PR_NUMBER}".misc
|
||||
git add changelog.d
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config user.name "GitHub Actions"
|
||||
git commit -m "Changelog"
|
||||
git push
|
||||
shell: bash
|
||||
# The `git push` above does not trigger CI on the dependabot PR.
|
||||
#
|
||||
# By default, workflows can't trigger other workflows when they're just using the
|
||||
# default `GITHUB_TOKEN` access token. (This is intended to stop you from writing
|
||||
# recursive workflow loops by accident, because that'll get very expensive very
|
||||
# quickly.) Instead, you have to manually call out to another workflow, or else
|
||||
# make your changes (i.e. the `git push` above) using a personal access token.
|
||||
# See
|
||||
# https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
|
||||
#
|
||||
# I have tried and failed to find a way to trigger CI on the "merge ref" of the PR.
|
||||
# See git commit history for previous attempts. If anyone desperately wants to try
|
||||
# again in the future, make a matrix-bot account and use its access token to git push.
|
||||
|
||||
# THIS WORKFLOW HAS WRITE PERMISSIONS---do not add other jobs here unless they
|
||||
# are sufficiently locked down to dependabot only as above.
|
||||
|
|
@ -314,8 +314,9 @@ jobs:
|
|||
# There aren't wheels for some of the older deps, so we need to install
|
||||
# their build dependencies
|
||||
- run: |
|
||||
sudo apt-get -qq update
|
||||
sudo apt-get -qq install build-essential libffi-dev python-dev \
|
||||
libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
|
||||
libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
|
|
|
|||
94
CHANGES.md
94
CHANGES.md
|
|
@ -1,3 +1,97 @@
|
|||
Synapse 1.85.0rc1 (2023-05-30)
|
||||
==============================
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
- Improve performance of backfill requests by performing backfill of previously failed requests in the background. ([\#15585](https://github.com/matrix-org/synapse/issues/15585))
|
||||
- Add a new [admin API](https://matrix-org.github.io/synapse/v1.85/usage/administration/admin_api/index.html) to [create a new device for a user](https://matrix-org.github.io/synapse/v1.85/admin_api/user_admin_api.html#create-a-device). ([\#15611](https://github.com/matrix-org/synapse/issues/15611))
|
||||
- Add Unix socket support for Redis connections. Contributed by Jason Little. ([\#15644](https://github.com/matrix-org/synapse/issues/15644))
|
||||
|
||||
|
||||
Bugfixes
|
||||
--------
|
||||
|
||||
- Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar). ([\#15464](https://github.com/matrix-org/synapse/issues/15464))
|
||||
- Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL. ([\#15601](https://github.com/matrix-org/synapse/issues/15601))
|
||||
- Fix a long-standing bug where filters with multiple backslashes were rejected. ([\#15607](https://github.com/matrix-org/synapse/issues/15607))
|
||||
- Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted. ([\#15614](https://github.com/matrix-org/synapse/issues/15614))
|
||||
- Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled). ([\#15624](https://github.com/matrix-org/synapse/issues/15624))
|
||||
- Fix a long-standing bug where deactivated users were able to login in uncommon situations. ([\#15634](https://github.com/matrix-org/synapse/issues/15634))
|
||||
|
||||
|
||||
Improved Documentation
|
||||
----------------------
|
||||
|
||||
- Warn users that at least 3.75GB of space is needed for the nix Synapse development environment. ([\#15613](https://github.com/matrix-org/synapse/issues/15613))
|
||||
- Remove outdated comment from the generated and sample homeserver log configs. ([\#15648](https://github.com/matrix-org/synapse/issues/15648))
|
||||
- Improve contributor docs to make it more clear that Rust is a necessary prerequisite. Contributed by @grantm. ([\#15668](https://github.com/matrix-org/synapse/issues/15668))
|
||||
|
||||
|
||||
Deprecations and Removals
|
||||
-------------------------
|
||||
|
||||
- Remove the old version of the R30 (30-day retained users) phone-home metric. ([\#10428](https://github.com/matrix-org/synapse/issues/10428))
|
||||
|
||||
|
||||
Internal Changes
|
||||
----------------
|
||||
|
||||
- Create dependabot changelogs at release time. ([\#15481](https://github.com/matrix-org/synapse/issues/15481))
|
||||
- Add not null constraint to column `full_user_id` of tables `profiles` and `user_filters`. ([\#15537](https://github.com/matrix-org/synapse/issues/15537))
|
||||
- Allow connecting to HTTP Replication Endpoints by using `worker_name` when constructing the request. ([\#15578](https://github.com/matrix-org/synapse/issues/15578))
|
||||
- Make the `thread_id` column on `event_push_actions`, `event_push_actions_staging`, and `event_push_summary` non-null. ([\#15597](https://github.com/matrix-org/synapse/issues/15597))
|
||||
- Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible. ([\#15602](https://github.com/matrix-org/synapse/issues/15602))
|
||||
- Fix subscriptable type usage in Python <3.9. ([\#15604](https://github.com/matrix-org/synapse/issues/15604))
|
||||
- Update internal terminology. ([\#15606](https://github.com/matrix-org/synapse/issues/15606), [\#15620](https://github.com/matrix-org/synapse/issues/15620))
|
||||
- Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing. ([\#15610](https://github.com/matrix-org/synapse/issues/15610), [\#15647](https://github.com/matrix-org/synapse/issues/15647))
|
||||
- Trace how many new events from the backfill response we need to process. ([\#15633](https://github.com/matrix-org/synapse/issues/15633))
|
||||
- Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s. ([\#15615](https://github.com/matrix-org/synapse/issues/15615))
|
||||
- Update Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) implementation to match new proposal text. ([\#15621](https://github.com/matrix-org/synapse/issues/15621))
|
||||
- Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706)). ([\#15625](https://github.com/matrix-org/synapse/issues/15625))
|
||||
- Fix the olddeps CI. ([\#15626](https://github.com/matrix-org/synapse/issues/15626))
|
||||
- Remove duplicate timestamp from test logs (`_trial_temp/test.log`). ([\#15636](https://github.com/matrix-org/synapse/issues/15636))
|
||||
- Fix two memory leaks in `trial` test runs. ([\#15630](https://github.com/matrix-org/synapse/issues/15630))
|
||||
- Limit the size of the `HomeServerConfig` cache in trial test runs. ([\#15646](https://github.com/matrix-org/synapse/issues/15646))
|
||||
- Improve type hints. ([\#15658](https://github.com/matrix-org/synapse/issues/15658), [\#15659](https://github.com/matrix-org/synapse/issues/15659))
|
||||
- Add requesting user id parameter to key claim methods in `TransportLayerClient`. ([\#15663](https://github.com/matrix-org/synapse/issues/15663))
|
||||
- Speed up rebuilding of the user directory for local users. ([\#15665](https://github.com/matrix-org/synapse/issues/15665))
|
||||
- Implement "option 2" for [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820): Room version 11. ([\#15666](https://github.com/matrix-org/synapse/issues/15666), [\#15678](https://github.com/matrix-org/synapse/issues/15678))
|
||||
|
||||
### Updates to locked dependencies
|
||||
|
||||
* Bump furo from 2023.3.27 to 2023.5.20. ([\#15642](https://github.com/matrix-org/synapse/issues/15642))
|
||||
* Bump log from 0.4.17 to 0.4.18. ([\#15681](https://github.com/matrix-org/synapse/issues/15681))
|
||||
* Bump prometheus-client from 0.16.0 to 0.17.0. ([\#15682](https://github.com/matrix-org/synapse/issues/15682))
|
||||
* Bump pydantic from 1.10.7 to 1.10.8. ([\#15685](https://github.com/matrix-org/synapse/issues/15685))
|
||||
* Bump pygithub from 1.58.1 to 1.58.2. ([\#15643](https://github.com/matrix-org/synapse/issues/15643))
|
||||
* Bump requests from 2.28.2 to 2.31.0. ([\#15651](https://github.com/matrix-org/synapse/issues/15651))
|
||||
* Bump sphinx from 6.1.3 to 6.2.1. ([\#15641](https://github.com/matrix-org/synapse/issues/15641))
|
||||
* Bump types-bleach from 6.0.0.1 to 6.0.0.3. ([\#15686](https://github.com/matrix-org/synapse/issues/15686))
|
||||
* Bump types-pillow from 9.5.0.2 to 9.5.0.4. ([\#15640](https://github.com/matrix-org/synapse/issues/15640))
|
||||
* Bump types-pyyaml from 6.0.12.9 to 6.0.12.10. ([\#15683](https://github.com/matrix-org/synapse/issues/15683))
|
||||
* Bump types-requests from 2.30.0.0 to 2.31.0.0. ([\#15684](https://github.com/matrix-org/synapse/issues/15684))
|
||||
* Bump types-setuptools from 67.7.0.2 to 67.8.0.0. ([\#15639](https://github.com/matrix-org/synapse/issues/15639))
|
||||
|
||||
Synapse 1.84.1 (2023-05-26)
|
||||
===========================
|
||||
|
||||
This patch release fixes a major issue with homeservers that do not have an `instance_map` defined but which do use workers.
|
||||
If you have already upgraded to Synapse 1.84.0 and your homeserver is working normally, then there is no need to update to this patch release.
|
||||
|
||||
|
||||
Bugfixes
|
||||
--------
|
||||
|
||||
- Fix a bug introduced in Synapse v1.84.0 where workers do not start up when no `instance_map` was provided. ([\#15672](https://github.com/matrix-org/synapse/issues/15672))
|
||||
|
||||
|
||||
Internal Changes
|
||||
----------------
|
||||
|
||||
- Add `dch` and `notify-send` to the development Nix flake so that the release script can be used. ([\#15673](https://github.com/matrix-org/synapse/issues/15673))
|
||||
|
||||
|
||||
Synapse 1.84.0 (2023-05-23)
|
||||
===========================
|
||||
|
||||
|
|
|
|||
|
|
@ -132,12 +132,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.17"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,15 @@
|
|||
matrix-synapse-py3 (1.85.0~rc1) stable; urgency=medium
|
||||
|
||||
* New Synapse release 1.85.0rc1.
|
||||
|
||||
-- Synapse Packaging team <packages@matrix.org> Tue, 30 May 2023 13:56:54 +0100
|
||||
|
||||
matrix-synapse-py3 (1.84.1) stable; urgency=medium
|
||||
|
||||
* New Synapse release 1.84.1.
|
||||
|
||||
-- Synapse Packaging team <packages@matrix.org> Fri, 26 May 2023 16:15:30 +0100
|
||||
|
||||
matrix-synapse-py3 (1.84.0) stable; urgency=medium
|
||||
|
||||
* New Synapse release 1.84.0.
|
||||
|
|
|
|||
|
|
@ -813,6 +813,33 @@ The following fields are returned in the JSON response body:
|
|||
|
||||
- `total` - Total number of user's devices.
|
||||
|
||||
### Create a device
|
||||
|
||||
Creates a new device for a specific `user_id` and `device_id`. Does nothing if the `device_id`
|
||||
exists already.
|
||||
|
||||
The API is:
|
||||
|
||||
```
|
||||
POST /_synapse/admin/v2/users/<user_id>/devices
|
||||
|
||||
{
|
||||
"device_id": "QBUAZIFURK"
|
||||
}
|
||||
```
|
||||
|
||||
An empty JSON dict is returned.
|
||||
|
||||
**Parameters**
|
||||
|
||||
The following parameters should be set in the URL:
|
||||
|
||||
- `user_id` - fully qualified: for example, `@user:server.com`.
|
||||
|
||||
The following fields are required in the JSON request body:
|
||||
|
||||
- `device_id` - The device ID to create.
|
||||
|
||||
### Delete multiple devices
|
||||
Deletes the given devices for a specific `user_id`, and invalidates
|
||||
any access token associated with them.
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ on Windows is not officially supported.
|
|||
|
||||
The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://www.python.org/downloads/). Your Python also needs support for [virtual environments](https://docs.python.org/3/library/venv.html). This is usually built-in, but some Linux distributions like Debian and Ubuntu split it out into its own package. Running `sudo apt install python3-venv` should be enough.
|
||||
|
||||
A recent version of the Rust compiler is needed to build the native modules. The
|
||||
easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
|
||||
|
||||
Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.
|
||||
|
||||
Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`.
|
||||
|
|
@ -30,9 +33,6 @@ The source code of Synapse is hosted on GitHub. You will also need [a recent ver
|
|||
|
||||
For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
|
||||
|
||||
A recent version of the Rust compiler is needed to build the native modules. The
|
||||
easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
|
||||
|
||||
|
||||
# 3. Get the source.
|
||||
|
||||
|
|
@ -53,6 +53,11 @@ can find many good git tutorials on the web.
|
|||
|
||||
# 4. Install the dependencies
|
||||
|
||||
|
||||
Before installing the Python dependencies, make sure you have installed a recent version
|
||||
of Rust (see the "What do I need?" section above). The easiest way of installing the
|
||||
latest version is to use [rustup](https://rustup.rs/).
|
||||
|
||||
Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies
|
||||
and development environment. Once you have installed Python 3 and added the
|
||||
source, you should install `poetry`.
|
||||
|
|
@ -76,7 +81,8 @@ cd path/where/you/have/cloned/the/repository
|
|||
poetry install --extras all
|
||||
```
|
||||
|
||||
This will install the runtime and developer dependencies for the project.
|
||||
This will install the runtime and developer dependencies for the project. Be sure to check
|
||||
that the `poetry install` step completed cleanly.
|
||||
|
||||
## Running Synapse via poetry
|
||||
|
||||
|
|
@ -84,14 +90,31 @@ To start a local instance of Synapse in the locked poetry environment, create a
|
|||
|
||||
```sh
|
||||
cp docs/sample_config.yaml homeserver.yaml
|
||||
cp docs/sample_log_config.yaml log_config.yaml
|
||||
```
|
||||
|
||||
Now edit homeserver.yaml, and run Synapse with:
|
||||
Now edit `homeserver.yaml`, things you might want to change include:
|
||||
|
||||
- Set a `server_name`
|
||||
- Adjusting paths to be correct for your system like the `log_config` to point to the log config you just copied
|
||||
- Using a [PostgreSQL database instead of SQLite](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#database)
|
||||
- Adding a [`registration_shared_secret`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#registration_shared_secret) so you can use [`register_new_matrix_user` command](https://matrix-org.github.io/synapse/latest/setup/installation.html#registering-a-user).
|
||||
|
||||
And then run Synapse with the following command:
|
||||
|
||||
```sh
|
||||
poetry run python -m synapse.app.homeserver -c homeserver.yaml
|
||||
```
|
||||
|
||||
If you get an error like the following:
|
||||
|
||||
```
|
||||
importlib.metadata.PackageNotFoundError: matrix-synapse
|
||||
```
|
||||
|
||||
this probably indicates that the `poetry install` step did not complete cleanly - go back and
|
||||
resolve any issues and re-run until successful.
|
||||
|
||||
# 5. Get in touch.
|
||||
|
||||
Join our developer community on Matrix: [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)!
|
||||
|
|
|
|||
|
|
@ -260,15 +260,17 @@ doesn't require poetry. (It's what we use in CI too). However, you could try
|
|||
|
||||
## ...handle a Dependabot pull request?
|
||||
|
||||
Synapse uses Dependabot to keep the `poetry.lock` file up-to-date. When it
|
||||
creates a pull request a GitHub Action will run to automatically create a changelog
|
||||
file. Ensure that:
|
||||
Synapse uses Dependabot to keep the `poetry.lock` and `Cargo.lock` file
|
||||
up-to-date with the latest releases of our dependencies. The changelog check is
|
||||
omitted for Dependabot PRs; the release script will include them in the
|
||||
changelog.
|
||||
|
||||
When reviewing a dependabot PR, ensure that:
|
||||
|
||||
* the lockfile changes look reasonable;
|
||||
* the upstream changelog file (linked in the description) doesn't include any
|
||||
breaking changes;
|
||||
* continuous integration passes (due to permissions, the GitHub Actions run on
|
||||
the changelog commit will fail, look at the initial commit of the pull request);
|
||||
* continuous integration passes.
|
||||
|
||||
In particular, any updates to the type hints (usually packages which start with `types-`)
|
||||
should be safe to merge if linting passes.
|
||||
|
|
|
|||
|
|
@ -46,6 +46,9 @@ instead.
|
|||
|
||||
If the authentication is unsuccessful, the module must return `None`.
|
||||
|
||||
Note that the user is not automatically registered, the `register_user(..)` method of
|
||||
the [module API](writing_a_module.html) can be used to lazily create users.
|
||||
|
||||
If multiple modules register an auth checker for the same login type but with different
|
||||
fields, Synapse will refuse to start.
|
||||
|
||||
|
|
|
|||
|
|
@ -30,12 +30,6 @@ minimal.
|
|||
|
||||
See [the TCP replication documentation](tcp_replication.md).
|
||||
|
||||
### The Slaved DataStore
|
||||
|
||||
There are read-only version of the synapse storage layer in
|
||||
`synapse/replication/slave/storage` that use the response of the
|
||||
replication API to invalidate their caches.
|
||||
|
||||
### The TCP Replication Module
|
||||
Information about how the tcp replication module is structured, including how
|
||||
the classes interact, can be found in
|
||||
|
|
|
|||
|
|
@ -68,9 +68,7 @@ root:
|
|||
# Write logs to the `buffer` handler, which will buffer them together in memory,
|
||||
# then write them to a file.
|
||||
#
|
||||
# Replace "buffer" with "console" to log to stderr instead. (Note that you'll
|
||||
# also need to update the configuration for the `twisted` logger above, in
|
||||
# this case.)
|
||||
# Replace "buffer" with "console" to log to stderr instead.
|
||||
#
|
||||
handlers: [buffer]
|
||||
|
||||
|
|
|
|||
|
|
@ -42,11 +42,6 @@ The following statistics are sent to the configured reporting endpoint:
|
|||
| `daily_e2ee_messages` | int | The number of (state) events with the type `m.room.encrypted` seen in the last 24 hours. |
|
||||
| `daily_sent_messages` | int | The number of (state) events sent by a local user with the type `m.room.message` seen in the last 24 hours. |
|
||||
| `daily_sent_e2ee_messages` | int | The number of (state) events sent by a local user with the type `m.room.encrypted` seen in the last 24 hours. |
|
||||
| `r30_users_all` | int | The number of 30 day retained users, defined as users who have created their accounts more than 30 days ago, where they were last seen at most 30 days ago and where those two timestamps are over 30 days apart. Includes clients that do not fit into the below r30 client types. |
|
||||
| `r30_users_android` | int | The number of 30 day retained users, as defined above. Filtered only to clients with "Android" in the user agent string. |
|
||||
| `r30_users_ios` | int | The number of 30 day retained users, as defined above. Filtered only to clients with "iOS" in the user agent string. |
|
||||
| `r30_users_electron` | int | The number of 30 day retained users, as defined above. Filtered only to clients with "Electron" in the user agent string. |
|
||||
| `r30_users_web` | int | The number of 30 day retained users, as defined above. Filtered only to clients with "Mozilla" or "Gecko" in the user agent string. |
|
||||
| `r30v2_users_all` | int | The number of 30 day retained users, with a revised algorithm. Defined as users that appear more than once in the past 60 days, and have more than 30 days between the most and least recent appearances in the past 60 days. Includes clients that do not fit into the below r30 client types. |
|
||||
| `r30v2_users_android` | int | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "android" (case-insensitive) in the user agent string. |
|
||||
| `r30v2_users_ios` | int | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "ios" (case-insensitive) in the user agent string. |
|
||||
|
|
|
|||
|
|
@ -3979,6 +3979,8 @@ This setting has the following sub-options:
|
|||
* `enabled`: whether to use Redis support. Defaults to false.
|
||||
* `host` and `port`: Optional host and port to use to connect to redis. Defaults to
|
||||
localhost and 6379
|
||||
* `path`: The full path to a local Unix socket file. **If this is used, `host` and
|
||||
`port` are ignored.** Defaults to `/tmp/redis.sock'
|
||||
* `password`: Optional password if configured on the Redis instance.
|
||||
* `dbid`: Optional redis dbid if needs to connect to specific redis logical db.
|
||||
* `use_tls`: Whether to use tls connection. Defaults to false.
|
||||
|
|
@ -3991,6 +3993,8 @@ This setting has the following sub-options:
|
|||
|
||||
_Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_
|
||||
|
||||
_Changed in Synapse 1.85.0: Added path option to use a local Unix socket_
|
||||
|
||||
Example configuration:
|
||||
```yaml
|
||||
redis:
|
||||
|
|
|
|||
57
flake.nix
57
flake.nix
|
|
@ -1,35 +1,30 @@
|
|||
# A nix flake that sets up a complete Synapse development environment. Dependencies
|
||||
# A Nix flake that sets up a complete Synapse development environment. Dependencies
|
||||
# for the SyTest (https://github.com/matrix-org/sytest) and Complement
|
||||
# (https://github.com/matrix-org/complement) Matrix homeserver test suites are also
|
||||
# installed automatically.
|
||||
#
|
||||
# You must have already installed nix (https://nixos.org) on your system to use this.
|
||||
# nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
|
||||
# directly supported, but nix can be installed inside of WSL2 or even Docker
|
||||
# You must have already installed Nix (https://nixos.org) on your system to use this.
|
||||
# Nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
|
||||
# directly supported, but Nix can be installed inside of WSL2 or even Docker
|
||||
# containers. Please refer to https://nixos.org/download for details.
|
||||
#
|
||||
# You must also enable support for flakes in Nix. See the following for how to
|
||||
# do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes
|
||||
#
|
||||
# Be warned: you'll need over 3.75 GB of free space to download all the dependencies.
|
||||
#
|
||||
# Usage:
|
||||
#
|
||||
# With nix installed, navigate to the directory containing this flake and run
|
||||
# With Nix installed, navigate to the directory containing this flake and run
|
||||
# `nix develop --impure`. The `--impure` is necessary in order to store state
|
||||
# locally from "services", such as PostgreSQL and Redis.
|
||||
#
|
||||
# You should now be dropped into a new shell with all programs and dependencies
|
||||
# availabile to you!
|
||||
#
|
||||
# You can start up pre-configured, local PostgreSQL and Redis instances by
|
||||
# You can start up pre-configured local Synapse, PostgreSQL and Redis instances by
|
||||
# running: `devenv up`. To stop them, use Ctrl-C.
|
||||
#
|
||||
# A PostgreSQL database called 'synapse' will be set up for you, along with
|
||||
# a PostgreSQL user named 'synapse_user'.
|
||||
# The 'host' can be found by running `echo $PGHOST` with the development
|
||||
# shell activated. Use these values to configure your Synapse to connect
|
||||
# to the local PostgreSQL database. You do not need to specify a password.
|
||||
# https://matrix-org.github.io/synapse/latest/postgres
|
||||
#
|
||||
# All state (the venv, postgres and redis data and config) are stored in
|
||||
# .devenv/state. Deleting a file from here and then re-entering the shell
|
||||
# will recreate these files from scratch.
|
||||
|
|
@ -66,7 +61,7 @@
|
|||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
in {
|
||||
# Everything is configured via devenv - a nix module for creating declarative
|
||||
# Everything is configured via devenv - a Nix module for creating declarative
|
||||
# developer environments. See https://devenv.sh/reference/options/ for a list
|
||||
# of all possible options.
|
||||
default = devenv.lib.mkShell {
|
||||
|
|
@ -100,6 +95,10 @@
|
|||
|
||||
# For building the Synapse documentation website.
|
||||
mdbook
|
||||
|
||||
# For releasing Synapse
|
||||
debian-devscripts # (`dch` for manipulating the Debian changelog)
|
||||
libnotify # (the release script uses `notify-send` to tell you when CI jobs are done)
|
||||
];
|
||||
|
||||
# Install Python and manage a virtualenv with Poetry.
|
||||
|
|
@ -153,11 +152,39 @@
|
|||
# Redis is needed in order to run Synapse in worker mode.
|
||||
services.redis.enable = true;
|
||||
|
||||
# Configure and start Synapse. Before starting Synapse, this shell code:
|
||||
# * generates a default homeserver.yaml config file if one does not exist, and
|
||||
# * ensures a directory containing two additional homeserver config files exists;
|
||||
# one to configure using the development environment's PostgreSQL as the
|
||||
# database backend and another for enabling Redis support.
|
||||
process.before = ''
|
||||
python -m synapse.app.homeserver -c homeserver.yaml --generate-config --server-name=synapse.dev --report-stats=no
|
||||
mkdir -p homeserver-config-overrides.d
|
||||
cat > homeserver-config-overrides.d/database.yaml << EOF
|
||||
## Do not edit this file. This file is generated by flake.nix
|
||||
database:
|
||||
name: psycopg2
|
||||
args:
|
||||
user: synapse_user
|
||||
database: synapse
|
||||
host: $PGHOST
|
||||
cp_min: 5
|
||||
cp_max: 10
|
||||
EOF
|
||||
cat > homeserver-config-overrides.d/redis.yaml << EOF
|
||||
## Do not edit this file. This file is generated by flake.nix
|
||||
redis:
|
||||
enabled: true
|
||||
EOF
|
||||
'';
|
||||
# Start synapse when `devenv up` is run.
|
||||
processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
|
||||
|
||||
# Define the perl modules we require to run SyTest.
|
||||
#
|
||||
# This list was compiled by cross-referencing https://metacpan.org/
|
||||
# with the modules defined in './cpanfile' and then finding the
|
||||
# corresponding nix packages on https://search.nixos.org/packages.
|
||||
# corresponding Nix packages on https://search.nixos.org/packages.
|
||||
#
|
||||
# This was done until `./install-deps.pl --dryrun` produced no output.
|
||||
env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [
|
||||
|
|
|
|||
6
mypy.ini
6
mypy.ini
|
|
@ -13,6 +13,9 @@ no_implicit_optional = True
|
|||
disallow_untyped_defs = True
|
||||
strict_equality = True
|
||||
warn_redundant_casts = True
|
||||
# Run mypy type checking with the minimum supported Python version to catch new usage
|
||||
# that isn't backwards-compatible (types, overloads, etc).
|
||||
python_version = 3.8
|
||||
|
||||
files =
|
||||
docker/,
|
||||
|
|
@ -29,9 +32,6 @@ warn_unused_ignores = False
|
|||
[mypy-synapse.util.caches.treecache]
|
||||
disallow_untyped_defs = False
|
||||
|
||||
[mypy-tests.util.caches.test_descriptors]
|
||||
disallow_untyped_defs = False
|
||||
|
||||
;; Dependencies without annotations
|
||||
;; Before ignoring a module, check to see if type stubs are available.
|
||||
;; The `typeshed` project maintains stubs here:
|
||||
|
|
|
|||
|
|
@ -580,20 +580,20 @@ dev = ["Sphinx", "coverage", "flake8", "lxml", "lxml-stubs", "memory-profiler",
|
|||
|
||||
[[package]]
|
||||
name = "furo"
|
||||
version = "2023.3.27"
|
||||
version = "2023.5.20"
|
||||
description = "A clean customisable Sphinx documentation theme."
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "furo-2023.3.27-py3-none-any.whl", hash = "sha256:4ab2be254a2d5e52792d0ca793a12c35582dd09897228a6dd47885dabd5c9521"},
|
||||
{file = "furo-2023.3.27.tar.gz", hash = "sha256:b99e7867a5cc833b2b34d7230631dd6558c7a29f93071fdbb5709634bb33c5a5"},
|
||||
{file = "furo-2023.5.20-py3-none-any.whl", hash = "sha256:594a8436ddfe0c071f3a9e9a209c314a219d8341f3f1af33fdf7c69544fab9e6"},
|
||||
{file = "furo-2023.5.20.tar.gz", hash = "sha256:40e09fa17c6f4b22419d122e933089226dcdb59747b5b6c79363089827dea16f"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
beautifulsoup4 = "*"
|
||||
pygments = ">=2.7"
|
||||
sphinx = ">=5.0,<7.0"
|
||||
sphinx = ">=6.0,<8.0"
|
||||
sphinx-basic-ng = "*"
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1781,14 +1781,14 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytes
|
|||
|
||||
[[package]]
|
||||
name = "prometheus-client"
|
||||
version = "0.16.0"
|
||||
version = "0.17.0"
|
||||
description = "Python client for the Prometheus monitoring system."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "prometheus_client-0.16.0-py3-none-any.whl", hash = "sha256:0836af6eb2c8f4fed712b2f279f6c0a8bbab29f9f4aa15276b91c7cb0d1616ab"},
|
||||
{file = "prometheus_client-0.16.0.tar.gz", hash = "sha256:a03e35b359f14dd1630898543e2120addfdeacd1a6069c1367ae90fd93ad3f48"},
|
||||
{file = "prometheus_client-0.17.0-py3-none-any.whl", hash = "sha256:a77b708cf083f4d1a3fb3ce5c95b4afa32b9c521ae363354a4a910204ea095ce"},
|
||||
{file = "prometheus_client-0.17.0.tar.gz", hash = "sha256:9c3b26f1535945e85b8934fb374678d263137b78ef85f305b1156c7c881cd11b"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
|
|
@ -1887,48 +1887,48 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "pydantic"
|
||||
version = "1.10.7"
|
||||
version = "1.10.8"
|
||||
description = "Data validation and settings management using python type hints"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "pydantic-1.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e79e999e539872e903767c417c897e729e015872040e56b96e67968c3b918b2d"},
|
||||
{file = "pydantic-1.10.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:01aea3a42c13f2602b7ecbbea484a98169fb568ebd9e247593ea05f01b884b2e"},
|
||||
{file = "pydantic-1.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:516f1ed9bc2406a0467dd777afc636c7091d71f214d5e413d64fef45174cfc7a"},
|
||||
{file = "pydantic-1.10.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae150a63564929c675d7f2303008d88426a0add46efd76c3fc797cd71cb1b46f"},
|
||||
{file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209"},
|
||||
{file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af"},
|
||||
{file = "pydantic-1.10.7-cp310-cp310-win_amd64.whl", hash = "sha256:a7cd2251439988b413cb0a985c4ed82b6c6aac382dbaff53ae03c4b23a70e80a"},
|
||||
{file = "pydantic-1.10.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:68792151e174a4aa9e9fc1b4e653e65a354a2fa0fed169f7b3d09902ad2cb6f1"},
|
||||
{file = "pydantic-1.10.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe2507b8ef209da71b6fb5f4e597b50c5a34b78d7e857c4f8f3115effaef5fe"},
|
||||
{file = "pydantic-1.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10a86d8c8db68086f1e30a530f7d5f83eb0685e632e411dbbcf2d5c0150e8dcd"},
|
||||
{file = "pydantic-1.10.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75ae19d2a3dbb146b6f324031c24f8a3f52ff5d6a9f22f0683694b3afcb16fb"},
|
||||
{file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:464855a7ff7f2cc2cf537ecc421291b9132aa9c79aef44e917ad711b4a93163b"},
|
||||
{file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:193924c563fae6ddcb71d3f06fa153866423ac1b793a47936656e806b64e24ca"},
|
||||
{file = "pydantic-1.10.7-cp311-cp311-win_amd64.whl", hash = "sha256:b4a849d10f211389502059c33332e91327bc154acc1845f375a99eca3afa802d"},
|
||||
{file = "pydantic-1.10.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cc1dde4e50a5fc1336ee0581c1612215bc64ed6d28d2c7c6f25d2fe3e7c3e918"},
|
||||
{file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0cfe895a504c060e5d36b287ee696e2fdad02d89e0d895f83037245218a87fe"},
|
||||
{file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:670bb4683ad1e48b0ecb06f0cfe2178dcf74ff27921cdf1606e527d2617a81ee"},
|
||||
{file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:950ce33857841f9a337ce07ddf46bc84e1c4946d2a3bba18f8280297157a3fd1"},
|
||||
{file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c15582f9055fbc1bfe50266a19771bbbef33dd28c45e78afbe1996fd70966c2a"},
|
||||
{file = "pydantic-1.10.7-cp37-cp37m-win_amd64.whl", hash = "sha256:82dffb306dd20bd5268fd6379bc4bfe75242a9c2b79fec58e1041fbbdb1f7914"},
|
||||
{file = "pydantic-1.10.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c7f51861d73e8b9ddcb9916ae7ac39fb52761d9ea0df41128e81e2ba42886cd"},
|
||||
{file = "pydantic-1.10.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6434b49c0b03a51021ade5c4daa7d70c98f7a79e95b551201fff682fc1661245"},
|
||||
{file = "pydantic-1.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d34ab766fa056df49013bb6e79921a0265204c071984e75a09cbceacbbdd5d"},
|
||||
{file = "pydantic-1.10.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:701daea9ffe9d26f97b52f1d157e0d4121644f0fcf80b443248434958fd03dc3"},
|
||||
{file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf135c46099ff3f919d2150a948ce94b9ce545598ef2c6c7bf55dca98a304b52"},
|
||||
{file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0f85904f73161817b80781cc150f8b906d521fa11e3cdabae19a581c3606209"},
|
||||
{file = "pydantic-1.10.7-cp38-cp38-win_amd64.whl", hash = "sha256:9f6f0fd68d73257ad6685419478c5aece46432f4bdd8d32c7345f1986496171e"},
|
||||
{file = "pydantic-1.10.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c230c0d8a322276d6e7b88c3f7ce885f9ed16e0910354510e0bae84d54991143"},
|
||||
{file = "pydantic-1.10.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:976cae77ba6a49d80f461fd8bba183ff7ba79f44aa5cfa82f1346b5626542f8e"},
|
||||
{file = "pydantic-1.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d45fc99d64af9aaf7e308054a0067fdcd87ffe974f2442312372dfa66e1001d"},
|
||||
{file = "pydantic-1.10.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d2a5ebb48958754d386195fe9e9c5106f11275867051bf017a8059410e9abf1f"},
|
||||
{file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:abfb7d4a7cd5cc4e1d1887c43503a7c5dd608eadf8bc615413fc498d3e4645cd"},
|
||||
{file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:80b1fab4deb08a8292d15e43a6edccdffa5377a36a4597bb545b93e79c5ff0a5"},
|
||||
{file = "pydantic-1.10.7-cp39-cp39-win_amd64.whl", hash = "sha256:d71e69699498b020ea198468e2480a2f1e7433e32a3a99760058c6520e2bea7e"},
|
||||
{file = "pydantic-1.10.7-py3-none-any.whl", hash = "sha256:0cd181f1d0b1d00e2b705f1bf1ac7799a2d938cce3376b8007df62b29be3c2c6"},
|
||||
{file = "pydantic-1.10.7.tar.gz", hash = "sha256:cfc83c0678b6ba51b0532bea66860617c4cd4251ecf76e9846fa5a9f3454e97e"},
|
||||
{file = "pydantic-1.10.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d"},
|
||||
{file = "pydantic-1.10.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f"},
|
||||
{file = "pydantic-1.10.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f"},
|
||||
{file = "pydantic-1.10.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319"},
|
||||
{file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277"},
|
||||
{file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab"},
|
||||
{file = "pydantic-1.10.8-cp310-cp310-win_amd64.whl", hash = "sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800"},
|
||||
{file = "pydantic-1.10.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33"},
|
||||
{file = "pydantic-1.10.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5"},
|
||||
{file = "pydantic-1.10.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85"},
|
||||
{file = "pydantic-1.10.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f"},
|
||||
{file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e"},
|
||||
{file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4"},
|
||||
{file = "pydantic-1.10.8-cp311-cp311-win_amd64.whl", hash = "sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd"},
|
||||
{file = "pydantic-1.10.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878"},
|
||||
{file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4"},
|
||||
{file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b"},
|
||||
{file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68"},
|
||||
{file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea"},
|
||||
{file = "pydantic-1.10.8-cp37-cp37m-win_amd64.whl", hash = "sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c"},
|
||||
{file = "pydantic-1.10.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887"},
|
||||
{file = "pydantic-1.10.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6"},
|
||||
{file = "pydantic-1.10.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18"},
|
||||
{file = "pydantic-1.10.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375"},
|
||||
{file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1"},
|
||||
{file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108"},
|
||||
{file = "pydantic-1.10.8-cp38-cp38-win_amd64.whl", hash = "sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56"},
|
||||
{file = "pydantic-1.10.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e"},
|
||||
{file = "pydantic-1.10.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0"},
|
||||
{file = "pydantic-1.10.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459"},
|
||||
{file = "pydantic-1.10.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4"},
|
||||
{file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1"},
|
||||
{file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01"},
|
||||
{file = "pydantic-1.10.8-cp39-cp39-win_amd64.whl", hash = "sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a"},
|
||||
{file = "pydantic-1.10.8-py3-none-any.whl", hash = "sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2"},
|
||||
{file = "pydantic-1.10.8.tar.gz", hash = "sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -1940,14 +1940,14 @@ email = ["email-validator (>=1.0.3)"]
|
|||
|
||||
[[package]]
|
||||
name = "pygithub"
|
||||
version = "1.58.1"
|
||||
version = "1.58.2"
|
||||
description = "Use the full Github API v3"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "PyGithub-1.58.1-py3-none-any.whl", hash = "sha256:4e7fe9c3ec30d5fde5b4fbb97f18821c9dbf372bf6df337fe66f6689a65e0a83"},
|
||||
{file = "PyGithub-1.58.1.tar.gz", hash = "sha256:7d528b4ad92bc13122129fafd444ce3d04c47d2d801f6446b6e6ee2d410235b3"},
|
||||
{file = "PyGithub-1.58.2-py3-none-any.whl", hash = "sha256:f435884af617c6debaa76cbc355372d1027445a56fbc39972a3b9ed4968badc8"},
|
||||
{file = "PyGithub-1.58.2.tar.gz", hash = "sha256:1e6b1b7afe31f75151fb81f7ab6b984a7188a852bdb123dbb9ae90023c3ce60f"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -2251,21 +2251,21 @@ md = ["cmarkgfm (>=0.8.0)"]
|
|||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
version = "2.28.2"
|
||||
version = "2.31.0"
|
||||
description = "Python HTTP for Humans."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7, <4"
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"},
|
||||
{file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"},
|
||||
{file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
|
||||
{file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
certifi = ">=2017.4.17"
|
||||
charset-normalizer = ">=2,<4"
|
||||
idna = ">=2.5,<4"
|
||||
urllib3 = ">=1.21.1,<1.27"
|
||||
urllib3 = ">=1.21.1,<3"
|
||||
|
||||
[package.extras]
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
|
||||
|
|
@ -2565,21 +2565,21 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "sphinx"
|
||||
version = "6.1.3"
|
||||
version = "6.2.1"
|
||||
description = "Python documentation generator"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "Sphinx-6.1.3.tar.gz", hash = "sha256:0dac3b698538ffef41716cf97ba26c1c7788dba73ce6f150c1ff5b4720786dd2"},
|
||||
{file = "sphinx-6.1.3-py3-none-any.whl", hash = "sha256:807d1cb3d6be87eb78a381c3e70ebd8d346b9a25f3753e9947e866b2786865fc"},
|
||||
{file = "Sphinx-6.2.1.tar.gz", hash = "sha256:6d56a34697bb749ffa0152feafc4b19836c755d90a7c59b72bc7dfd371b9cc6b"},
|
||||
{file = "sphinx-6.2.1-py3-none-any.whl", hash = "sha256:97787ff1fa3256a3eef9eda523a63dbf299f7b47e053cfcf684a1c2a8380c912"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
alabaster = ">=0.7,<0.8"
|
||||
babel = ">=2.9"
|
||||
colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""}
|
||||
docutils = ">=0.18,<0.20"
|
||||
docutils = ">=0.18.1,<0.20"
|
||||
imagesize = ">=1.3"
|
||||
importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""}
|
||||
Jinja2 = ">=3.0"
|
||||
|
|
@ -2597,7 +2597,7 @@ sphinxcontrib-serializinghtml = ">=1.1.5"
|
|||
[package.extras]
|
||||
docs = ["sphinxcontrib-websupport"]
|
||||
lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-simplify", "isort", "mypy (>=0.990)", "ruff", "sphinx-lint", "types-requests"]
|
||||
test = ["cython", "html5lib", "pytest (>=4.6)"]
|
||||
test = ["cython", "filelock", "html5lib", "pytest (>=4.6)"]
|
||||
|
||||
[[package]]
|
||||
name = "sphinx-autodoc2"
|
||||
|
|
@ -2998,14 +2998,14 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "types-bleach"
|
||||
version = "6.0.0.1"
|
||||
version = "6.0.0.3"
|
||||
description = "Typing stubs for bleach"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "types-bleach-6.0.0.1.tar.gz", hash = "sha256:43d9129deb9e82918747437edf78f09ff440f2973f4702625b61994f3e698518"},
|
||||
{file = "types_bleach-6.0.0.1-py3-none-any.whl", hash = "sha256:440df967254007be80bb0f4d851f026c29c709cc48359bf4935d2b2f3a6f9f90"},
|
||||
{file = "types-bleach-6.0.0.3.tar.gz", hash = "sha256:8ce7896d4f658c562768674ffcf07492c7730e128018f03edd163ff912bfadee"},
|
||||
{file = "types_bleach-6.0.0.3-py3-none-any.whl", hash = "sha256:d43eaf30a643ca824e16e2dcdb0c87ef9226237e2fa3ac4732a50cb3f32e145f"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3058,14 +3058,14 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "types-pillow"
|
||||
version = "9.5.0.2"
|
||||
version = "9.5.0.4"
|
||||
description = "Typing stubs for Pillow"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "types-Pillow-9.5.0.2.tar.gz", hash = "sha256:b3f9f621f259566c19c1deca21901017c8b1e3e200ed2e49e0a2d83c0a5175db"},
|
||||
{file = "types_Pillow-9.5.0.2-py3-none-any.whl", hash = "sha256:58fdebd0ffa2353ecccdd622adde23bce89da5c0c8b96c34f2d1eca7b7e42d0e"},
|
||||
{file = "types-Pillow-9.5.0.4.tar.gz", hash = "sha256:f1b6af47abd151847ee25911ffeba784899bc7dc7f9eba8ca6a5aac522b012ef"},
|
||||
{file = "types_Pillow-9.5.0.4-py3-none-any.whl", hash = "sha256:69427d9fa4320ff6e30f00fb9c0dd71185dc0a16de4757774220104759483466"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3097,26 +3097,26 @@ cryptography = ">=35.0.0"
|
|||
|
||||
[[package]]
|
||||
name = "types-pyyaml"
|
||||
version = "6.0.12.9"
|
||||
version = "6.0.12.10"
|
||||
description = "Typing stubs for PyYAML"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "types-PyYAML-6.0.12.9.tar.gz", hash = "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"},
|
||||
{file = "types_PyYAML-6.0.12.9-py3-none-any.whl", hash = "sha256:5aed5aa66bd2d2e158f75dda22b059570ede988559f030cf294871d3b647e3e8"},
|
||||
{file = "types-PyYAML-6.0.12.10.tar.gz", hash = "sha256:ebab3d0700b946553724ae6ca636ea932c1b0868701d4af121630e78d695fc97"},
|
||||
{file = "types_PyYAML-6.0.12.10-py3-none-any.whl", hash = "sha256:662fa444963eff9b68120d70cda1af5a5f2aa57900003c2006d7626450eaae5f"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "types-requests"
|
||||
version = "2.30.0.0"
|
||||
version = "2.31.0.0"
|
||||
description = "Typing stubs for requests"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "types-requests-2.30.0.0.tar.gz", hash = "sha256:dec781054324a70ba64430ae9e62e7e9c8e4618c185a5cb3f87a6738251b5a31"},
|
||||
{file = "types_requests-2.30.0.0-py3-none-any.whl", hash = "sha256:c6cf08e120ca9f0dc4fa4e32c3f953c3fba222bcc1db6b97695bce8da1ba9864"},
|
||||
{file = "types-requests-2.31.0.0.tar.gz", hash = "sha256:c1c29d20ab8d84dff468d7febfe8e0cb0b4664543221b386605e14672b44ea25"},
|
||||
{file = "types_requests-2.31.0.0-py3-none-any.whl", hash = "sha256:7c5cea7940f8e92ec560bbc468f65bf684aa3dcf0554a6f8c4710f5f708dc598"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -3124,14 +3124,14 @@ types-urllib3 = "*"
|
|||
|
||||
[[package]]
|
||||
name = "types-setuptools"
|
||||
version = "67.7.0.2"
|
||||
version = "67.8.0.0"
|
||||
description = "Typing stubs for setuptools"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "types-setuptools-67.7.0.2.tar.gz", hash = "sha256:155789e85e79d5682b0d341919d4beb6140408ae52bac922af25b54e36ab25c0"},
|
||||
{file = "types_setuptools-67.7.0.2-py3-none-any.whl", hash = "sha256:bd30f6dbe9b83f0a7e6e3eab6d2df748aa4f55700d54e9f077d3aa30cc019445"},
|
||||
{file = "types-setuptools-67.8.0.0.tar.gz", hash = "sha256:95c9ed61871d6c0e258433373a4e1753c0a7c3627a46f4d4058c7b5a08ab844f"},
|
||||
{file = "types_setuptools-67.8.0.0-py3-none-any.whl", hash = "sha256:6df73340d96b238a4188b7b7668814b37e8018168aef1eef94a3b1872e3f60ff"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"
|
|||
|
||||
[tool.poetry]
|
||||
name = "matrix-synapse"
|
||||
version = "1.84.0"
|
||||
version = "1.85.0rc1"
|
||||
description = "Homeserver for the Matrix decentralised comms protocol"
|
||||
authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
|
||||
license = "Apache-2.0"
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ bleach==6.0.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0"
|
|||
canonicaljson==2.0.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:c38a315de3b5a0532f1ec1f9153cd3d716abfc565a558d00a4835428a34fca5b \
|
||||
--hash=sha256:e2fdaef1d7fadc5d9cb59bd3d0d41b064ddda697809ac4325dced721d12f113f
|
||||
certifi==2022.12.7 ; python_full_version >= "3.7.1" and python_version < "4" \
|
||||
certifi==2022.12.7 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \
|
||||
--hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18
|
||||
cffi==1.15.1 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
|
|
@ -103,7 +103,7 @@ cffi==1.15.1 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0"
|
|||
--hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \
|
||||
--hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \
|
||||
--hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0
|
||||
charset-normalizer==3.1.0 ; python_full_version >= "3.7.1" and python_version < "4" \
|
||||
charset-normalizer==3.1.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \
|
||||
--hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \
|
||||
--hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \
|
||||
|
|
@ -295,7 +295,7 @@ hiredis==2.2.3 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0
|
|||
hyperlink==21.0.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b \
|
||||
--hash=sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4
|
||||
idna==3.4 ; python_full_version >= "3.7.1" and python_version < "4" \
|
||||
idna==3.4 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \
|
||||
--hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2
|
||||
ijson==3.2.0.post0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
|
|
@ -684,9 +684,9 @@ pillow==9.4.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0"
|
|||
pkgutil-resolve-name==1.3.10 ; python_full_version >= "3.7.1" and python_version < "3.9" \
|
||||
--hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \
|
||||
--hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e
|
||||
prometheus-client==0.16.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:0836af6eb2c8f4fed712b2f279f6c0a8bbab29f9f4aa15276b91c7cb0d1616ab \
|
||||
--hash=sha256:a03e35b359f14dd1630898543e2120addfdeacd1a6069c1367ae90fd93ad3f48
|
||||
prometheus-client==0.17.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:9c3b26f1535945e85b8934fb374678d263137b78ef85f305b1156c7c881cd11b \
|
||||
--hash=sha256:a77b708cf083f4d1a3fb3ce5c95b4afa32b9c521ae363354a4a910204ea095ce
|
||||
psycopg2==2.9.6 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:11aca705ec888e4f4cea97289a0bf0f22a067a32614f6ef64fcf7b8bfbc53744 \
|
||||
--hash=sha256:1861a53a6a0fd248e42ea37c957d36950da00266378746588eab4f4b5649e95f \
|
||||
|
|
@ -714,43 +714,43 @@ pyasn1==0.4.8 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0"
|
|||
pycparser==2.21 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
|
||||
--hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
|
||||
pydantic==1.10.7 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:01aea3a42c13f2602b7ecbbea484a98169fb568ebd9e247593ea05f01b884b2e \
|
||||
--hash=sha256:0cd181f1d0b1d00e2b705f1bf1ac7799a2d938cce3376b8007df62b29be3c2c6 \
|
||||
--hash=sha256:10a86d8c8db68086f1e30a530f7d5f83eb0685e632e411dbbcf2d5c0150e8dcd \
|
||||
--hash=sha256:193924c563fae6ddcb71d3f06fa153866423ac1b793a47936656e806b64e24ca \
|
||||
--hash=sha256:464855a7ff7f2cc2cf537ecc421291b9132aa9c79aef44e917ad711b4a93163b \
|
||||
--hash=sha256:516f1ed9bc2406a0467dd777afc636c7091d71f214d5e413d64fef45174cfc7a \
|
||||
--hash=sha256:6434b49c0b03a51021ade5c4daa7d70c98f7a79e95b551201fff682fc1661245 \
|
||||
--hash=sha256:64d34ab766fa056df49013bb6e79921a0265204c071984e75a09cbceacbbdd5d \
|
||||
--hash=sha256:670bb4683ad1e48b0ecb06f0cfe2178dcf74ff27921cdf1606e527d2617a81ee \
|
||||
--hash=sha256:68792151e174a4aa9e9fc1b4e653e65a354a2fa0fed169f7b3d09902ad2cb6f1 \
|
||||
--hash=sha256:701daea9ffe9d26f97b52f1d157e0d4121644f0fcf80b443248434958fd03dc3 \
|
||||
--hash=sha256:7d45fc99d64af9aaf7e308054a0067fdcd87ffe974f2442312372dfa66e1001d \
|
||||
--hash=sha256:80b1fab4deb08a8292d15e43a6edccdffa5377a36a4597bb545b93e79c5ff0a5 \
|
||||
--hash=sha256:82dffb306dd20bd5268fd6379bc4bfe75242a9c2b79fec58e1041fbbdb1f7914 \
|
||||
--hash=sha256:8c7f51861d73e8b9ddcb9916ae7ac39fb52761d9ea0df41128e81e2ba42886cd \
|
||||
--hash=sha256:950ce33857841f9a337ce07ddf46bc84e1c4946d2a3bba18f8280297157a3fd1 \
|
||||
--hash=sha256:976cae77ba6a49d80f461fd8bba183ff7ba79f44aa5cfa82f1346b5626542f8e \
|
||||
--hash=sha256:9f6f0fd68d73257ad6685419478c5aece46432f4bdd8d32c7345f1986496171e \
|
||||
--hash=sha256:a7cd2251439988b413cb0a985c4ed82b6c6aac382dbaff53ae03c4b23a70e80a \
|
||||
--hash=sha256:abfb7d4a7cd5cc4e1d1887c43503a7c5dd608eadf8bc615413fc498d3e4645cd \
|
||||
--hash=sha256:ae150a63564929c675d7f2303008d88426a0add46efd76c3fc797cd71cb1b46f \
|
||||
--hash=sha256:b0f85904f73161817b80781cc150f8b906d521fa11e3cdabae19a581c3606209 \
|
||||
--hash=sha256:b4a849d10f211389502059c33332e91327bc154acc1845f375a99eca3afa802d \
|
||||
--hash=sha256:c15582f9055fbc1bfe50266a19771bbbef33dd28c45e78afbe1996fd70966c2a \
|
||||
--hash=sha256:c230c0d8a322276d6e7b88c3f7ce885f9ed16e0910354510e0bae84d54991143 \
|
||||
--hash=sha256:cc1dde4e50a5fc1336ee0581c1612215bc64ed6d28d2c7c6f25d2fe3e7c3e918 \
|
||||
--hash=sha256:cf135c46099ff3f919d2150a948ce94b9ce545598ef2c6c7bf55dca98a304b52 \
|
||||
--hash=sha256:cfc83c0678b6ba51b0532bea66860617c4cd4251ecf76e9846fa5a9f3454e97e \
|
||||
--hash=sha256:d2a5ebb48958754d386195fe9e9c5106f11275867051bf017a8059410e9abf1f \
|
||||
--hash=sha256:d71e69699498b020ea198468e2480a2f1e7433e32a3a99760058c6520e2bea7e \
|
||||
--hash=sha256:d75ae19d2a3dbb146b6f324031c24f8a3f52ff5d6a9f22f0683694b3afcb16fb \
|
||||
--hash=sha256:dfe2507b8ef209da71b6fb5f4e597b50c5a34b78d7e857c4f8f3115effaef5fe \
|
||||
--hash=sha256:e0cfe895a504c060e5d36b287ee696e2fdad02d89e0d895f83037245218a87fe \
|
||||
--hash=sha256:e79e999e539872e903767c417c897e729e015872040e56b96e67968c3b918b2d \
|
||||
--hash=sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209 \
|
||||
--hash=sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af
|
||||
pydantic==1.10.8 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375 \
|
||||
--hash=sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277 \
|
||||
--hash=sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d \
|
||||
--hash=sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4 \
|
||||
--hash=sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca \
|
||||
--hash=sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c \
|
||||
--hash=sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01 \
|
||||
--hash=sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18 \
|
||||
--hash=sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68 \
|
||||
--hash=sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887 \
|
||||
--hash=sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459 \
|
||||
--hash=sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4 \
|
||||
--hash=sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5 \
|
||||
--hash=sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e \
|
||||
--hash=sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1 \
|
||||
--hash=sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33 \
|
||||
--hash=sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a \
|
||||
--hash=sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56 \
|
||||
--hash=sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108 \
|
||||
--hash=sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2 \
|
||||
--hash=sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4 \
|
||||
--hash=sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878 \
|
||||
--hash=sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0 \
|
||||
--hash=sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e \
|
||||
--hash=sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6 \
|
||||
--hash=sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f \
|
||||
--hash=sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800 \
|
||||
--hash=sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea \
|
||||
--hash=sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f \
|
||||
--hash=sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b \
|
||||
--hash=sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1 \
|
||||
--hash=sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd \
|
||||
--hash=sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319 \
|
||||
--hash=sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab \
|
||||
--hash=sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85 \
|
||||
--hash=sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f
|
||||
pymacaroons==0.13.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:1e6bba42a5f66c245adf38a5a4006a99dcc06a0703786ea636098667d42903b8 \
|
||||
--hash=sha256:3e14dff6a262fdbf1a15e769ce635a8aea72e6f8f91e408f9a97166c53b91907
|
||||
|
|
@ -837,9 +837,9 @@ pyyaml==6.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
|||
--hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \
|
||||
--hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \
|
||||
--hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5
|
||||
requests==2.28.2 ; python_full_version >= "3.7.1" and python_version < "4" \
|
||||
--hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \
|
||||
--hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf
|
||||
requests==2.31.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
|
||||
--hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
|
||||
semantic-version==2.10.0 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c \
|
||||
--hash=sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177
|
||||
|
|
@ -892,7 +892,7 @@ typing-extensions==4.5.0 ; python_full_version >= "3.7.1" and python_full_versio
|
|||
unpaddedbase64==2.1.0 ; python_full_version >= "3.7.1" and python_version < "4.0" \
|
||||
--hash=sha256:485eff129c30175d2cd6f0cd8d2310dff51e666f7f36175f738d75dfdbd0b1c6 \
|
||||
--hash=sha256:7273c60c089de39d90f5d6d4a7883a79e319dc9d9b1c8924a7fab96178a5f005
|
||||
urllib3==1.26.15 ; python_full_version >= "3.7.1" and python_version < "4" \
|
||||
urllib3==1.26.15 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
--hash=sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 \
|
||||
--hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42
|
||||
webencodings==0.5.1 ; python_full_version >= "3.7.1" and python_full_version < "4.0.0" \
|
||||
|
|
|
|||
|
|
@ -18,10 +18,11 @@ can crop up, e.g the cache descriptors.
|
|||
|
||||
from typing import Callable, Optional, Type
|
||||
|
||||
from mypy.erasetype import remove_instance_last_known_values
|
||||
from mypy.nodes import ARG_NAMED_OPT
|
||||
from mypy.plugin import MethodSigContext, Plugin
|
||||
from mypy.typeops import bind_self
|
||||
from mypy.types import CallableType, NoneType, UnionType
|
||||
from mypy.types import CallableType, Instance, NoneType, UnionType
|
||||
|
||||
|
||||
class SynapsePlugin(Plugin):
|
||||
|
|
@ -92,10 +93,41 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
|
|||
arg_names.append("on_invalidate")
|
||||
arg_kinds.append(ARG_NAMED_OPT) # Arg is an optional kwarg.
|
||||
|
||||
# Finally we ensure the return type is a Deferred.
|
||||
if (
|
||||
isinstance(signature.ret_type, Instance)
|
||||
and signature.ret_type.type.fullname == "twisted.internet.defer.Deferred"
|
||||
):
|
||||
# If it is already a Deferred, nothing to do.
|
||||
ret_type = signature.ret_type
|
||||
else:
|
||||
ret_arg = None
|
||||
if isinstance(signature.ret_type, Instance):
|
||||
# If a coroutine, wrap the coroutine's return type in a Deferred.
|
||||
if signature.ret_type.type.fullname == "typing.Coroutine":
|
||||
ret_arg = signature.ret_type.args[2]
|
||||
|
||||
# If an awaitable, wrap the awaitable's final value in a Deferred.
|
||||
elif signature.ret_type.type.fullname == "typing.Awaitable":
|
||||
ret_arg = signature.ret_type.args[0]
|
||||
|
||||
# Otherwise, wrap the return value in a Deferred.
|
||||
if ret_arg is None:
|
||||
ret_arg = signature.ret_type
|
||||
|
||||
# This should be able to use ctx.api.named_generic_type, but that doesn't seem
|
||||
# to find the correct symbol for anything more than 1 module deep.
|
||||
#
|
||||
# modules is not part of CheckerPluginInterface. The following is a combination
|
||||
# of TypeChecker.named_generic_type and TypeChecker.lookup_typeinfo.
|
||||
sym = ctx.api.modules["twisted.internet.defer"].names.get("Deferred") # type: ignore[attr-defined]
|
||||
ret_type = Instance(sym.node, [remove_instance_last_known_values(ret_arg)])
|
||||
|
||||
signature = signature.copy_modified(
|
||||
arg_types=arg_types,
|
||||
arg_names=arg_names,
|
||||
arg_kinds=arg_kinds,
|
||||
ret_type=ret_type,
|
||||
)
|
||||
|
||||
return signature
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import time
|
|||
import urllib.request
|
||||
from os import path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any, List, Optional
|
||||
from typing import Any, List, Match, Optional, Union
|
||||
|
||||
import attr
|
||||
import click
|
||||
|
|
@ -233,7 +233,7 @@ def _prepare() -> None:
|
|||
subprocess.check_output(["poetry", "version", new_version])
|
||||
|
||||
# Generate changelogs.
|
||||
generate_and_write_changelog(current_version, new_version)
|
||||
generate_and_write_changelog(synapse_repo, current_version, new_version)
|
||||
|
||||
# Generate debian changelogs
|
||||
if parsed_new_version.pre is not None:
|
||||
|
|
@ -814,7 +814,7 @@ def get_changes_for_version(wanted_version: version.Version) -> str:
|
|||
|
||||
|
||||
def generate_and_write_changelog(
|
||||
current_version: version.Version, new_version: str
|
||||
repo: Repo, current_version: version.Version, new_version: str
|
||||
) -> None:
|
||||
# We do this by getting a draft so that we can edit it before writing to the
|
||||
# changelog.
|
||||
|
|
@ -827,6 +827,10 @@ def generate_and_write_changelog(
|
|||
new_changes = new_changes.replace(
|
||||
"No significant changes.", f"No significant changes since {current_version}."
|
||||
)
|
||||
new_changes += build_dependabot_changelog(
|
||||
repo,
|
||||
current_version,
|
||||
)
|
||||
|
||||
# Prepend changes to changelog
|
||||
with open("CHANGES.md", "r+") as f:
|
||||
|
|
@ -841,5 +845,49 @@ def generate_and_write_changelog(
|
|||
os.remove(filename)
|
||||
|
||||
|
||||
def build_dependabot_changelog(repo: Repo, current_version: version.Version) -> str:
|
||||
"""Summarise dependabot commits between `current_version` and `release_branch`.
|
||||
|
||||
Returns an empty string if there have been no such commits; otherwise outputs a
|
||||
third-level markdown header followed by an unordered list."""
|
||||
last_release_commit = repo.tag("v" + str(current_version)).commit
|
||||
rev_spec = f"{last_release_commit.hexsha}.."
|
||||
commits = list(git.objects.Commit.iter_items(repo, rev_spec))
|
||||
messages = []
|
||||
for commit in reversed(commits):
|
||||
if commit.author.name == "dependabot[bot]":
|
||||
message: Union[str, bytes] = commit.message
|
||||
if isinstance(message, bytes):
|
||||
message = message.decode("utf-8")
|
||||
messages.append(message.split("\n", maxsplit=1)[0])
|
||||
|
||||
if not messages:
|
||||
print(f"No dependabot commits in range {rev_spec}", file=sys.stderr)
|
||||
return ""
|
||||
|
||||
messages.sort()
|
||||
|
||||
def replacer(match: Match[str]) -> str:
|
||||
desc = match.group(1)
|
||||
number = match.group(2)
|
||||
return f"* {desc}. ([\\#{number}](https://github.com/matrix-org/synapse/issues/{number}))"
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
messages[i] = re.sub(r"(.*) \(#(\d+)\)$", replacer, message)
|
||||
messages.insert(0, "### Updates to locked dependencies\n")
|
||||
# Add an extra blank line to the bottom of the section
|
||||
messages.append("")
|
||||
return "\n".join(messages)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument("since")
|
||||
def test_dependabot_changelog(since: str) -> None:
|
||||
"""Test building the dependabot changelog.
|
||||
|
||||
Summarises all dependabot commits between the SINCE tag and the current git HEAD."""
|
||||
print(build_dependabot_changelog(git.Repo("."), version.Version(since)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
|
|
|||
|
|
@ -61,6 +61,9 @@ def lazyConnection(
|
|||
# most methods to it via ConnectionHandler.__getattr__.
|
||||
class ConnectionHandler(RedisProtocol):
|
||||
def disconnect(self) -> "Deferred[None]": ...
|
||||
def __repr__(self) -> str: ...
|
||||
|
||||
class UnixConnectionHandler(ConnectionHandler): ...
|
||||
|
||||
class RedisFactory(protocol.ReconnectingClientFactory):
|
||||
continueTrying: bool
|
||||
|
|
|
|||
|
|
@ -128,20 +128,7 @@ USER_FILTER_SCHEMA = {
|
|||
"account_data": {"$ref": "#/definitions/filter"},
|
||||
"room": {"$ref": "#/definitions/room_filter"},
|
||||
"event_format": {"type": "string", "enum": ["client", "federation"]},
|
||||
"event_fields": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
# Don't allow '\\' in event field filters. This makes matching
|
||||
# events a lot easier as we can then use a negative lookbehind
|
||||
# assertion to split '\.' If we allowed \\ then it would
|
||||
# incorrectly split '\\.' See synapse.events.utils.serialize_event
|
||||
#
|
||||
# Note that because this is a regular expression, we have to escape
|
||||
# each backslash in the pattern.
|
||||
"pattern": r"^((?!\\\\).)*$",
|
||||
},
|
||||
},
|
||||
"event_fields": {"type": "array", "items": {"type": "string"}},
|
||||
},
|
||||
"additionalProperties": True, # Allow new fields for forward compatibility
|
||||
}
|
||||
|
|
|
|||
|
|
@ -485,6 +485,30 @@ class RoomVersions:
|
|||
msc3931_push_features=(),
|
||||
msc3989_redaction_rules=True,
|
||||
)
|
||||
MSC3820opt2 = RoomVersion(
|
||||
# Based upon v10
|
||||
"org.matrix.msc3820.opt2",
|
||||
RoomDisposition.UNSTABLE,
|
||||
EventFormatVersions.ROOM_V4_PLUS,
|
||||
StateResolutionVersions.V2,
|
||||
enforce_key_validity=True,
|
||||
special_case_aliases_auth=False,
|
||||
strict_canonicaljson=True,
|
||||
limit_notifications_power_levels=True,
|
||||
msc2175_implicit_room_creator=True, # Used by MSC3820
|
||||
msc2176_redaction_rules=True, # Used by MSC3820
|
||||
msc3083_join_rules=True,
|
||||
msc3375_redaction_rules=True,
|
||||
msc2403_knocking=True,
|
||||
msc2716_historical=False,
|
||||
msc2716_redactions=False,
|
||||
msc3389_relation_redactions=False,
|
||||
msc3787_knock_restricted_join_rule=True,
|
||||
msc3667_int_only_power_levels=True,
|
||||
msc3821_redaction_rules=True, # Used by MSC3820
|
||||
msc3931_push_features=(),
|
||||
msc3989_redaction_rules=True, # Used by MSC3820
|
||||
)
|
||||
|
||||
|
||||
KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
|
||||
|
|
@ -504,6 +528,7 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
|
|||
RoomVersions.V10,
|
||||
RoomVersions.MSC2716v4,
|
||||
RoomVersions.MSC3989,
|
||||
RoomVersions.MSC3820opt2,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ from synapse.util.logcontext import LoggingContext
|
|||
logger = logging.getLogger("synapse.app.admin_cmd")
|
||||
|
||||
|
||||
class AdminCmdSlavedStore(
|
||||
class AdminCmdStore(
|
||||
FilteringWorkerStore,
|
||||
ClientIpWorkerStore,
|
||||
DeviceWorkerStore,
|
||||
|
|
@ -103,7 +103,7 @@ class AdminCmdSlavedStore(
|
|||
|
||||
|
||||
class AdminCmdServer(HomeServer):
|
||||
DATASTORE_CLASS = AdminCmdSlavedStore # type: ignore
|
||||
DATASTORE_CLASS = AdminCmdStore # type: ignore
|
||||
|
||||
|
||||
async def export_data_command(hs: HomeServer, args: argparse.Namespace) -> None:
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ from synapse.util.httpresourcetree import create_resource_tree
|
|||
logger = logging.getLogger("synapse.app.generic_worker")
|
||||
|
||||
|
||||
class GenericWorkerSlavedStore(
|
||||
class GenericWorkerStore(
|
||||
# FIXME(#3714): We need to add UserDirectoryStore as we write directly
|
||||
# rather than going via the correct worker.
|
||||
UserDirectoryStore,
|
||||
|
|
@ -154,7 +154,7 @@ class GenericWorkerSlavedStore(
|
|||
|
||||
|
||||
class GenericWorkerServer(HomeServer):
|
||||
DATASTORE_CLASS = GenericWorkerSlavedStore # type: ignore
|
||||
DATASTORE_CLASS = GenericWorkerStore # type: ignore
|
||||
|
||||
def _listen_http(self, listener_config: ListenerConfig) -> None:
|
||||
assert listener_config.http_options is not None
|
||||
|
|
|
|||
|
|
@ -127,10 +127,6 @@ async def phone_stats_home(
|
|||
daily_sent_messages = await store.count_daily_sent_messages()
|
||||
stats["daily_sent_messages"] = daily_sent_messages
|
||||
|
||||
r30_results = await store.count_r30_users()
|
||||
for name, count in r30_results.items():
|
||||
stats["r30_users_" + name] = count
|
||||
|
||||
r30v2_results = await store.count_r30v2_users()
|
||||
for name, count in r30v2_results.items():
|
||||
stats["r30v2_users_" + name] = count
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ class ApplicationService:
|
|||
url.rstrip("/") if isinstance(url, str) else None
|
||||
) # url must not end with a slash
|
||||
self.hs_token = hs_token
|
||||
# The full Matrix ID for this application service's sender.
|
||||
self.sender = sender
|
||||
self.namespaces = self._check_namespaces(namespaces)
|
||||
self.id = id
|
||||
|
|
@ -212,7 +213,7 @@ class ApplicationService:
|
|||
True if the application service is interested in the user, False if not.
|
||||
"""
|
||||
return (
|
||||
# User is the appservice's sender_localpart user
|
||||
# User is the appservice's configured sender_localpart user
|
||||
user_id == self.sender
|
||||
# User is in the appservice's user namespace
|
||||
or self.is_user_in_namespace(user_id)
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ import jinja2
|
|||
import pkg_resources
|
||||
import yaml
|
||||
|
||||
from synapse.types import StrSequence
|
||||
from synapse.util.templates import _create_mxc_to_http_filter, _format_ts_filter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -58,7 +59,7 @@ class ConfigError(Exception):
|
|||
the problem lies.
|
||||
"""
|
||||
|
||||
def __init__(self, msg: str, path: Optional[Iterable[str]] = None):
|
||||
def __init__(self, msg: str, path: Optional[StrSequence] = None):
|
||||
self.msg = msg
|
||||
self.path = path
|
||||
|
||||
|
|
|
|||
|
|
@ -62,9 +62,10 @@ from synapse.config import ( # noqa: F401
|
|||
voip,
|
||||
workers,
|
||||
)
|
||||
from synapse.types import StrSequence
|
||||
|
||||
class ConfigError(Exception):
|
||||
def __init__(self, msg: str, path: Optional[Iterable[str]] = None):
|
||||
def __init__(self, msg: str, path: Optional[StrSequence] = None):
|
||||
self.msg = msg
|
||||
self.path = path
|
||||
|
||||
|
|
|
|||
|
|
@ -11,17 +11,17 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import Any, Dict, Iterable, Type, TypeVar
|
||||
from typing import Any, Dict, Type, TypeVar
|
||||
|
||||
import jsonschema
|
||||
from pydantic import BaseModel, ValidationError, parse_obj_as
|
||||
|
||||
from synapse.config._base import ConfigError
|
||||
from synapse.types import JsonDict
|
||||
from synapse.types import JsonDict, StrSequence
|
||||
|
||||
|
||||
def validate_config(
|
||||
json_schema: JsonDict, config: Any, config_path: Iterable[str]
|
||||
json_schema: JsonDict, config: Any, config_path: StrSequence
|
||||
) -> None:
|
||||
"""Validates a config setting against a JsonSchema definition
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ def validate_config(
|
|||
|
||||
|
||||
def json_error_to_config_error(
|
||||
e: jsonschema.ValidationError, config_path: Iterable[str]
|
||||
e: jsonschema.ValidationError, config_path: StrSequence
|
||||
) -> ConfigError:
|
||||
"""Converts a json validation error to a user-readable ConfigError
|
||||
|
||||
|
|
|
|||
|
|
@ -36,11 +36,10 @@ class AppServiceConfig(Config):
|
|||
if not isinstance(self.app_service_config_files, list) or not all(
|
||||
type(x) is str for x in self.app_service_config_files
|
||||
):
|
||||
# type-ignore: this function gets arbitrary json value; we do use this path.
|
||||
raise ConfigError(
|
||||
"Expected '%s' to be a list of AS config files:"
|
||||
% (self.app_service_config_files),
|
||||
"app_service_config_files",
|
||||
("app_service_config_files",),
|
||||
)
|
||||
|
||||
self.track_appservice_user_ips = config.get("track_appservice_user_ips", False)
|
||||
|
|
|
|||
|
|
@ -84,18 +84,6 @@ class ExperimentalConfig(Config):
|
|||
"msc3984_appservice_key_query", False
|
||||
)
|
||||
|
||||
# MSC3706 (server-side support for partial state in /send_join responses)
|
||||
# Synapse will always serve partial state responses to requests using the stable
|
||||
# query parameter `omit_members`. If this flag is set, Synapse will also serve
|
||||
# partial state responses to requests using the unstable query parameter
|
||||
# `org.matrix.msc3706.partial_state`.
|
||||
self.msc3706_enabled: bool = experimental.get("msc3706_enabled", False)
|
||||
|
||||
# experimental support for faster joins over federation
|
||||
# (MSC2775, MSC3706, MSC3895)
|
||||
# requires a target server that can provide a partial join response (MSC3706)
|
||||
self.faster_joins_enabled: bool = experimental.get("faster_joins", True)
|
||||
|
||||
# MSC3720 (Account status endpoint)
|
||||
self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False)
|
||||
|
||||
|
|
|
|||
|
|
@ -117,9 +117,7 @@ root:
|
|||
# Write logs to the `buffer` handler, which will buffer them together in memory,
|
||||
# then write them to a file.
|
||||
#
|
||||
# Replace "buffer" with "console" to log to stderr instead. (Note that you'll
|
||||
# also need to update the configuration for the `twisted` logger above, in
|
||||
# this case.)
|
||||
# Replace "buffer" with "console" to log to stderr instead.
|
||||
#
|
||||
handlers: [buffer]
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from urllib import parse as urlparse
|
|||
import attr
|
||||
import pkg_resources
|
||||
|
||||
from synapse.types import JsonDict
|
||||
from synapse.types import JsonDict, StrSequence
|
||||
|
||||
from ._base import Config, ConfigError
|
||||
from ._util import validate_config
|
||||
|
|
@ -80,7 +80,7 @@ class OembedConfig(Config):
|
|||
)
|
||||
|
||||
def _parse_and_validate_provider(
|
||||
self, providers: List[JsonDict], config_path: Iterable[str]
|
||||
self, providers: List[JsonDict], config_path: StrSequence
|
||||
) -> Iterable[OEmbedEndpointConfig]:
|
||||
# Ensure it is the proper form.
|
||||
validate_config(
|
||||
|
|
@ -112,7 +112,7 @@ class OembedConfig(Config):
|
|||
api_endpoint, patterns, endpoint.get("formats")
|
||||
)
|
||||
|
||||
def _glob_to_pattern(self, glob: str, config_path: Iterable[str]) -> Pattern:
|
||||
def _glob_to_pattern(self, glob: str, config_path: StrSequence) -> Pattern:
|
||||
"""
|
||||
Convert the glob into a sane regular expression to match against. The
|
||||
rules followed will be slightly different for the domain portion vs.
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ class RedisConfig(Config):
|
|||
|
||||
self.redis_host = redis_config.get("host", "localhost")
|
||||
self.redis_port = redis_config.get("port", 6379)
|
||||
self.redis_path = redis_config.get("path", None)
|
||||
self.redis_dbid = redis_config.get("dbid", None)
|
||||
self.redis_password = redis_config.get("password")
|
||||
|
||||
|
|
|
|||
|
|
@ -226,20 +226,20 @@ class ContentRepositoryConfig(Config):
|
|||
if "http" in proxy_env or "https" in proxy_env:
|
||||
logger.warning("".join(HTTP_PROXY_SET_WARNING))
|
||||
|
||||
# we always blacklist '0.0.0.0' and '::', which are supposed to be
|
||||
# we always block '0.0.0.0' and '::', which are supposed to be
|
||||
# unroutable addresses.
|
||||
self.url_preview_ip_range_blacklist = generate_ip_set(
|
||||
self.url_preview_ip_range_blocklist = generate_ip_set(
|
||||
config["url_preview_ip_range_blacklist"],
|
||||
["0.0.0.0", "::"],
|
||||
config_path=("url_preview_ip_range_blacklist",),
|
||||
)
|
||||
|
||||
self.url_preview_ip_range_whitelist = generate_ip_set(
|
||||
self.url_preview_ip_range_allowlist = generate_ip_set(
|
||||
config.get("url_preview_ip_range_whitelist", ()),
|
||||
config_path=("url_preview_ip_range_whitelist",),
|
||||
)
|
||||
|
||||
self.url_preview_url_blacklist = config.get("url_preview_url_blacklist", ())
|
||||
self.url_preview_url_blocklist = config.get("url_preview_url_blacklist", ())
|
||||
|
||||
self.url_preview_accept_language = config.get(
|
||||
"url_preview_accept_language"
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from netaddr import AddrFormatError, IPNetwork, IPSet
|
|||
from twisted.conch.ssh.keys import Key
|
||||
|
||||
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
|
||||
from synapse.types import JsonDict
|
||||
from synapse.types import JsonDict, StrSequence
|
||||
from synapse.util.module_loader import load_module
|
||||
from synapse.util.stringutils import parse_and_validate_server_name
|
||||
|
||||
|
|
@ -73,7 +73,7 @@ def _6to4(network: IPNetwork) -> IPNetwork:
|
|||
def generate_ip_set(
|
||||
ip_addresses: Optional[Iterable[str]],
|
||||
extra_addresses: Optional[Iterable[str]] = None,
|
||||
config_path: Optional[Iterable[str]] = None,
|
||||
config_path: Optional[StrSequence] = None,
|
||||
) -> IPSet:
|
||||
"""
|
||||
Generate an IPSet from a list of IP addresses or CIDRs.
|
||||
|
|
@ -115,7 +115,7 @@ def generate_ip_set(
|
|||
|
||||
|
||||
# IP ranges that are considered private / unroutable / don't make sense.
|
||||
DEFAULT_IP_RANGE_BLACKLIST = [
|
||||
DEFAULT_IP_RANGE_BLOCKLIST = [
|
||||
# Localhost
|
||||
"127.0.0.0/8",
|
||||
# Private networks.
|
||||
|
|
@ -501,36 +501,36 @@ class ServerConfig(Config):
|
|||
# due to resource constraints
|
||||
self.admin_contact = config.get("admin_contact", None)
|
||||
|
||||
ip_range_blacklist = config.get(
|
||||
"ip_range_blacklist", DEFAULT_IP_RANGE_BLACKLIST
|
||||
ip_range_blocklist = config.get(
|
||||
"ip_range_blacklist", DEFAULT_IP_RANGE_BLOCKLIST
|
||||
)
|
||||
|
||||
# Attempt to create an IPSet from the given ranges
|
||||
|
||||
# Always blacklist 0.0.0.0, ::
|
||||
self.ip_range_blacklist = generate_ip_set(
|
||||
ip_range_blacklist, ["0.0.0.0", "::"], config_path=("ip_range_blacklist",)
|
||||
# Always block 0.0.0.0, ::
|
||||
self.ip_range_blocklist = generate_ip_set(
|
||||
ip_range_blocklist, ["0.0.0.0", "::"], config_path=("ip_range_blacklist",)
|
||||
)
|
||||
|
||||
self.ip_range_whitelist = generate_ip_set(
|
||||
self.ip_range_allowlist = generate_ip_set(
|
||||
config.get("ip_range_whitelist", ()), config_path=("ip_range_whitelist",)
|
||||
)
|
||||
# The federation_ip_range_blacklist is used for backwards-compatibility
|
||||
# and only applies to federation and identity servers.
|
||||
if "federation_ip_range_blacklist" in config:
|
||||
# Always blacklist 0.0.0.0, ::
|
||||
self.federation_ip_range_blacklist = generate_ip_set(
|
||||
# Always block 0.0.0.0, ::
|
||||
self.federation_ip_range_blocklist = generate_ip_set(
|
||||
config["federation_ip_range_blacklist"],
|
||||
["0.0.0.0", "::"],
|
||||
config_path=("federation_ip_range_blacklist",),
|
||||
)
|
||||
# 'federation_ip_range_whitelist' was never a supported configuration option.
|
||||
self.federation_ip_range_whitelist = None
|
||||
self.federation_ip_range_allowlist = None
|
||||
else:
|
||||
# No backwards-compatiblity requrired, as federation_ip_range_blacklist
|
||||
# is not given. Default to ip_range_blacklist and ip_range_whitelist.
|
||||
self.federation_ip_range_blacklist = self.ip_range_blacklist
|
||||
self.federation_ip_range_whitelist = self.ip_range_whitelist
|
||||
self.federation_ip_range_blocklist = self.ip_range_blocklist
|
||||
self.federation_ip_range_allowlist = self.ip_range_allowlist
|
||||
|
||||
# (undocumented) option for torturing the worker-mode replication a bit,
|
||||
# for testing. The value defines the number of milliseconds to pause before
|
||||
|
|
|
|||
|
|
@ -222,7 +222,7 @@ class WorkerConfig(Config):
|
|||
# itself doesn't need this data as it would never have to talk to itself.
|
||||
instance_map: Dict[str, Any] = config.get("instance_map", {})
|
||||
|
||||
if instance_map and self.instance_name is not MAIN_PROCESS_INSTANCE_NAME:
|
||||
if self.instance_name is not MAIN_PROCESS_INSTANCE_NAME:
|
||||
# The host used to connect to the main synapse
|
||||
main_host = config.get("worker_replication_host", None)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from immutabledict import immutabledict
|
|||
|
||||
from synapse.appservice import ApplicationService
|
||||
from synapse.events import EventBase
|
||||
from synapse.logging.opentracing import tag_args, trace
|
||||
from synapse.types import JsonDict, StateMap
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -242,6 +243,8 @@ class EventContext(UnpersistedEventContextBase):
|
|||
|
||||
return self._state_group
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_current_state_ids(
|
||||
self, state_filter: Optional["StateFilter"] = None
|
||||
) -> Optional[StateMap[str]]:
|
||||
|
|
@ -275,6 +278,8 @@ class EventContext(UnpersistedEventContextBase):
|
|||
|
||||
return prev_state_ids
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_prev_state_ids(
|
||||
self, state_filter: Optional["StateFilter"] = None
|
||||
) -> StateMap[str]:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from typing import (
|
|||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
Match,
|
||||
MutableMapping,
|
||||
Optional,
|
||||
Union,
|
||||
|
|
@ -46,12 +47,10 @@ if TYPE_CHECKING:
|
|||
from synapse.handlers.relations import BundledAggregations
|
||||
|
||||
|
||||
# Split strings on "." but not "\." This uses a negative lookbehind assertion for '\'
|
||||
# (?<!stuff) matches if the current position in the string is not preceded
|
||||
# by a match for 'stuff'.
|
||||
# TODO: This is fast, but fails to handle "foo\\.bar" which should be treated as
|
||||
# the literal fields "foo\" and "bar" but will instead be treated as "foo\\.bar"
|
||||
SPLIT_FIELD_REGEX = re.compile(r"(?<!\\)\.")
|
||||
# Split strings on "." but not "\." (or "\\\.").
|
||||
SPLIT_FIELD_REGEX = re.compile(r"\\*\.")
|
||||
# Find escaped characters, e.g. those with a \ in front of them.
|
||||
ESCAPE_SEQUENCE_PATTERN = re.compile(r"\\(.)")
|
||||
|
||||
CANONICALJSON_MAX_INT = (2**53) - 1
|
||||
CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT
|
||||
|
|
@ -253,6 +252,57 @@ def _copy_field(src: JsonDict, dst: JsonDict, field: List[str]) -> None:
|
|||
sub_out_dict[key_to_move] = sub_dict[key_to_move]
|
||||
|
||||
|
||||
def _escape_slash(m: Match[str]) -> str:
|
||||
"""
|
||||
Replacement function; replace a backslash-backslash or backslash-dot with the
|
||||
second character. Leaves any other string alone.
|
||||
"""
|
||||
if m.group(1) in ("\\", "."):
|
||||
return m.group(1)
|
||||
return m.group(0)
|
||||
|
||||
|
||||
def _split_field(field: str) -> List[str]:
|
||||
"""
|
||||
Splits strings on unescaped dots and removes escaping.
|
||||
|
||||
Args:
|
||||
field: A string representing a path to a field.
|
||||
|
||||
Returns:
|
||||
A list of nested fields to traverse.
|
||||
"""
|
||||
|
||||
# Convert the field and remove escaping:
|
||||
#
|
||||
# 1. "content.body.thing\.with\.dots"
|
||||
# 2. ["content", "body", "thing\.with\.dots"]
|
||||
# 3. ["content", "body", "thing.with.dots"]
|
||||
|
||||
# Find all dots (and their preceding backslashes). If the dot is unescaped
|
||||
# then emit a new field part.
|
||||
result = []
|
||||
prev_start = 0
|
||||
for match in SPLIT_FIELD_REGEX.finditer(field):
|
||||
# If the match is an *even* number of characters than the dot was escaped.
|
||||
if len(match.group()) % 2 == 0:
|
||||
continue
|
||||
|
||||
# Add a new part (up to the dot, exclusive) after escaping.
|
||||
result.append(
|
||||
ESCAPE_SEQUENCE_PATTERN.sub(
|
||||
_escape_slash, field[prev_start : match.end() - 1]
|
||||
)
|
||||
)
|
||||
prev_start = match.end()
|
||||
|
||||
# Add any part of the field after the last unescaped dot. (Note that if the
|
||||
# character is a dot this correctly adds a blank string.)
|
||||
result.append(re.sub(r"\\(.)", _escape_slash, field[prev_start:]))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def only_fields(dictionary: JsonDict, fields: List[str]) -> JsonDict:
|
||||
"""Return a new dict with only the fields in 'dictionary' which are present
|
||||
in 'fields'.
|
||||
|
|
@ -260,7 +310,7 @@ def only_fields(dictionary: JsonDict, fields: List[str]) -> JsonDict:
|
|||
If there are no event fields specified then all fields are included.
|
||||
The entries may include '.' characters to indicate sub-fields.
|
||||
So ['content.body'] will include the 'body' field of the 'content' object.
|
||||
A literal '.' character in a field name may be escaped using a '\'.
|
||||
A literal '.' or '\' character in a field name may be escaped using a '\'.
|
||||
|
||||
Args:
|
||||
dictionary: The dictionary to read from.
|
||||
|
|
@ -275,13 +325,7 @@ def only_fields(dictionary: JsonDict, fields: List[str]) -> JsonDict:
|
|||
|
||||
# for each field, convert it:
|
||||
# ["content.body.thing\.with\.dots"] => [["content", "body", "thing\.with\.dots"]]
|
||||
split_fields = [SPLIT_FIELD_REGEX.split(f) for f in fields]
|
||||
|
||||
# for each element of the output array of arrays:
|
||||
# remove escaping so we can use the right key names.
|
||||
split_fields[:] = [
|
||||
[f.replace(r"\.", r".") for f in field_array] for field_array in split_fields
|
||||
]
|
||||
split_fields = [_split_field(f) for f in fields]
|
||||
|
||||
output: JsonDict = {}
|
||||
for field_array in split_fields:
|
||||
|
|
|
|||
|
|
@ -236,6 +236,7 @@ class FederationClient(FederationBase):
|
|||
|
||||
async def claim_client_keys(
|
||||
self,
|
||||
user: UserID,
|
||||
destination: str,
|
||||
query: Dict[str, Dict[str, Dict[str, int]]],
|
||||
timeout: Optional[int],
|
||||
|
|
@ -243,6 +244,7 @@ class FederationClient(FederationBase):
|
|||
"""Claims one-time keys for a device hosted on a remote server.
|
||||
|
||||
Args:
|
||||
user: The user id of the requesting user
|
||||
destination: Domain name of the remote homeserver
|
||||
content: The query content.
|
||||
|
||||
|
|
@ -279,7 +281,7 @@ class FederationClient(FederationBase):
|
|||
if use_unstable:
|
||||
try:
|
||||
return await self.transport_layer.claim_client_keys_unstable(
|
||||
destination, unstable_content, timeout
|
||||
user, destination, unstable_content, timeout
|
||||
)
|
||||
except HttpResponseException as e:
|
||||
# If an error is received that is due to an unrecognised endpoint,
|
||||
|
|
@ -295,7 +297,7 @@ class FederationClient(FederationBase):
|
|||
logger.debug("Skipping unstable claim client keys API")
|
||||
|
||||
return await self.transport_layer.claim_client_keys(
|
||||
destination, content, timeout
|
||||
user, destination, content, timeout
|
||||
)
|
||||
|
||||
@trace
|
||||
|
|
|
|||
|
|
@ -739,12 +739,10 @@ class FederationServer(FederationBase):
|
|||
"event": event_json,
|
||||
"state": [p.get_pdu_json(time_now) for p in state_events],
|
||||
"auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events],
|
||||
"org.matrix.msc3706.partial_state": caller_supports_partial_state,
|
||||
"members_omitted": caller_supports_partial_state,
|
||||
}
|
||||
|
||||
if servers_in_room is not None:
|
||||
resp["org.matrix.msc3706.servers_in_room"] = list(servers_in_room)
|
||||
resp["servers_in_room"] = list(servers_in_room)
|
||||
|
||||
return resp
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ from synapse.events import EventBase, make_event_from_dict
|
|||
from synapse.federation.units import Transaction
|
||||
from synapse.http.matrixfederationclient import ByteParser, LegacyJsonSendParser
|
||||
from synapse.http.types import QueryParams
|
||||
from synapse.types import JsonDict
|
||||
from synapse.types import JsonDict, UserID
|
||||
from synapse.util import ExceptionBundle
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -59,7 +59,6 @@ class TransportLayerClient:
|
|||
|
||||
def __init__(self, hs: "HomeServer"):
|
||||
self.client = hs.get_federation_http_client()
|
||||
self._faster_joins_enabled = hs.config.experimental.faster_joins_enabled
|
||||
self._is_mine_server_name = hs.is_mine_server_name
|
||||
|
||||
async def get_room_state_ids(
|
||||
|
|
@ -363,12 +362,8 @@ class TransportLayerClient:
|
|||
) -> "SendJoinResponse":
|
||||
path = _create_v2_path("/send_join/%s/%s", room_id, event_id)
|
||||
query_params: Dict[str, str] = {}
|
||||
if self._faster_joins_enabled:
|
||||
# lazy-load state on join
|
||||
query_params["org.matrix.msc3706.partial_state"] = (
|
||||
"true" if omit_members else "false"
|
||||
)
|
||||
query_params["omit_members"] = "true" if omit_members else "false"
|
||||
# lazy-load state on join
|
||||
query_params["omit_members"] = "true" if omit_members else "false"
|
||||
|
||||
return await self.client.put_json(
|
||||
destination=destination,
|
||||
|
|
@ -635,7 +630,11 @@ class TransportLayerClient:
|
|||
)
|
||||
|
||||
async def claim_client_keys(
|
||||
self, destination: str, query_content: JsonDict, timeout: Optional[int]
|
||||
self,
|
||||
user: UserID,
|
||||
destination: str,
|
||||
query_content: JsonDict,
|
||||
timeout: Optional[int],
|
||||
) -> JsonDict:
|
||||
"""Claim one-time keys for a list of devices hosted on a remote server.
|
||||
|
||||
|
|
@ -660,6 +659,7 @@ class TransportLayerClient:
|
|||
}
|
||||
|
||||
Args:
|
||||
user: the user_id of the requesting user
|
||||
destination: The server to query.
|
||||
query_content: The user ids to query.
|
||||
Returns:
|
||||
|
|
@ -676,7 +676,11 @@ class TransportLayerClient:
|
|||
)
|
||||
|
||||
async def claim_client_keys_unstable(
|
||||
self, destination: str, query_content: JsonDict, timeout: Optional[int]
|
||||
self,
|
||||
user: UserID,
|
||||
destination: str,
|
||||
query_content: JsonDict,
|
||||
timeout: Optional[int],
|
||||
) -> JsonDict:
|
||||
"""Claim one-time keys for a list of devices hosted on a remote server.
|
||||
|
||||
|
|
@ -701,6 +705,7 @@ class TransportLayerClient:
|
|||
}
|
||||
|
||||
Args:
|
||||
user: the user_id of the requesting user
|
||||
destination: The server to query.
|
||||
query_content: The user ids to query.
|
||||
Returns:
|
||||
|
|
@ -902,9 +907,7 @@ def _members_omitted_parser(response: SendJoinResponse) -> Generator[None, Any,
|
|||
while True:
|
||||
val = yield
|
||||
if not isinstance(val, bool):
|
||||
raise TypeError(
|
||||
"members_omitted (formerly org.matrix.msc370c.partial_state) must be a boolean"
|
||||
)
|
||||
raise TypeError("members_omitted must be a boolean")
|
||||
response.members_omitted = val
|
||||
|
||||
|
||||
|
|
@ -964,14 +967,6 @@ class SendJoinParser(ByteParser[SendJoinResponse]):
|
|||
]
|
||||
|
||||
if not v1_api:
|
||||
self._coros.append(
|
||||
ijson.items_coro(
|
||||
_members_omitted_parser(self._response),
|
||||
"org.matrix.msc3706.partial_state",
|
||||
use_float="True",
|
||||
)
|
||||
)
|
||||
# The stable field name comes last, so it "wins" if the fields disagree
|
||||
self._coros.append(
|
||||
ijson.items_coro(
|
||||
_members_omitted_parser(self._response),
|
||||
|
|
@ -980,14 +975,6 @@ class SendJoinParser(ByteParser[SendJoinResponse]):
|
|||
)
|
||||
)
|
||||
|
||||
self._coros.append(
|
||||
ijson.items_coro(
|
||||
_servers_in_room_parser(self._response),
|
||||
"org.matrix.msc3706.servers_in_room",
|
||||
use_float="True",
|
||||
)
|
||||
)
|
||||
|
||||
# Again, stable field name comes last
|
||||
self._coros.append(
|
||||
ijson.items_coro(
|
||||
|
|
|
|||
|
|
@ -440,7 +440,6 @@ class FederationV2SendJoinServlet(BaseFederationServerServlet):
|
|||
server_name: str,
|
||||
):
|
||||
super().__init__(hs, authenticator, ratelimiter, server_name)
|
||||
self._read_msc3706_query_param = hs.config.experimental.msc3706_enabled
|
||||
|
||||
async def on_PUT(
|
||||
self,
|
||||
|
|
@ -453,16 +452,7 @@ class FederationV2SendJoinServlet(BaseFederationServerServlet):
|
|||
# TODO(paul): assert that event_id parsed from path actually
|
||||
# match those given in content
|
||||
|
||||
partial_state = False
|
||||
# The stable query parameter wins, if it disagrees with the unstable
|
||||
# parameter for some reason.
|
||||
stable_param = parse_boolean_from_args(query, "omit_members", default=None)
|
||||
if stable_param is not None:
|
||||
partial_state = stable_param
|
||||
elif self._read_msc3706_query_param:
|
||||
partial_state = parse_boolean_from_args(
|
||||
query, "org.matrix.msc3706.partial_state", default=False
|
||||
)
|
||||
partial_state = parse_boolean_from_args(query, "omit_members", default=False)
|
||||
|
||||
result = await self.handler.on_send_join_request(
|
||||
origin, content, room_id, caller_supports_partial_state=partial_state
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@ from synapse.api.errors import (
|
|||
NotFoundError,
|
||||
StoreError,
|
||||
SynapseError,
|
||||
UserDeactivatedError,
|
||||
)
|
||||
from synapse.api.ratelimiting import Ratelimiter
|
||||
from synapse.handlers.ui_auth import (
|
||||
|
|
@ -1419,12 +1418,6 @@ class AuthHandler:
|
|||
return None
|
||||
(user_id, password_hash) = lookupres
|
||||
|
||||
# If the password hash is None, the account has likely been deactivated
|
||||
if not password_hash:
|
||||
deactivated = await self.store.get_user_deactivated_status(user_id)
|
||||
if deactivated:
|
||||
raise UserDeactivatedError("This account has been deactivated")
|
||||
|
||||
result = await self.validate_hash(password, password_hash)
|
||||
if not result:
|
||||
logger.warning("Failed password login for user %s", user_id)
|
||||
|
|
@ -1749,8 +1742,11 @@ class AuthHandler:
|
|||
registered.
|
||||
auth_provider_session_id: The session ID from the SSO IdP received during login.
|
||||
"""
|
||||
# If the account has been deactivated, do not proceed with the login
|
||||
# flow.
|
||||
# If the account has been deactivated, do not proceed with the login.
|
||||
#
|
||||
# This gets checked again when the token is submitted but this lets us
|
||||
# provide an HTML error page to the user (instead of issuing a token and
|
||||
# having it error later).
|
||||
deactivated = await self.store.get_user_deactivated_status(registered_user_id)
|
||||
if deactivated:
|
||||
respond_with_html(request, 403, self._sso_account_deactivated_template)
|
||||
|
|
|
|||
|
|
@ -661,6 +661,7 @@ class E2eKeysHandler:
|
|||
async def claim_one_time_keys(
|
||||
self,
|
||||
query: Dict[str, Dict[str, Dict[str, int]]],
|
||||
user: UserID,
|
||||
timeout: Optional[int],
|
||||
always_include_fallback_keys: bool,
|
||||
) -> JsonDict:
|
||||
|
|
@ -703,7 +704,7 @@ class E2eKeysHandler:
|
|||
device_keys = remote_queries[destination]
|
||||
try:
|
||||
remote_result = await self.federation.claim_client_keys(
|
||||
destination, device_keys, timeout=timeout
|
||||
user, destination, device_keys, timeout=timeout
|
||||
)
|
||||
for user_id, keys in remote_result["one_time_keys"].items():
|
||||
if user_id in device_keys:
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ class FederationHandler:
|
|||
self._event_auth_handler = hs.get_event_auth_handler()
|
||||
self._server_notices_mxid = hs.config.servernotices.server_notices_mxid
|
||||
self.config = hs.config
|
||||
self.http_client = hs.get_proxied_blacklisted_http_client()
|
||||
self.http_client = hs.get_proxied_blocklisted_http_client()
|
||||
self._replication = hs.get_replication_data_handler()
|
||||
self._federation_event_handler = hs.get_federation_event_handler()
|
||||
self._device_handler = hs.get_device_handler()
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ from synapse.types import (
|
|||
)
|
||||
from synapse.types.state import StateFilter
|
||||
from synapse.util.async_helpers import Linearizer, concurrently_execute
|
||||
from synapse.util.iterutils import batch_iter
|
||||
from synapse.util.iterutils import batch_iter, partition
|
||||
from synapse.util.retryutils import NotRetryingDestination
|
||||
from synapse.util.stringutils import shortstr
|
||||
|
||||
|
|
@ -865,7 +865,7 @@ class FederationEventHandler:
|
|||
[event.event_id for event in events]
|
||||
)
|
||||
|
||||
new_events = []
|
||||
new_events: List[EventBase] = []
|
||||
for event in events:
|
||||
event_id = event.event_id
|
||||
|
||||
|
|
@ -890,12 +890,71 @@ class FederationEventHandler:
|
|||
# Continue on with the events that are new to us.
|
||||
new_events.append(event)
|
||||
|
||||
# We want to sort these by depth so we process them and
|
||||
# tell clients about them in order.
|
||||
sorted_events = sorted(new_events, key=lambda x: x.depth)
|
||||
for ev in sorted_events:
|
||||
with nested_logging_context(ev.event_id):
|
||||
await self._process_pulled_event(origin, ev, backfilled=backfilled)
|
||||
set_tag(
|
||||
SynapseTags.RESULT_PREFIX + "new_events.length",
|
||||
str(len(new_events)),
|
||||
)
|
||||
|
||||
@trace
|
||||
async def _process_new_pulled_events(new_events: Collection[EventBase]) -> None:
|
||||
# We want to sort these by depth so we process them and tell clients about
|
||||
# them in order. It's also more efficient to backfill this way (`depth`
|
||||
# ascending) because one backfill event is likely to be the `prev_event` of
|
||||
# the next event we're going to process.
|
||||
sorted_events = sorted(new_events, key=lambda x: x.depth)
|
||||
for ev in sorted_events:
|
||||
with nested_logging_context(ev.event_id):
|
||||
await self._process_pulled_event(origin, ev, backfilled=backfilled)
|
||||
|
||||
# Check if we've already tried to process these events at some point in the
|
||||
# past. We aren't concerned with the expontntial backoff here, just whether it
|
||||
# has failed to be processed before.
|
||||
event_ids_with_failed_pull_attempts = (
|
||||
await self._store.get_event_ids_with_failed_pull_attempts(
|
||||
[event.event_id for event in new_events]
|
||||
)
|
||||
)
|
||||
|
||||
# We construct the event lists in source order from `/backfill` response because
|
||||
# it's a) easiest, but also b) the order in which we process things matters for
|
||||
# MSC2716 historical batches because many historical events are all at the same
|
||||
# `depth` and we rely on the tenuous sort that the other server gave us and hope
|
||||
# they're doing their best. The brittle nature of this ordering for historical
|
||||
# messages over federation is one of the reasons why we don't want to continue
|
||||
# on MSC2716 until we have online topological ordering.
|
||||
events_with_failed_pull_attempts, fresh_events = partition(
|
||||
new_events, lambda e: e.event_id in event_ids_with_failed_pull_attempts
|
||||
)
|
||||
set_tag(
|
||||
SynapseTags.FUNC_ARG_PREFIX + "events_with_failed_pull_attempts",
|
||||
str(event_ids_with_failed_pull_attempts),
|
||||
)
|
||||
set_tag(
|
||||
SynapseTags.RESULT_PREFIX + "events_with_failed_pull_attempts.length",
|
||||
str(len(events_with_failed_pull_attempts)),
|
||||
)
|
||||
set_tag(
|
||||
SynapseTags.FUNC_ARG_PREFIX + "fresh_events",
|
||||
str([event.event_id for event in fresh_events]),
|
||||
)
|
||||
set_tag(
|
||||
SynapseTags.RESULT_PREFIX + "fresh_events.length",
|
||||
str(len(fresh_events)),
|
||||
)
|
||||
|
||||
# Process previously failed backfill events in the background to not waste
|
||||
# time on something that is likely to fail again.
|
||||
if len(events_with_failed_pull_attempts) > 0:
|
||||
run_as_background_process(
|
||||
"_process_new_pulled_events_with_failed_pull_attempts",
|
||||
_process_new_pulled_events,
|
||||
events_with_failed_pull_attempts,
|
||||
)
|
||||
|
||||
# We can optimistically try to process and wait for the event to be fully
|
||||
# persisted if we've never tried before.
|
||||
if len(fresh_events) > 0:
|
||||
await _process_new_pulled_events(fresh_events)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
|
|
|
|||
|
|
@ -52,10 +52,10 @@ class IdentityHandler:
|
|||
# An HTTP client for contacting trusted URLs.
|
||||
self.http_client = SimpleHttpClient(hs)
|
||||
# An HTTP client for contacting identity servers specified by clients.
|
||||
self.blacklisting_http_client = SimpleHttpClient(
|
||||
self._http_client = SimpleHttpClient(
|
||||
hs,
|
||||
ip_blacklist=hs.config.server.federation_ip_range_blacklist,
|
||||
ip_whitelist=hs.config.server.federation_ip_range_whitelist,
|
||||
ip_blocklist=hs.config.server.federation_ip_range_blocklist,
|
||||
ip_allowlist=hs.config.server.federation_ip_range_allowlist,
|
||||
)
|
||||
self.federation_http_client = hs.get_federation_http_client()
|
||||
self.hs = hs
|
||||
|
|
@ -197,7 +197,7 @@ class IdentityHandler:
|
|||
try:
|
||||
# Use the blacklisting http client as this call is only to identity servers
|
||||
# provided by a client
|
||||
data = await self.blacklisting_http_client.post_json_get_json(
|
||||
data = await self._http_client.post_json_get_json(
|
||||
bind_url, bind_data, headers=headers
|
||||
)
|
||||
|
||||
|
|
@ -308,9 +308,7 @@ class IdentityHandler:
|
|||
try:
|
||||
# Use the blacklisting http client as this call is only to identity servers
|
||||
# provided by a client
|
||||
await self.blacklisting_http_client.post_json_get_json(
|
||||
url, content, headers
|
||||
)
|
||||
await self._http_client.post_json_get_json(url, content, headers)
|
||||
changed = True
|
||||
except HttpResponseException as e:
|
||||
changed = False
|
||||
|
|
@ -579,7 +577,7 @@ class IdentityHandler:
|
|||
"""
|
||||
# Check what hashing details are supported by this identity server
|
||||
try:
|
||||
hash_details = await self.blacklisting_http_client.get_json(
|
||||
hash_details = await self._http_client.get_json(
|
||||
"%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
|
||||
{"access_token": id_access_token},
|
||||
)
|
||||
|
|
@ -646,7 +644,7 @@ class IdentityHandler:
|
|||
headers = {"Authorization": create_id_access_token_header(id_access_token)}
|
||||
|
||||
try:
|
||||
lookup_results = await self.blacklisting_http_client.post_json_get_json(
|
||||
lookup_results = await self._http_client.post_json_get_json(
|
||||
"%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
|
||||
{
|
||||
"addresses": [lookup_value],
|
||||
|
|
@ -752,7 +750,7 @@ class IdentityHandler:
|
|||
|
||||
url = "%s%s/_matrix/identity/v2/store-invite" % (id_server_scheme, id_server)
|
||||
try:
|
||||
data = await self.blacklisting_http_client.post_json_get_json(
|
||||
data = await self._http_client.post_json_get_json(
|
||||
url,
|
||||
invite_config,
|
||||
{"Authorization": create_id_access_token_header(id_access_token)},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
# Copyright 2023 Matrix.org Foundation C.I.C.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from authlib.jose import JsonWebToken, JWTClaims
|
||||
from authlib.jose.errors import BadSignatureError, InvalidClaimError, JoseError
|
||||
|
||||
from synapse.api.errors import Codes, LoginError
|
||||
from synapse.types import JsonDict, UserID
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from synapse.server import HomeServer
|
||||
|
||||
|
||||
class JwtHandler:
|
||||
def __init__(self, hs: "HomeServer"):
|
||||
self.hs = hs
|
||||
|
||||
self.jwt_secret = hs.config.jwt.jwt_secret
|
||||
self.jwt_subject_claim = hs.config.jwt.jwt_subject_claim
|
||||
self.jwt_algorithm = hs.config.jwt.jwt_algorithm
|
||||
self.jwt_issuer = hs.config.jwt.jwt_issuer
|
||||
self.jwt_audiences = hs.config.jwt.jwt_audiences
|
||||
|
||||
def validate_login(self, login_submission: JsonDict) -> str:
|
||||
"""
|
||||
Authenticates the user for the /login API
|
||||
|
||||
Args:
|
||||
login_submission: the whole of the login submission
|
||||
(including 'type' and other relevant fields)
|
||||
|
||||
Returns:
|
||||
The user ID that is logging in.
|
||||
|
||||
Raises:
|
||||
LoginError if there was an authentication problem.
|
||||
"""
|
||||
token = login_submission.get("token", None)
|
||||
if token is None:
|
||||
raise LoginError(
|
||||
403, "Token field for JWT is missing", errcode=Codes.FORBIDDEN
|
||||
)
|
||||
|
||||
jwt = JsonWebToken([self.jwt_algorithm])
|
||||
claim_options = {}
|
||||
if self.jwt_issuer is not None:
|
||||
claim_options["iss"] = {"value": self.jwt_issuer, "essential": True}
|
||||
if self.jwt_audiences is not None:
|
||||
claim_options["aud"] = {"values": self.jwt_audiences, "essential": True}
|
||||
|
||||
try:
|
||||
claims = jwt.decode(
|
||||
token,
|
||||
key=self.jwt_secret,
|
||||
claims_cls=JWTClaims,
|
||||
claims_options=claim_options,
|
||||
)
|
||||
except BadSignatureError:
|
||||
# We handle this case separately to provide a better error message
|
||||
raise LoginError(
|
||||
403,
|
||||
"JWT validation failed: Signature verification failed",
|
||||
errcode=Codes.FORBIDDEN,
|
||||
)
|
||||
except JoseError as e:
|
||||
# A JWT error occurred, return some info back to the client.
|
||||
raise LoginError(
|
||||
403,
|
||||
"JWT validation failed: %s" % (str(e),),
|
||||
errcode=Codes.FORBIDDEN,
|
||||
)
|
||||
|
||||
try:
|
||||
claims.validate(leeway=120) # allows 2 min of clock skew
|
||||
|
||||
# Enforce the old behavior which is rolled out in productive
|
||||
# servers: if the JWT contains an 'aud' claim but none is
|
||||
# configured, the login attempt will fail
|
||||
if claims.get("aud") is not None:
|
||||
if self.jwt_audiences is None or len(self.jwt_audiences) == 0:
|
||||
raise InvalidClaimError("aud")
|
||||
except JoseError as e:
|
||||
raise LoginError(
|
||||
403,
|
||||
"JWT validation failed: %s" % (str(e),),
|
||||
errcode=Codes.FORBIDDEN,
|
||||
)
|
||||
|
||||
user = claims.get(self.jwt_subject_claim, None)
|
||||
if user is None:
|
||||
raise LoginError(403, "Invalid JWT", errcode=Codes.FORBIDDEN)
|
||||
|
||||
return UserID(user, self.hs.hostname).to_string()
|
||||
|
|
@ -16,6 +16,7 @@ import logging
|
|||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from synapse.api.constants import ReceiptTypes
|
||||
from synapse.api.errors import SynapseError
|
||||
from synapse.util.async_helpers import Linearizer
|
||||
from synapse.types import JsonDict
|
||||
|
||||
|
|
@ -52,12 +53,21 @@ class ReadMarkerHandler:
|
|||
)
|
||||
|
||||
should_update = True
|
||||
# Get event ordering, this also ensures we know about the event
|
||||
event_ordering = await self.store.get_event_ordering(event_id)
|
||||
|
||||
if existing_read_marker:
|
||||
# Only update if the new marker is ahead in the stream
|
||||
should_update = await self.store.is_event_after(
|
||||
event_id, existing_read_marker["event_id"]
|
||||
)
|
||||
try:
|
||||
old_event_ordering = await self.store.get_event_ordering(
|
||||
existing_read_marker["event_id"]
|
||||
)
|
||||
except SynapseError:
|
||||
# Old event no longer exists, assume new is ahead. This may
|
||||
# happen if the old event was removed due to retention.
|
||||
pass
|
||||
else:
|
||||
# Only update if the new marker is ahead in the stream
|
||||
should_update = event_ordering > old_event_ordering
|
||||
|
||||
if should_update:
|
||||
content = {"event_id": event_id, **(extra_content or {})}
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@ class SsoHandler:
|
|||
self._media_repo = (
|
||||
hs.get_media_repository() if hs.config.media.can_load_media_repo else None
|
||||
)
|
||||
self._http_client = hs.get_proxied_blacklisted_http_client()
|
||||
self._http_client = hs.get_proxied_blocklisted_http_client()
|
||||
|
||||
# The following template is shown after a successful user interactive
|
||||
# authentication session. It tells the user they can close the window.
|
||||
|
|
|
|||
|
|
@ -117,22 +117,22 @@ RawHeaderValue = Union[
|
|||
]
|
||||
|
||||
|
||||
def check_against_blacklist(
|
||||
ip_address: IPAddress, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet
|
||||
def _is_ip_blocked(
|
||||
ip_address: IPAddress, allowlist: Optional[IPSet], blocklist: IPSet
|
||||
) -> bool:
|
||||
"""
|
||||
Compares an IP address to allowed and disallowed IP sets.
|
||||
|
||||
Args:
|
||||
ip_address: The IP address to check
|
||||
ip_whitelist: Allowed IP addresses.
|
||||
ip_blacklist: Disallowed IP addresses.
|
||||
allowlist: Allowed IP addresses.
|
||||
blocklist: Disallowed IP addresses.
|
||||
|
||||
Returns:
|
||||
True if the IP address is in the blacklist and not in the whitelist.
|
||||
True if the IP address is in the blocklist and not in the allowlist.
|
||||
"""
|
||||
if ip_address in ip_blacklist:
|
||||
if ip_whitelist is None or ip_address not in ip_whitelist:
|
||||
if ip_address in blocklist:
|
||||
if allowlist is None or ip_address not in allowlist:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
@ -154,27 +154,27 @@ def _make_scheduler(
|
|||
return _scheduler
|
||||
|
||||
|
||||
class _IPBlacklistingResolver:
|
||||
class _IPBlockingResolver:
|
||||
"""
|
||||
A proxy for reactor.nameResolver which only produces non-blacklisted IP
|
||||
addresses, preventing DNS rebinding attacks on URL preview.
|
||||
A proxy for reactor.nameResolver which only produces non-blocklisted IP
|
||||
addresses, preventing DNS rebinding attacks.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
reactor: IReactorPluggableNameResolver,
|
||||
ip_whitelist: Optional[IPSet],
|
||||
ip_blacklist: IPSet,
|
||||
ip_allowlist: Optional[IPSet],
|
||||
ip_blocklist: IPSet,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
reactor: The twisted reactor.
|
||||
ip_whitelist: IP addresses to allow.
|
||||
ip_blacklist: IP addresses to disallow.
|
||||
ip_allowlist: IP addresses to allow.
|
||||
ip_blocklist: IP addresses to disallow.
|
||||
"""
|
||||
self._reactor = reactor
|
||||
self._ip_whitelist = ip_whitelist
|
||||
self._ip_blacklist = ip_blacklist
|
||||
self._ip_allowlist = ip_allowlist
|
||||
self._ip_blocklist = ip_blocklist
|
||||
|
||||
def resolveHostName(
|
||||
self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0
|
||||
|
|
@ -191,16 +191,13 @@ class _IPBlacklistingResolver:
|
|||
|
||||
ip_address = IPAddress(address.host)
|
||||
|
||||
if check_against_blacklist(
|
||||
ip_address, self._ip_whitelist, self._ip_blacklist
|
||||
):
|
||||
if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist):
|
||||
logger.info(
|
||||
"Dropped %s from DNS resolution to %s due to blacklist"
|
||||
% (ip_address, hostname)
|
||||
"Blocked %s from DNS resolution to %s" % (ip_address, hostname)
|
||||
)
|
||||
has_bad_ip = True
|
||||
|
||||
# if we have a blacklisted IP, we'd like to raise an error to block the
|
||||
# if we have a blocked IP, we'd like to raise an error to block the
|
||||
# request, but all we can really do from here is claim that there were no
|
||||
# valid results.
|
||||
if not has_bad_ip:
|
||||
|
|
@ -232,24 +229,24 @@ class _IPBlacklistingResolver:
|
|||
# ISynapseReactor implies IReactorCore, but explicitly marking it this as an implementer
|
||||
# of IReactorCore seems to keep mypy-zope happier.
|
||||
@implementer(IReactorCore, ISynapseReactor)
|
||||
class BlacklistingReactorWrapper:
|
||||
class BlocklistingReactorWrapper:
|
||||
"""
|
||||
A Reactor wrapper which will prevent DNS resolution to blacklisted IP
|
||||
A Reactor wrapper which will prevent DNS resolution to blocked IP
|
||||
addresses, to prevent DNS rebinding.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
reactor: IReactorPluggableNameResolver,
|
||||
ip_whitelist: Optional[IPSet],
|
||||
ip_blacklist: IPSet,
|
||||
ip_allowlist: Optional[IPSet],
|
||||
ip_blocklist: IPSet,
|
||||
):
|
||||
self._reactor = reactor
|
||||
|
||||
# We need to use a DNS resolver which filters out blacklisted IP
|
||||
# We need to use a DNS resolver which filters out blocked IP
|
||||
# addresses, to prevent DNS rebinding.
|
||||
self._nameResolver = _IPBlacklistingResolver(
|
||||
self._reactor, ip_whitelist, ip_blacklist
|
||||
self._nameResolver = _IPBlockingResolver(
|
||||
self._reactor, ip_allowlist, ip_blocklist
|
||||
)
|
||||
|
||||
def __getattr__(self, attr: str) -> Any:
|
||||
|
|
@ -260,7 +257,7 @@ class BlacklistingReactorWrapper:
|
|||
return getattr(self._reactor, attr)
|
||||
|
||||
|
||||
class BlacklistingAgentWrapper(Agent):
|
||||
class BlocklistingAgentWrapper(Agent):
|
||||
"""
|
||||
An Agent wrapper which will prevent access to IP addresses being accessed
|
||||
directly (without an IP address lookup).
|
||||
|
|
@ -269,18 +266,18 @@ class BlacklistingAgentWrapper(Agent):
|
|||
def __init__(
|
||||
self,
|
||||
agent: IAgent,
|
||||
ip_blacklist: IPSet,
|
||||
ip_whitelist: Optional[IPSet] = None,
|
||||
ip_blocklist: IPSet,
|
||||
ip_allowlist: Optional[IPSet] = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
agent: The Agent to wrap.
|
||||
ip_whitelist: IP addresses to allow.
|
||||
ip_blacklist: IP addresses to disallow.
|
||||
ip_allowlist: IP addresses to allow.
|
||||
ip_blocklist: IP addresses to disallow.
|
||||
"""
|
||||
self._agent = agent
|
||||
self._ip_whitelist = ip_whitelist
|
||||
self._ip_blacklist = ip_blacklist
|
||||
self._ip_allowlist = ip_allowlist
|
||||
self._ip_blocklist = ip_blocklist
|
||||
|
||||
def request(
|
||||
self,
|
||||
|
|
@ -299,13 +296,9 @@ class BlacklistingAgentWrapper(Agent):
|
|||
# Not an IP
|
||||
pass
|
||||
else:
|
||||
if check_against_blacklist(
|
||||
ip_address, self._ip_whitelist, self._ip_blacklist
|
||||
):
|
||||
logger.info("Blocking access to %s due to blacklist" % (ip_address,))
|
||||
e = SynapseError(
|
||||
HTTPStatus.FORBIDDEN, "IP address blocked by IP blacklist entry"
|
||||
)
|
||||
if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist):
|
||||
logger.info("Blocking access to %s" % (ip_address,))
|
||||
e = SynapseError(HTTPStatus.FORBIDDEN, "IP address blocked")
|
||||
return defer.fail(Failure(e))
|
||||
|
||||
return self._agent.request(
|
||||
|
|
@ -763,10 +756,9 @@ class SimpleHttpClient(BaseHttpClient):
|
|||
Args:
|
||||
hs: The HomeServer instance to pass in
|
||||
treq_args: Extra keyword arguments to be given to treq.request.
|
||||
ip_blacklist: The IP addresses that are blacklisted that
|
||||
we may not request.
|
||||
ip_whitelist: The whitelisted IP addresses, that we can
|
||||
request if it were otherwise caught in a blacklist.
|
||||
ip_blocklist: The IP addresses that we may not request.
|
||||
ip_allowlist: The allowed IP addresses, that we can
|
||||
request if it were otherwise caught in a blocklist.
|
||||
use_proxy: Whether proxy settings should be discovered and used
|
||||
from conventional environment variables.
|
||||
"""
|
||||
|
|
@ -775,19 +767,19 @@ class SimpleHttpClient(BaseHttpClient):
|
|||
self,
|
||||
hs: "HomeServer",
|
||||
treq_args: Optional[Dict[str, Any]] = None,
|
||||
ip_whitelist: Optional[IPSet] = None,
|
||||
ip_blacklist: Optional[IPSet] = None,
|
||||
ip_allowlist: Optional[IPSet] = None,
|
||||
ip_blocklist: Optional[IPSet] = None,
|
||||
use_proxy: bool = False,
|
||||
):
|
||||
super().__init__(hs, treq_args=treq_args)
|
||||
self._ip_whitelist = ip_whitelist
|
||||
self._ip_blacklist = ip_blacklist
|
||||
self._ip_allowlist = ip_allowlist
|
||||
self._ip_blocklist = ip_blocklist
|
||||
|
||||
if self._ip_blacklist:
|
||||
# If we have an IP blacklist, we need to use a DNS resolver which
|
||||
# filters out blacklisted IP addresses, to prevent DNS rebinding.
|
||||
self.reactor: ISynapseReactor = BlacklistingReactorWrapper(
|
||||
self.reactor, self._ip_whitelist, self._ip_blacklist
|
||||
if self._ip_blocklist:
|
||||
# If we have an IP blocklist, we need to use a DNS resolver which
|
||||
# filters out blocked IP addresses, to prevent DNS rebinding.
|
||||
self.reactor: ISynapseReactor = BlocklistingReactorWrapper(
|
||||
self.reactor, self._ip_allowlist, self._ip_blocklist
|
||||
)
|
||||
|
||||
# the pusher makes lots of concurrent SSL connections to Sygnal, and tends to
|
||||
|
|
@ -809,14 +801,13 @@ class SimpleHttpClient(BaseHttpClient):
|
|||
use_proxy=use_proxy,
|
||||
)
|
||||
|
||||
if self._ip_blacklist:
|
||||
# If we have an IP blacklist, we then install the blacklisting Agent
|
||||
# which prevents direct access to IP addresses, that are not caught
|
||||
# by the DNS resolution.
|
||||
self.agent = BlacklistingAgentWrapper(
|
||||
if self._ip_blocklist:
|
||||
# If we have an IP blocklist, we then install the Agent which prevents
|
||||
# direct access to IP addresses, that are not caught by the DNS resolution.
|
||||
self.agent = BlocklistingAgentWrapper(
|
||||
self.agent,
|
||||
ip_blacklist=self._ip_blacklist,
|
||||
ip_whitelist=self._ip_whitelist,
|
||||
ip_blocklist=self._ip_blocklist,
|
||||
ip_allowlist=self._ip_allowlist,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -844,6 +835,7 @@ class ReplicationClient(BaseHttpClient):
|
|||
|
||||
self.agent: IAgent = ReplicationAgent(
|
||||
hs.get_reactor(),
|
||||
hs.config.worker.instance_map,
|
||||
contextFactory=hs.get_http_client_context_factory(),
|
||||
pool=pool,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer, IResp
|
|||
|
||||
from synapse.crypto.context_factory import FederationPolicyForHTTPS
|
||||
from synapse.http import proxyagent
|
||||
from synapse.http.client import BlacklistingAgentWrapper, BlacklistingReactorWrapper
|
||||
from synapse.http.client import BlocklistingAgentWrapper, BlocklistingReactorWrapper
|
||||
from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint
|
||||
from synapse.http.federation.srv_resolver import Server, SrvResolver
|
||||
from synapse.http.federation.well_known_resolver import WellKnownResolver
|
||||
|
|
@ -65,12 +65,12 @@ class MatrixFederationAgent:
|
|||
user_agent:
|
||||
The user agent header to use for federation requests.
|
||||
|
||||
ip_whitelist: Allowed IP addresses.
|
||||
ip_allowlist: Allowed IP addresses.
|
||||
|
||||
ip_blacklist: Disallowed IP addresses.
|
||||
ip_blocklist: Disallowed IP addresses.
|
||||
|
||||
proxy_reactor: twisted reactor to use for connections to the proxy server
|
||||
reactor might have some blacklisting applied (i.e. for DNS queries),
|
||||
reactor might have some blocking applied (i.e. for DNS queries),
|
||||
but we need unblocked access to the proxy.
|
||||
|
||||
_srv_resolver:
|
||||
|
|
@ -87,17 +87,17 @@ class MatrixFederationAgent:
|
|||
reactor: ISynapseReactor,
|
||||
tls_client_options_factory: Optional[FederationPolicyForHTTPS],
|
||||
user_agent: bytes,
|
||||
ip_whitelist: Optional[IPSet],
|
||||
ip_blacklist: IPSet,
|
||||
ip_allowlist: Optional[IPSet],
|
||||
ip_blocklist: IPSet,
|
||||
_srv_resolver: Optional[SrvResolver] = None,
|
||||
_well_known_resolver: Optional[WellKnownResolver] = None,
|
||||
):
|
||||
# proxy_reactor is not blacklisted
|
||||
# proxy_reactor is not blocklisting reactor
|
||||
proxy_reactor = reactor
|
||||
|
||||
# We need to use a DNS resolver which filters out blacklisted IP
|
||||
# We need to use a DNS resolver which filters out blocked IP
|
||||
# addresses, to prevent DNS rebinding.
|
||||
reactor = BlacklistingReactorWrapper(reactor, ip_whitelist, ip_blacklist)
|
||||
reactor = BlocklistingReactorWrapper(reactor, ip_allowlist, ip_blocklist)
|
||||
|
||||
self._clock = Clock(reactor)
|
||||
self._pool = HTTPConnectionPool(reactor)
|
||||
|
|
@ -120,7 +120,7 @@ class MatrixFederationAgent:
|
|||
if _well_known_resolver is None:
|
||||
_well_known_resolver = WellKnownResolver(
|
||||
reactor,
|
||||
agent=BlacklistingAgentWrapper(
|
||||
agent=BlocklistingAgentWrapper(
|
||||
ProxyAgent(
|
||||
reactor,
|
||||
proxy_reactor,
|
||||
|
|
@ -128,7 +128,7 @@ class MatrixFederationAgent:
|
|||
contextFactory=tls_client_options_factory,
|
||||
use_proxy=True,
|
||||
),
|
||||
ip_blacklist=ip_blacklist,
|
||||
ip_blocklist=ip_blocklist,
|
||||
),
|
||||
user_agent=self.user_agent,
|
||||
)
|
||||
|
|
@ -256,7 +256,7 @@ class MatrixHostnameEndpoint:
|
|||
Args:
|
||||
reactor: twisted reactor to use for underlying requests
|
||||
proxy_reactor: twisted reactor to use for connections to the proxy server.
|
||||
'reactor' might have some blacklisting applied (i.e. for DNS queries),
|
||||
'reactor' might have some blocking applied (i.e. for DNS queries),
|
||||
but we need unblocked access to the proxy.
|
||||
tls_client_options_factory:
|
||||
factory to use for fetching client tls options, or none to disable TLS.
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ from synapse.api.errors import (
|
|||
from synapse.crypto.context_factory import FederationPolicyForHTTPS
|
||||
from synapse.http import QuieterFileBodyProducer
|
||||
from synapse.http.client import (
|
||||
BlacklistingAgentWrapper,
|
||||
BlocklistingAgentWrapper,
|
||||
BodyExceededMaxSize,
|
||||
ByteWriteable,
|
||||
_make_scheduler,
|
||||
|
|
@ -392,15 +392,15 @@ class MatrixFederationHttpClient:
|
|||
self.reactor,
|
||||
tls_client_options_factory,
|
||||
user_agent.encode("ascii"),
|
||||
hs.config.server.federation_ip_range_whitelist,
|
||||
hs.config.server.federation_ip_range_blacklist,
|
||||
hs.config.server.federation_ip_range_allowlist,
|
||||
hs.config.server.federation_ip_range_blocklist,
|
||||
)
|
||||
|
||||
# Use a BlacklistingAgentWrapper to prevent circumventing the IP
|
||||
# blacklist via IP literals in server names
|
||||
self.agent = BlacklistingAgentWrapper(
|
||||
# Use a BlocklistingAgentWrapper to prevent circumventing the IP
|
||||
# blocking via IP literals in server names
|
||||
self.agent = BlocklistingAgentWrapper(
|
||||
federation_agent,
|
||||
ip_blacklist=hs.config.server.federation_ip_range_blacklist,
|
||||
ip_blocklist=hs.config.server.federation_ip_range_blocklist,
|
||||
)
|
||||
|
||||
self.clock = hs.get_clock()
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class ProxyAgent(_AgentBase):
|
|||
connections.
|
||||
|
||||
proxy_reactor: twisted reactor to use for connections to the proxy server
|
||||
reactor might have some blacklisting applied (i.e. for DNS queries),
|
||||
reactor might have some blocking applied (i.e. for DNS queries),
|
||||
but we need unblocked access to the proxy.
|
||||
|
||||
contextFactory: A factory for TLS contexts, to control the
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
from zope.interface import implementer
|
||||
|
||||
|
|
@ -32,6 +32,7 @@ from twisted.web.iweb import (
|
|||
IResponse,
|
||||
)
|
||||
|
||||
from synapse.config.workers import InstanceLocationConfig
|
||||
from synapse.types import ISynapseReactor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -44,9 +45,11 @@ class ReplicationEndpointFactory:
|
|||
def __init__(
|
||||
self,
|
||||
reactor: ISynapseReactor,
|
||||
instance_map: Dict[str, InstanceLocationConfig],
|
||||
context_factory: IPolicyForHTTPS,
|
||||
) -> None:
|
||||
self.reactor = reactor
|
||||
self.instance_map = instance_map
|
||||
self.context_factory = context_factory
|
||||
|
||||
def endpointForURI(self, uri: URI) -> IStreamClientEndpoint:
|
||||
|
|
@ -58,15 +61,29 @@ class ReplicationEndpointFactory:
|
|||
|
||||
Returns: The correct client endpoint object
|
||||
"""
|
||||
if uri.scheme in (b"http", b"https"):
|
||||
endpoint = HostnameEndpoint(self.reactor, uri.host, uri.port)
|
||||
if uri.scheme == b"https":
|
||||
# The given URI has a special scheme and includes the worker name. The
|
||||
# actual connection details are pulled from the instance map.
|
||||
worker_name = uri.netloc.decode("utf-8")
|
||||
scheme = self.instance_map[worker_name].scheme()
|
||||
|
||||
if scheme in ("http", "https"):
|
||||
endpoint = HostnameEndpoint(
|
||||
self.reactor,
|
||||
self.instance_map[worker_name].host,
|
||||
self.instance_map[worker_name].port,
|
||||
)
|
||||
if scheme == "https":
|
||||
endpoint = wrapClientTLS(
|
||||
self.context_factory.creatorForNetloc(uri.host, uri.port), endpoint
|
||||
# The 'port' argument below isn't actually used by the function
|
||||
self.context_factory.creatorForNetloc(
|
||||
self.instance_map[worker_name].host,
|
||||
self.instance_map[worker_name].port,
|
||||
),
|
||||
endpoint,
|
||||
)
|
||||
return endpoint
|
||||
else:
|
||||
raise SchemeNotSupported(f"Unsupported scheme: {uri.scheme!r}")
|
||||
raise SchemeNotSupported(f"Unsupported scheme: {scheme}")
|
||||
|
||||
|
||||
@implementer(IAgent)
|
||||
|
|
@ -80,6 +97,7 @@ class ReplicationAgent(_AgentBase):
|
|||
def __init__(
|
||||
self,
|
||||
reactor: ISynapseReactor,
|
||||
instance_map: Dict[str, InstanceLocationConfig],
|
||||
contextFactory: IPolicyForHTTPS,
|
||||
connectTimeout: Optional[float] = None,
|
||||
bindAddress: Optional[bytes] = None,
|
||||
|
|
@ -102,7 +120,9 @@ class ReplicationAgent(_AgentBase):
|
|||
created.
|
||||
"""
|
||||
_AgentBase.__init__(self, reactor, pool)
|
||||
endpoint_factory = ReplicationEndpointFactory(reactor, contextFactory)
|
||||
endpoint_factory = ReplicationEndpointFactory(
|
||||
reactor, instance_map, contextFactory
|
||||
)
|
||||
self._endpointFactory = endpoint_factory
|
||||
|
||||
def request(
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ class UrlPreviewer:
|
|||
|
||||
When Synapse is asked to preview a URL it does the following:
|
||||
|
||||
1. Checks against a URL blacklist (defined as `url_preview_url_blacklist` in the
|
||||
1. Checks against a URL blocklist (defined as `url_preview_url_blacklist` in the
|
||||
config).
|
||||
2. Checks the URL against an in-memory cache and returns the result if it exists. (This
|
||||
is also used to de-duplicate processing of multiple in-flight requests at once.)
|
||||
|
|
@ -113,7 +113,7 @@ class UrlPreviewer:
|
|||
1. Checks URL and timestamp against the database cache and returns the result if it
|
||||
has not expired and was successful (a 2xx return code).
|
||||
2. Checks if the URL matches an oEmbed (https://oembed.com/) pattern. If it
|
||||
does, update the URL to download.
|
||||
does and the new URL is not blocked, update the URL to download.
|
||||
3. Downloads the URL and stores it into a file via the media storage provider
|
||||
and saves the local media metadata.
|
||||
4. If the media is an image:
|
||||
|
|
@ -127,14 +127,14 @@ class UrlPreviewer:
|
|||
and saves the local media metadata.
|
||||
2. Convert the oEmbed response to an Open Graph response.
|
||||
3. Override any Open Graph data from the HTML with data from oEmbed.
|
||||
4. If an image exists in the Open Graph response:
|
||||
4. If an image URL exists in the Open Graph response:
|
||||
1. Downloads the URL and stores it into a file via the media storage
|
||||
provider and saves the local media metadata.
|
||||
2. Generates thumbnails.
|
||||
3. Updates the Open Graph response based on image properties.
|
||||
6. If the media is JSON and an oEmbed URL was found:
|
||||
6. If an oEmbed URL was found and the media is JSON:
|
||||
1. Convert the oEmbed response to an Open Graph response.
|
||||
2. If a thumbnail or image is in the oEmbed response:
|
||||
2. If an image URL is in the oEmbed response:
|
||||
1. Downloads the URL and stores it into a file via the media storage
|
||||
provider and saves the local media metadata.
|
||||
2. Generates thumbnails.
|
||||
|
|
@ -144,7 +144,8 @@ class UrlPreviewer:
|
|||
|
||||
If any additional requests (e.g. from oEmbed autodiscovery, step 5.3 or
|
||||
image thumbnailing, step 5.4 or 6.4) fails then the URL preview as a whole
|
||||
does not fail. As much information as possible is returned.
|
||||
does not fail. If any of them are blocked, then those additional requests
|
||||
are skipped. As much information as possible is returned.
|
||||
|
||||
The in-memory cache expires after 1 hour.
|
||||
|
||||
|
|
@ -166,8 +167,8 @@ class UrlPreviewer:
|
|||
self.client = SimpleHttpClient(
|
||||
hs,
|
||||
treq_args={"browser_like_redirects": True},
|
||||
ip_whitelist=hs.config.media.url_preview_ip_range_whitelist,
|
||||
ip_blacklist=hs.config.media.url_preview_ip_range_blacklist,
|
||||
ip_allowlist=hs.config.media.url_preview_ip_range_allowlist,
|
||||
ip_blocklist=hs.config.media.url_preview_ip_range_blocklist,
|
||||
use_proxy=True,
|
||||
)
|
||||
self.media_repo = media_repo
|
||||
|
|
@ -185,7 +186,7 @@ class UrlPreviewer:
|
|||
or instance_running_jobs == hs.get_instance_name()
|
||||
)
|
||||
|
||||
self.url_preview_url_blacklist = hs.config.media.url_preview_url_blacklist
|
||||
self.url_preview_url_blocklist = hs.config.media.url_preview_url_blocklist
|
||||
self.url_preview_accept_language = hs.config.media.url_preview_accept_language
|
||||
|
||||
# memory cache mapping urls to an ObservableDeferred returning
|
||||
|
|
@ -203,48 +204,14 @@ class UrlPreviewer:
|
|||
)
|
||||
|
||||
async def preview(self, url: str, user: UserID, ts: int) -> bytes:
|
||||
# XXX: we could move this into _do_preview if we wanted.
|
||||
url_tuple = urlsplit(url)
|
||||
for entry in self.url_preview_url_blacklist:
|
||||
match = True
|
||||
for attrib in entry:
|
||||
pattern = entry[attrib]
|
||||
value = getattr(url_tuple, attrib)
|
||||
logger.debug(
|
||||
"Matching attrib '%s' with value '%s' against pattern '%s'",
|
||||
attrib,
|
||||
value,
|
||||
pattern,
|
||||
)
|
||||
|
||||
if value is None:
|
||||
match = False
|
||||
continue
|
||||
|
||||
# Some attributes might not be parsed as strings by urlsplit (such as the
|
||||
# port, which is parsed as an int). Because we use match functions that
|
||||
# expect strings, we want to make sure that's what we give them.
|
||||
value_str = str(value)
|
||||
|
||||
if pattern.startswith("^"):
|
||||
if not re.match(pattern, value_str):
|
||||
match = False
|
||||
continue
|
||||
else:
|
||||
if not fnmatch.fnmatch(value_str, pattern):
|
||||
match = False
|
||||
continue
|
||||
if match:
|
||||
logger.warning("URL %s blocked by url_blacklist entry %s", url, entry)
|
||||
raise SynapseError(
|
||||
403, "URL blocked by url pattern blacklist entry", Codes.UNKNOWN
|
||||
)
|
||||
|
||||
# the in-memory cache:
|
||||
# * ensures that only one request is active at a time
|
||||
# * ensures that only one request to a URL is active at a time
|
||||
# * takes load off the DB for the thundering herds
|
||||
# * also caches any failures (unlike the DB) so we don't keep
|
||||
# requesting the same endpoint
|
||||
# requesting the same endpoint
|
||||
#
|
||||
# Note that autodiscovered oEmbed URLs and pre-caching of images
|
||||
# are not captured in the in-memory cache.
|
||||
|
||||
observable = self._cache.get(url)
|
||||
|
||||
|
|
@ -283,7 +250,7 @@ class UrlPreviewer:
|
|||
og = og.encode("utf8")
|
||||
return og
|
||||
|
||||
# If this URL can be accessed via oEmbed, use that instead.
|
||||
# If this URL can be accessed via an allowed oEmbed, use that instead.
|
||||
url_to_download = url
|
||||
oembed_url = self._oembed.get_oembed_url(url)
|
||||
if oembed_url:
|
||||
|
|
@ -329,6 +296,7 @@ class UrlPreviewer:
|
|||
# defer to that.
|
||||
oembed_url = self._oembed.autodiscover_from_html(tree)
|
||||
og_from_oembed: JsonDict = {}
|
||||
# Only download to the oEmbed URL if it is allowed.
|
||||
if oembed_url:
|
||||
try:
|
||||
oembed_info = await self._handle_url(
|
||||
|
|
@ -411,6 +379,59 @@ class UrlPreviewer:
|
|||
|
||||
return jsonog.encode("utf8")
|
||||
|
||||
def _is_url_blocked(self, url: str) -> bool:
|
||||
"""
|
||||
Check whether the URL is allowed to be previewed (according to the homeserver
|
||||
configuration).
|
||||
|
||||
Args:
|
||||
url: The requested URL.
|
||||
|
||||
Return:
|
||||
True if the URL is blocked, False if it is allowed.
|
||||
"""
|
||||
url_tuple = urlsplit(url)
|
||||
for entry in self.url_preview_url_blocklist:
|
||||
match = True
|
||||
# Iterate over each entry. If *all* attributes of that entry match
|
||||
# the current URL, then reject it.
|
||||
for attrib, pattern in entry.items():
|
||||
value = getattr(url_tuple, attrib)
|
||||
logger.debug(
|
||||
"Matching attrib '%s' with value '%s' against pattern '%s'",
|
||||
attrib,
|
||||
value,
|
||||
pattern,
|
||||
)
|
||||
|
||||
if value is None:
|
||||
match = False
|
||||
break
|
||||
|
||||
# Some attributes might not be parsed as strings by urlsplit (such as the
|
||||
# port, which is parsed as an int). Because we use match functions that
|
||||
# expect strings, we want to make sure that's what we give them.
|
||||
value_str = str(value)
|
||||
|
||||
# Check the value against the pattern as either a regular expression or
|
||||
# a glob. If it doesn't match, the entry doesn't match.
|
||||
if pattern.startswith("^"):
|
||||
if not re.match(pattern, value_str):
|
||||
match = False
|
||||
break
|
||||
else:
|
||||
if not fnmatch.fnmatch(value_str, pattern):
|
||||
match = False
|
||||
break
|
||||
|
||||
# All fields matched, return true (the URL is blocked).
|
||||
if match:
|
||||
logger.warning("URL %s blocked by entry %s", url, entry)
|
||||
return match
|
||||
|
||||
# No matches were found, the URL is allowed.
|
||||
return False
|
||||
|
||||
async def _download_url(self, url: str, output_stream: BinaryIO) -> DownloadResult:
|
||||
"""
|
||||
Fetches a remote URL and parses the headers.
|
||||
|
|
@ -451,7 +472,7 @@ class UrlPreviewer:
|
|||
except DNSLookupError:
|
||||
# DNS lookup returned no results
|
||||
# Note: This will also be the case if one of the resolved IP
|
||||
# addresses is blacklisted
|
||||
# addresses is blocked.
|
||||
raise SynapseError(
|
||||
502,
|
||||
"DNS resolution failure during URL preview generation",
|
||||
|
|
@ -547,8 +568,16 @@ class UrlPreviewer:
|
|||
|
||||
Returns:
|
||||
A MediaInfo object describing the fetched content.
|
||||
|
||||
Raises:
|
||||
SynapseError if the URL is blocked.
|
||||
"""
|
||||
|
||||
if self._is_url_blocked(url):
|
||||
raise SynapseError(
|
||||
403, "URL blocked by url pattern blocklist entry", Codes.UNKNOWN
|
||||
)
|
||||
|
||||
# TODO: we should probably honour robots.txt... except in practice
|
||||
# we're most likely being explicitly triggered by a human rather than a
|
||||
# bot, so are we really a robot?
|
||||
|
|
@ -624,7 +653,7 @@ class UrlPreviewer:
|
|||
return
|
||||
|
||||
# The image URL from the HTML might be relative to the previewed page,
|
||||
# convert it to an URL which can be requested directly.
|
||||
# convert it to a URL which can be requested directly.
|
||||
url_parts = urlparse(image_url)
|
||||
if url_parts.scheme != "data":
|
||||
image_url = urljoin(media_info.uri, image_url)
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ from synapse.util.caches.descriptors import CachedFunction, cached as _cached
|
|||
from synapse.util.frozenutils import freeze
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from synapse.app.generic_worker import GenericWorkerSlavedStore
|
||||
from synapse.app.generic_worker import GenericWorkerStore
|
||||
from synapse.server import HomeServer
|
||||
|
||||
|
||||
|
|
@ -237,9 +237,7 @@ class ModuleApi:
|
|||
|
||||
# TODO: Fix this type hint once the types for the data stores have been ironed
|
||||
# out.
|
||||
self._store: Union[
|
||||
DataStore, "GenericWorkerSlavedStore"
|
||||
] = hs.get_datastores().main
|
||||
self._store: Union[DataStore, "GenericWorkerStore"] = hs.get_datastores().main
|
||||
self._storage_controllers = hs.get_storage_controllers()
|
||||
self._auth = hs.get_auth()
|
||||
self._auth_handler = auth_handler
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ class HttpPusher(Pusher):
|
|||
raise PusherConfigException("'url' must be a string")
|
||||
|
||||
self.url = url
|
||||
self.http_client = hs.get_proxied_blacklisted_http_client()
|
||||
self.http_client = hs.get_proxied_blocklisted_http_client()
|
||||
self.data_minus_url = {}
|
||||
self.data_minus_url.update(self.data)
|
||||
del self.data_minus_url["url"]
|
||||
|
|
|
|||
|
|
@ -219,11 +219,7 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta):
|
|||
with outgoing_gauge.track_inprogress():
|
||||
if instance_name == local_instance_name:
|
||||
raise Exception("Trying to send HTTP request to self")
|
||||
if instance_name in instance_map:
|
||||
host = instance_map[instance_name].host
|
||||
port = instance_map[instance_name].port
|
||||
tls = instance_map[instance_name].tls
|
||||
else:
|
||||
if instance_name not in instance_map:
|
||||
raise Exception(
|
||||
"Instance %r not in 'instance_map' config" % (instance_name,)
|
||||
)
|
||||
|
|
@ -271,13 +267,11 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta):
|
|||
"Unknown METHOD on %s replication endpoint" % (cls.NAME,)
|
||||
)
|
||||
|
||||
# Here the protocol is hard coded to be http by default or https in case the replication
|
||||
# port is set to have tls true.
|
||||
scheme = "https" if tls else "http"
|
||||
uri = "%s://%s:%s/_synapse/replication/%s/%s" % (
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
# Hard code a special scheme to show this only used for replication. The
|
||||
# instance_name will be passed into the ReplicationEndpointFactory to
|
||||
# determine connection details from the instance_map.
|
||||
uri = "synapse-replication://%s/_synapse/replication/%s/%s" % (
|
||||
instance_name,
|
||||
cls.NAME,
|
||||
"/".join(url_args),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS = 5
|
|||
class ReplicationDataHandler:
|
||||
"""Handles incoming stream updates from replication.
|
||||
|
||||
This instance notifies the slave data store about updates. Can be subclassed
|
||||
This instance notifies the data store about updates. Can be subclassed
|
||||
to handle updates in additional ways.
|
||||
"""
|
||||
|
||||
|
|
@ -91,7 +91,7 @@ class ReplicationDataHandler:
|
|||
) -> None:
|
||||
"""Called to handle a batch of replication data with a given stream token.
|
||||
|
||||
By default this just pokes the slave store. Can be overridden in subclasses to
|
||||
By default, this just pokes the data store. Can be overridden in subclasses to
|
||||
handle more.
|
||||
|
||||
Args:
|
||||
|
|
|
|||
|
|
@ -352,7 +352,15 @@ class ReplicationCommandHandler:
|
|||
|
||||
reactor = hs.get_reactor()
|
||||
redis_config = hs.config.redis
|
||||
if hs.config.redis.redis_use_tls:
|
||||
if redis_config.redis_path is not None:
|
||||
reactor.connectUNIX(
|
||||
redis_config.redis_path,
|
||||
self._factory,
|
||||
timeout=30,
|
||||
checkPID=False,
|
||||
)
|
||||
|
||||
elif hs.config.redis.redis_use_tls:
|
||||
ssl_context_factory = ClientContextFactory(hs.config.redis)
|
||||
reactor.connectSSL(
|
||||
redis_config.redis_host,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,12 @@ from inspect import isawaitable
|
|||
from typing import TYPE_CHECKING, Any, Generic, List, Optional, Type, TypeVar, cast
|
||||
|
||||
import attr
|
||||
import txredisapi
|
||||
from txredisapi import (
|
||||
ConnectionHandler,
|
||||
RedisFactory,
|
||||
SubscriberProtocol,
|
||||
UnixConnectionHandler,
|
||||
)
|
||||
from zope.interface import implementer
|
||||
|
||||
from twisted.internet.address import IPv4Address, IPv6Address
|
||||
|
|
@ -68,7 +73,7 @@ class ConstantProperty(Generic[T, V]):
|
|||
|
||||
|
||||
@implementer(IReplicationConnection)
|
||||
class RedisSubscriber(txredisapi.SubscriberProtocol):
|
||||
class RedisSubscriber(SubscriberProtocol):
|
||||
"""Connection to redis subscribed to replication stream.
|
||||
|
||||
This class fulfils two functions:
|
||||
|
|
@ -95,7 +100,7 @@ class RedisSubscriber(txredisapi.SubscriberProtocol):
|
|||
synapse_handler: "ReplicationCommandHandler"
|
||||
synapse_stream_prefix: str
|
||||
synapse_channel_names: List[str]
|
||||
synapse_outbound_redis_connection: txredisapi.ConnectionHandler
|
||||
synapse_outbound_redis_connection: ConnectionHandler
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -229,7 +234,7 @@ class RedisSubscriber(txredisapi.SubscriberProtocol):
|
|||
)
|
||||
|
||||
|
||||
class SynapseRedisFactory(txredisapi.RedisFactory):
|
||||
class SynapseRedisFactory(RedisFactory):
|
||||
"""A subclass of RedisFactory that periodically sends pings to ensure that
|
||||
we detect dead connections.
|
||||
"""
|
||||
|
|
@ -245,7 +250,7 @@ class SynapseRedisFactory(txredisapi.RedisFactory):
|
|||
dbid: Optional[int],
|
||||
poolsize: int,
|
||||
isLazy: bool = False,
|
||||
handler: Type = txredisapi.ConnectionHandler,
|
||||
handler: Type = ConnectionHandler,
|
||||
charset: str = "utf-8",
|
||||
password: Optional[str] = None,
|
||||
replyTimeout: int = 30,
|
||||
|
|
@ -326,7 +331,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory):
|
|||
def __init__(
|
||||
self,
|
||||
hs: "HomeServer",
|
||||
outbound_redis_connection: txredisapi.ConnectionHandler,
|
||||
outbound_redis_connection: ConnectionHandler,
|
||||
channel_names: List[str],
|
||||
):
|
||||
super().__init__(
|
||||
|
|
@ -368,7 +373,7 @@ def lazyConnection(
|
|||
reconnect: bool = True,
|
||||
password: Optional[str] = None,
|
||||
replyTimeout: int = 30,
|
||||
) -> txredisapi.ConnectionHandler:
|
||||
) -> ConnectionHandler:
|
||||
"""Creates a connection to Redis that is lazily set up and reconnects if the
|
||||
connections is lost.
|
||||
"""
|
||||
|
|
@ -380,7 +385,7 @@ def lazyConnection(
|
|||
dbid=dbid,
|
||||
poolsize=1,
|
||||
isLazy=True,
|
||||
handler=txredisapi.ConnectionHandler,
|
||||
handler=ConnectionHandler,
|
||||
password=password,
|
||||
replyTimeout=replyTimeout,
|
||||
)
|
||||
|
|
@ -408,3 +413,44 @@ def lazyConnection(
|
|||
)
|
||||
|
||||
return factory.handler
|
||||
|
||||
|
||||
def lazyUnixConnection(
|
||||
hs: "HomeServer",
|
||||
path: str = "/tmp/redis.sock",
|
||||
dbid: Optional[int] = None,
|
||||
reconnect: bool = True,
|
||||
password: Optional[str] = None,
|
||||
replyTimeout: int = 30,
|
||||
) -> ConnectionHandler:
|
||||
"""Creates a connection to Redis that is lazily set up and reconnects if the
|
||||
connection is lost.
|
||||
|
||||
Returns:
|
||||
A subclass of ConnectionHandler, which is a UnixConnectionHandler in this case.
|
||||
"""
|
||||
|
||||
uuid = path
|
||||
|
||||
factory = SynapseRedisFactory(
|
||||
hs,
|
||||
uuid=uuid,
|
||||
dbid=dbid,
|
||||
poolsize=1,
|
||||
isLazy=True,
|
||||
handler=UnixConnectionHandler,
|
||||
password=password,
|
||||
replyTimeout=replyTimeout,
|
||||
)
|
||||
factory.continueTrying = reconnect
|
||||
|
||||
reactor = hs.get_reactor()
|
||||
|
||||
reactor.connectUNIX(
|
||||
path,
|
||||
factory,
|
||||
timeout=30,
|
||||
checkPID=False,
|
||||
)
|
||||
|
||||
return factory.handler
|
||||
|
|
|
|||
|
|
@ -137,6 +137,35 @@ class DevicesRestServlet(RestServlet):
|
|||
devices = await self.device_handler.get_devices_by_user(target_user.to_string())
|
||||
return HTTPStatus.OK, {"devices": devices, "total": len(devices)}
|
||||
|
||||
async def on_POST(
|
||||
self, request: SynapseRequest, user_id: str
|
||||
) -> Tuple[int, JsonDict]:
|
||||
"""Creates a new device for the user."""
|
||||
await assert_requester_is_admin(self.auth, request)
|
||||
|
||||
target_user = UserID.from_string(user_id)
|
||||
if not self.is_mine(target_user):
|
||||
raise SynapseError(
|
||||
HTTPStatus.BAD_REQUEST, "Can only create devices for local users"
|
||||
)
|
||||
|
||||
u = await self.store.get_user_by_id(target_user.to_string())
|
||||
if u is None:
|
||||
raise NotFoundError("Unknown user")
|
||||
|
||||
body = parse_json_object_from_request(request)
|
||||
device_id = body.get("device_id")
|
||||
if not device_id:
|
||||
raise SynapseError(HTTPStatus.BAD_REQUEST, "Missing device_id")
|
||||
if not isinstance(device_id, str):
|
||||
raise SynapseError(HTTPStatus.BAD_REQUEST, "device_id must be a string")
|
||||
|
||||
await self.device_handler.check_device_registered(
|
||||
user_id=user_id, device_id=device_id
|
||||
)
|
||||
|
||||
return HTTPStatus.CREATED, {}
|
||||
|
||||
|
||||
class DeleteDevicesRestServlet(RestServlet):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -287,7 +287,7 @@ class OneTimeKeyServlet(RestServlet):
|
|||
self.e2e_keys_handler = hs.get_e2e_keys_handler()
|
||||
|
||||
async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
|
||||
await self.auth.get_user_by_req(request, allow_guest=True)
|
||||
requester = await self.auth.get_user_by_req(request, allow_guest=True)
|
||||
timeout = parse_integer(request, "timeout", 10 * 1000)
|
||||
body = parse_json_object_from_request(request)
|
||||
|
||||
|
|
@ -298,7 +298,7 @@ class OneTimeKeyServlet(RestServlet):
|
|||
query.setdefault(user_id, {})[device_id] = {algorithm: 1}
|
||||
|
||||
result = await self.e2e_keys_handler.claim_one_time_keys(
|
||||
query, timeout, always_include_fallback_keys=False
|
||||
query, requester.user, timeout, always_include_fallback_keys=False
|
||||
)
|
||||
return 200, result
|
||||
|
||||
|
|
@ -335,7 +335,7 @@ class UnstableOneTimeKeyServlet(RestServlet):
|
|||
self.e2e_keys_handler = hs.get_e2e_keys_handler()
|
||||
|
||||
async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
|
||||
await self.auth.get_user_by_req(request, allow_guest=True)
|
||||
requester = await self.auth.get_user_by_req(request, allow_guest=True)
|
||||
timeout = parse_integer(request, "timeout", 10 * 1000)
|
||||
body = parse_json_object_from_request(request)
|
||||
|
||||
|
|
@ -346,7 +346,7 @@ class UnstableOneTimeKeyServlet(RestServlet):
|
|||
query.setdefault(user_id, {})[device_id] = Counter(algorithms)
|
||||
|
||||
result = await self.e2e_keys_handler.claim_one_time_keys(
|
||||
query, timeout, always_include_fallback_keys=True
|
||||
query, requester.user, timeout, always_include_fallback_keys=True
|
||||
)
|
||||
return 200, result
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from synapse.api.errors import (
|
|||
LoginError,
|
||||
NotApprovedError,
|
||||
SynapseError,
|
||||
UserDeactivatedError,
|
||||
)
|
||||
from synapse.api.ratelimiting import Ratelimiter
|
||||
from synapse.api.urls import CLIENT_API_PREFIX
|
||||
|
|
@ -84,14 +85,10 @@ class LoginRestServlet(RestServlet):
|
|||
def __init__(self, hs: "HomeServer"):
|
||||
super().__init__()
|
||||
self.hs = hs
|
||||
self._main_store = hs.get_datastores().main
|
||||
|
||||
# JWT configuration variables.
|
||||
self.jwt_enabled = hs.config.jwt.jwt_enabled
|
||||
self.jwt_secret = hs.config.jwt.jwt_secret
|
||||
self.jwt_subject_claim = hs.config.jwt.jwt_subject_claim
|
||||
self.jwt_algorithm = hs.config.jwt.jwt_algorithm
|
||||
self.jwt_issuer = hs.config.jwt.jwt_issuer
|
||||
self.jwt_audiences = hs.config.jwt.jwt_audiences
|
||||
|
||||
# SSO configuration.
|
||||
self.saml2_enabled = hs.config.saml2.saml2_enabled
|
||||
|
|
@ -117,13 +114,13 @@ class LoginRestServlet(RestServlet):
|
|||
|
||||
self._well_known_builder = WellKnownBuilder(hs)
|
||||
self._address_ratelimiter = Ratelimiter(
|
||||
store=hs.get_datastores().main,
|
||||
store=self._main_store,
|
||||
clock=hs.get_clock(),
|
||||
rate_hz=self.hs.config.ratelimiting.rc_login_address.per_second,
|
||||
burst_count=self.hs.config.ratelimiting.rc_login_address.burst_count,
|
||||
)
|
||||
self._account_ratelimiter = Ratelimiter(
|
||||
store=hs.get_datastores().main,
|
||||
store=self._main_store,
|
||||
clock=hs.get_clock(),
|
||||
rate_hz=self.hs.config.ratelimiting.rc_login_account.per_second,
|
||||
burst_count=self.hs.config.ratelimiting.rc_login_account.burst_count,
|
||||
|
|
@ -285,6 +282,9 @@ class LoginRestServlet(RestServlet):
|
|||
login_submission,
|
||||
ratelimit=appservice.is_rate_limited(),
|
||||
should_issue_refresh_token=should_issue_refresh_token,
|
||||
# The user represented by an appservice's configured sender_localpart
|
||||
# is not actually created in Synapse.
|
||||
should_check_deactivated=qualified_user_id != appservice.sender,
|
||||
)
|
||||
|
||||
async def _do_other_login(
|
||||
|
|
@ -331,6 +331,7 @@ class LoginRestServlet(RestServlet):
|
|||
auth_provider_id: Optional[str] = None,
|
||||
should_issue_refresh_token: bool = False,
|
||||
auth_provider_session_id: Optional[str] = None,
|
||||
should_check_deactivated: bool = True,
|
||||
) -> LoginResponse:
|
||||
"""Called when we've successfully authed the user and now need to
|
||||
actually login them in (e.g. create devices). This gets called on
|
||||
|
|
@ -350,6 +351,11 @@ class LoginRestServlet(RestServlet):
|
|||
should_issue_refresh_token: True if this login should issue
|
||||
a refresh token alongside the access token.
|
||||
auth_provider_session_id: The session ID got during login from the SSO IdP.
|
||||
should_check_deactivated: True if the user should be checked for
|
||||
deactivation status before logging in.
|
||||
|
||||
This exists purely for appservice's configured sender_localpart
|
||||
which doesn't have an associated user in the database.
|
||||
|
||||
Returns:
|
||||
Dictionary of account information after successful login.
|
||||
|
|
@ -369,6 +375,12 @@ class LoginRestServlet(RestServlet):
|
|||
)
|
||||
user_id = canonical_uid
|
||||
|
||||
# If the account has been deactivated, do not proceed with the login.
|
||||
if should_check_deactivated:
|
||||
deactivated = await self._main_store.get_user_deactivated_status(user_id)
|
||||
if deactivated:
|
||||
raise UserDeactivatedError("This account has been deactivated")
|
||||
|
||||
device_id = login_submission.get("device_id")
|
||||
|
||||
# If device_id is present, check that device_id is not longer than a reasonable 512 characters
|
||||
|
|
@ -427,7 +439,7 @@ class LoginRestServlet(RestServlet):
|
|||
self, login_submission: JsonDict, should_issue_refresh_token: bool = False
|
||||
) -> LoginResponse:
|
||||
"""
|
||||
Handle the final stage of SSO login.
|
||||
Handle token login.
|
||||
|
||||
Args:
|
||||
login_submission: The JSON request body.
|
||||
|
|
@ -452,72 +464,24 @@ class LoginRestServlet(RestServlet):
|
|||
async def _do_jwt_login(
|
||||
self, login_submission: JsonDict, should_issue_refresh_token: bool = False
|
||||
) -> LoginResponse:
|
||||
token = login_submission.get("token", None)
|
||||
if token is None:
|
||||
raise LoginError(
|
||||
403, "Token field for JWT is missing", errcode=Codes.FORBIDDEN
|
||||
)
|
||||
"""
|
||||
Handle the custom JWT login.
|
||||
|
||||
from authlib.jose import JsonWebToken, JWTClaims
|
||||
from authlib.jose.errors import BadSignatureError, InvalidClaimError, JoseError
|
||||
Args:
|
||||
login_submission: The JSON request body.
|
||||
should_issue_refresh_token: True if this login should issue
|
||||
a refresh token alongside the access token.
|
||||
|
||||
jwt = JsonWebToken([self.jwt_algorithm])
|
||||
claim_options = {}
|
||||
if self.jwt_issuer is not None:
|
||||
claim_options["iss"] = {"value": self.jwt_issuer, "essential": True}
|
||||
if self.jwt_audiences is not None:
|
||||
claim_options["aud"] = {"values": self.jwt_audiences, "essential": True}
|
||||
|
||||
try:
|
||||
claims = jwt.decode(
|
||||
token,
|
||||
key=self.jwt_secret,
|
||||
claims_cls=JWTClaims,
|
||||
claims_options=claim_options,
|
||||
)
|
||||
except BadSignatureError:
|
||||
# We handle this case separately to provide a better error message
|
||||
raise LoginError(
|
||||
403,
|
||||
"JWT validation failed: Signature verification failed",
|
||||
errcode=Codes.FORBIDDEN,
|
||||
)
|
||||
except JoseError as e:
|
||||
# A JWT error occurred, return some info back to the client.
|
||||
raise LoginError(
|
||||
403,
|
||||
"JWT validation failed: %s" % (str(e),),
|
||||
errcode=Codes.FORBIDDEN,
|
||||
)
|
||||
|
||||
try:
|
||||
claims.validate(leeway=120) # allows 2 min of clock skew
|
||||
|
||||
# Enforce the old behavior which is rolled out in productive
|
||||
# servers: if the JWT contains an 'aud' claim but none is
|
||||
# configured, the login attempt will fail
|
||||
if claims.get("aud") is not None:
|
||||
if self.jwt_audiences is None or len(self.jwt_audiences) == 0:
|
||||
raise InvalidClaimError("aud")
|
||||
except JoseError as e:
|
||||
raise LoginError(
|
||||
403,
|
||||
"JWT validation failed: %s" % (str(e),),
|
||||
errcode=Codes.FORBIDDEN,
|
||||
)
|
||||
|
||||
user = claims.get(self.jwt_subject_claim, None)
|
||||
if user is None:
|
||||
raise LoginError(403, "Invalid JWT", errcode=Codes.FORBIDDEN)
|
||||
|
||||
user_id = UserID(user, self.hs.hostname).to_string()
|
||||
result = await self._complete_login(
|
||||
Returns:
|
||||
The body of the JSON response.
|
||||
"""
|
||||
user_id = self.hs.get_jwt_handler().validate_login(login_submission)
|
||||
return await self._complete_login(
|
||||
user_id,
|
||||
login_submission,
|
||||
create_non_existent_users=True,
|
||||
should_issue_refresh_token=should_issue_refresh_token,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _get_auth_flow_dict_for_idp(idp: SsoIdentityProvider) -> JsonDict:
|
||||
|
|
|
|||
|
|
@ -12,13 +12,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Tuple
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Dict, List, Tuple
|
||||
|
||||
from synapse.api.errors import Codes, SynapseError
|
||||
from synapse.http.server import HttpServer
|
||||
from synapse.http.servlet import RestServlet
|
||||
from synapse.http.servlet import RestServlet, parse_strings_from_args
|
||||
from synapse.http.site import SynapseRequest
|
||||
from synapse.types import JsonDict, UserID
|
||||
from synapse.types import JsonDict
|
||||
|
||||
from ._base import client_patterns
|
||||
|
||||
|
|
@ -30,11 +31,11 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class UserMutualRoomsServlet(RestServlet):
|
||||
"""
|
||||
GET /uk.half-shot.msc2666/user/mutual_rooms/{user_id} HTTP/1.1
|
||||
GET /uk.half-shot.msc2666/user/mutual_rooms?user_id={user_id} HTTP/1.1
|
||||
"""
|
||||
|
||||
PATTERNS = client_patterns(
|
||||
"/uk.half-shot.msc2666/user/mutual_rooms/(?P<user_id>[^/]*)",
|
||||
"/uk.half-shot.msc2666/user/mutual_rooms$",
|
||||
releases=(), # This is an unstable feature
|
||||
)
|
||||
|
||||
|
|
@ -43,17 +44,35 @@ class UserMutualRoomsServlet(RestServlet):
|
|||
self.auth = hs.get_auth()
|
||||
self.store = hs.get_datastores().main
|
||||
|
||||
async def on_GET(
|
||||
self, request: SynapseRequest, user_id: str
|
||||
) -> Tuple[int, JsonDict]:
|
||||
UserID.from_string(user_id)
|
||||
async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
|
||||
# twisted.web.server.Request.args is incorrectly defined as Optional[Any]
|
||||
args: Dict[bytes, List[bytes]] = request.args # type: ignore
|
||||
|
||||
user_ids = parse_strings_from_args(args, "user_id", required=True)
|
||||
|
||||
if len(user_ids) > 1:
|
||||
raise SynapseError(
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
"Duplicate user_id query parameter",
|
||||
errcode=Codes.INVALID_PARAM,
|
||||
)
|
||||
|
||||
# We don't do batching, so a batch token is illegal by default
|
||||
if b"batch_token" in args:
|
||||
raise SynapseError(
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
"Unknown batch_token",
|
||||
errcode=Codes.INVALID_PARAM,
|
||||
)
|
||||
|
||||
user_id = user_ids[0]
|
||||
|
||||
requester = await self.auth.get_user_by_req(request)
|
||||
if user_id == requester.user.to_string():
|
||||
raise SynapseError(
|
||||
code=400,
|
||||
msg="You cannot request a list of shared rooms with yourself",
|
||||
errcode=Codes.FORBIDDEN,
|
||||
HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
"You cannot request a list of shared rooms with yourself",
|
||||
errcode=Codes.INVALID_PARAM,
|
||||
)
|
||||
|
||||
rooms = await self.store.get_mutual_rooms_between_users(
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ class VersionsRestServlet(RestServlet):
|
|||
# Implements additional endpoints as described in MSC2432
|
||||
"org.matrix.msc2432": True,
|
||||
# Implements additional endpoints as described in MSC2666
|
||||
"uk.half-shot.msc2666.mutual_rooms": True,
|
||||
"uk.half-shot.msc2666.query_mutual_rooms": True,
|
||||
# Whether new rooms will be set to encrypted or not (based on presets).
|
||||
"io.element.e2ee_forced.public": self.e2ee_forced_public,
|
||||
"io.element.e2ee_forced.private": self.e2ee_forced_private,
|
||||
|
|
|
|||
|
|
@ -147,6 +147,7 @@ logger = logging.getLogger(__name__)
|
|||
if TYPE_CHECKING:
|
||||
from txredisapi import ConnectionHandler
|
||||
|
||||
from synapse.handlers.jwt import JwtHandler
|
||||
from synapse.handlers.oidc import OidcHandler
|
||||
from synapse.handlers.saml import SamlHandler
|
||||
|
||||
|
|
@ -453,15 +454,15 @@ class HomeServer(metaclass=abc.ABCMeta):
|
|||
return SimpleHttpClient(self, use_proxy=True)
|
||||
|
||||
@cache_in_self
|
||||
def get_proxied_blacklisted_http_client(self) -> SimpleHttpClient:
|
||||
def get_proxied_blocklisted_http_client(self) -> SimpleHttpClient:
|
||||
"""
|
||||
An HTTP client that uses configured HTTP(S) proxies and blacklists IPs
|
||||
based on the IP range blacklist/whitelist.
|
||||
An HTTP client that uses configured HTTP(S) proxies and blocks IPs
|
||||
based on the configured IP ranges.
|
||||
"""
|
||||
return SimpleHttpClient(
|
||||
self,
|
||||
ip_whitelist=self.config.server.ip_range_whitelist,
|
||||
ip_blacklist=self.config.server.ip_range_blacklist,
|
||||
ip_allowlist=self.config.server.ip_range_allowlist,
|
||||
ip_blocklist=self.config.server.ip_range_blocklist,
|
||||
use_proxy=True,
|
||||
)
|
||||
|
||||
|
|
@ -533,6 +534,12 @@ class HomeServer(metaclass=abc.ABCMeta):
|
|||
def get_sso_handler(self) -> SsoHandler:
|
||||
return SsoHandler(self)
|
||||
|
||||
@cache_in_self
|
||||
def get_jwt_handler(self) -> "JwtHandler":
|
||||
from synapse.handlers.jwt import JwtHandler
|
||||
|
||||
return JwtHandler(self)
|
||||
|
||||
@cache_in_self
|
||||
def get_sync_handler(self) -> SyncHandler:
|
||||
return SyncHandler(self)
|
||||
|
|
@ -857,22 +864,36 @@ class HomeServer(metaclass=abc.ABCMeta):
|
|||
|
||||
# We only want to import redis module if we're using it, as we have
|
||||
# `txredisapi` as an optional dependency.
|
||||
from synapse.replication.tcp.redis import lazyConnection
|
||||
from synapse.replication.tcp.redis import lazyConnection, lazyUnixConnection
|
||||
|
||||
logger.info(
|
||||
"Connecting to redis (host=%r port=%r) for external cache",
|
||||
self.config.redis.redis_host,
|
||||
self.config.redis.redis_port,
|
||||
)
|
||||
if self.config.redis.redis_path is None:
|
||||
logger.info(
|
||||
"Connecting to redis (host=%r port=%r) for external cache",
|
||||
self.config.redis.redis_host,
|
||||
self.config.redis.redis_port,
|
||||
)
|
||||
|
||||
return lazyConnection(
|
||||
hs=self,
|
||||
host=self.config.redis.redis_host,
|
||||
port=self.config.redis.redis_port,
|
||||
dbid=self.config.redis.redis_dbid,
|
||||
password=self.config.redis.redis_password,
|
||||
reconnect=True,
|
||||
)
|
||||
return lazyConnection(
|
||||
hs=self,
|
||||
host=self.config.redis.redis_host,
|
||||
port=self.config.redis.redis_port,
|
||||
dbid=self.config.redis.redis_dbid,
|
||||
password=self.config.redis.redis_password,
|
||||
reconnect=True,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Connecting to redis (path=%r) for external cache",
|
||||
self.config.redis.redis_path,
|
||||
)
|
||||
|
||||
return lazyUnixConnection(
|
||||
hs=self,
|
||||
path=self.config.redis.redis_path,
|
||||
dbid=self.config.redis.redis_dbid,
|
||||
password=self.config.redis.redis_password,
|
||||
reconnect=True,
|
||||
)
|
||||
|
||||
def should_send_federation(self) -> bool:
|
||||
"Should this server be sending federation traffic directly?"
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from synapse.events.snapshot import (
|
|||
UnpersistedEventContextBase,
|
||||
)
|
||||
from synapse.logging.context import ContextResourceUsage
|
||||
from synapse.logging.opentracing import tag_args, trace
|
||||
from synapse.replication.http.state import ReplicationUpdateCurrentStateRestServlet
|
||||
from synapse.state import v1, v2
|
||||
from synapse.storage.databases.main.events_worker import EventRedactBehaviour
|
||||
|
|
@ -270,6 +271,8 @@ class StateHandler:
|
|||
state = await entry.get_state(self._state_storage_controller, StateFilter.all())
|
||||
return await self.store.get_joined_hosts(room_id, state, entry)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def calculate_context_info(
|
||||
self,
|
||||
event: EventBase,
|
||||
|
|
@ -465,6 +468,7 @@ class StateHandler:
|
|||
|
||||
return await unpersisted_context.persist(event)
|
||||
|
||||
@trace
|
||||
@measure_func()
|
||||
async def resolve_state_groups_for_events(
|
||||
self, room_id: str, event_ids: Collection[str], await_full_state: bool = True
|
||||
|
|
|
|||
|
|
@ -561,6 +561,50 @@ class BackgroundUpdater:
|
|||
updater, oneshot=True
|
||||
)
|
||||
|
||||
def register_background_validate_constraint(
|
||||
self, update_name: str, constraint_name: str, table: str
|
||||
) -> None:
|
||||
"""Helper for store classes to do a background validate constraint.
|
||||
|
||||
This only applies on PostgreSQL.
|
||||
|
||||
To use:
|
||||
|
||||
1. use a schema delta file to add a background update. Example:
|
||||
INSERT INTO background_updates (update_name, progress_json) VALUES
|
||||
('validate_my_constraint', '{}');
|
||||
|
||||
2. In the Store constructor, call this method
|
||||
|
||||
Args:
|
||||
update_name: update_name to register for
|
||||
constraint_name: name of constraint to validate
|
||||
table: table the constraint is applied to
|
||||
"""
|
||||
|
||||
def runner(conn: Connection) -> None:
|
||||
c = conn.cursor()
|
||||
|
||||
sql = f"""
|
||||
ALTER TABLE {table} VALIDATE CONSTRAINT {constraint_name};
|
||||
"""
|
||||
logger.debug("[SQL] %s", sql)
|
||||
c.execute(sql)
|
||||
|
||||
async def updater(progress: JsonDict, batch_size: int) -> int:
|
||||
assert isinstance(
|
||||
self.db_pool.engine, engines.PostgresEngine
|
||||
), "validate constraint background update registered for non-Postres database"
|
||||
|
||||
logger.info("Validating constraint %s to %s", constraint_name, table)
|
||||
await self.db_pool.runWithConnection(runner)
|
||||
await self._end_background_update(update_name)
|
||||
return 1
|
||||
|
||||
self._background_update_handlers[update_name] = _BackgroundUpdateHandler(
|
||||
updater, oneshot=True
|
||||
)
|
||||
|
||||
async def create_index_in_background(
|
||||
self,
|
||||
index_name: str,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from typing import (
|
|||
TYPE_CHECKING,
|
||||
AbstractSet,
|
||||
Any,
|
||||
Awaitable,
|
||||
Callable,
|
||||
Collection,
|
||||
Dict,
|
||||
|
|
@ -67,6 +66,8 @@ class StateStorageController:
|
|||
"""
|
||||
self._partial_state_room_tracker.notify_un_partial_stated(room_id)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_group_delta(
|
||||
self, state_group: int
|
||||
) -> Tuple[Optional[int], Optional[StateMap[str]]]:
|
||||
|
|
@ -84,6 +85,8 @@ class StateStorageController:
|
|||
state_group_delta = await self.stores.state.get_state_group_delta(state_group)
|
||||
return state_group_delta.prev_group, state_group_delta.delta_ids
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_groups_ids(
|
||||
self, _room_id: str, event_ids: Collection[str], await_full_state: bool = True
|
||||
) -> Dict[int, MutableStateMap[str]]:
|
||||
|
|
@ -114,6 +117,8 @@ class StateStorageController:
|
|||
|
||||
return group_to_state
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_ids_for_group(
|
||||
self, state_group: int, state_filter: Optional[StateFilter] = None
|
||||
) -> StateMap[str]:
|
||||
|
|
@ -130,6 +135,8 @@ class StateStorageController:
|
|||
|
||||
return group_to_state[state_group]
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_groups(
|
||||
self, room_id: str, event_ids: Collection[str]
|
||||
) -> Dict[int, List[EventBase]]:
|
||||
|
|
@ -165,9 +172,11 @@ class StateStorageController:
|
|||
for group, event_id_map in group_to_ids.items()
|
||||
}
|
||||
|
||||
def _get_state_groups_from_groups(
|
||||
@trace
|
||||
@tag_args
|
||||
async def _get_state_groups_from_groups(
|
||||
self, groups: List[int], state_filter: StateFilter
|
||||
) -> Awaitable[Dict[int, StateMap[str]]]:
|
||||
) -> Dict[int, StateMap[str]]:
|
||||
"""Returns the state groups for a given set of groups, filtering on
|
||||
types of state events.
|
||||
|
||||
|
|
@ -180,9 +189,12 @@ class StateStorageController:
|
|||
Dict of state group to state map.
|
||||
"""
|
||||
|
||||
return self.stores.state._get_state_groups_from_groups(groups, state_filter)
|
||||
return await self.stores.state._get_state_groups_from_groups(
|
||||
groups, state_filter
|
||||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_for_events(
|
||||
self, event_ids: Collection[str], state_filter: Optional[StateFilter] = None
|
||||
) -> Dict[str, StateMap[EventBase]]:
|
||||
|
|
@ -280,6 +292,8 @@ class StateStorageController:
|
|||
|
||||
return {event: event_to_state[event] for event in event_ids}
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_for_event(
|
||||
self, event_id: str, state_filter: Optional[StateFilter] = None
|
||||
) -> StateMap[EventBase]:
|
||||
|
|
@ -303,6 +317,7 @@ class StateStorageController:
|
|||
return state_map[event_id]
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_ids_for_event(
|
||||
self,
|
||||
event_id: str,
|
||||
|
|
@ -333,9 +348,11 @@ class StateStorageController:
|
|||
)
|
||||
return state_map[event_id]
|
||||
|
||||
def get_state_for_groups(
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_state_for_groups(
|
||||
self, groups: Iterable[int], state_filter: Optional[StateFilter] = None
|
||||
) -> Awaitable[Dict[int, MutableStateMap[str]]]:
|
||||
) -> Dict[int, MutableStateMap[str]]:
|
||||
"""Gets the state at each of a list of state groups, optionally
|
||||
filtering by type/state_key
|
||||
|
||||
|
|
@ -347,7 +364,7 @@ class StateStorageController:
|
|||
Returns:
|
||||
Dict of state group to state map.
|
||||
"""
|
||||
return self.stores.state._get_state_for_groups(
|
||||
return await self.stores.state._get_state_for_groups(
|
||||
groups, state_filter or StateFilter.all()
|
||||
)
|
||||
|
||||
|
|
@ -402,6 +419,8 @@ class StateStorageController:
|
|||
event_id, room_id, prev_group, delta_ids, current_state_ids
|
||||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
@cancellable
|
||||
async def get_current_state_ids(
|
||||
self,
|
||||
|
|
@ -442,6 +461,8 @@ class StateStorageController:
|
|||
room_id, on_invalidate=on_invalidate
|
||||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_canonical_alias_for_room(self, room_id: str) -> Optional[str]:
|
||||
"""Get canonical alias for room, if any
|
||||
|
||||
|
|
@ -466,6 +487,8 @@ class StateStorageController:
|
|||
|
||||
return event.content.get("canonical_alias")
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_current_state_deltas(
|
||||
self, prev_stream_id: int, max_stream_id: int
|
||||
) -> Tuple[int, List[Dict[str, Any]]]:
|
||||
|
|
@ -500,6 +523,7 @@ class StateStorageController:
|
|||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_current_state(
|
||||
self, room_id: str, state_filter: Optional[StateFilter] = None
|
||||
) -> StateMap[EventBase]:
|
||||
|
|
@ -516,6 +540,8 @@ class StateStorageController:
|
|||
|
||||
return state_map
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_current_state_event(
|
||||
self, room_id: str, event_type: str, state_key: str
|
||||
) -> Optional[EventBase]:
|
||||
|
|
@ -527,6 +553,8 @@ class StateStorageController:
|
|||
)
|
||||
return state_map.get(key)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_current_hosts_in_room(self, room_id: str) -> AbstractSet[str]:
|
||||
"""Get current hosts in room based on current state.
|
||||
|
||||
|
|
@ -538,6 +566,8 @@ class StateStorageController:
|
|||
|
||||
return await self.stores.main.get_current_hosts_in_room(room_id)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_current_hosts_in_room_ordered(self, room_id: str) -> List[str]:
|
||||
"""Get current hosts in room based on current state.
|
||||
|
||||
|
|
@ -553,6 +583,8 @@ class StateStorageController:
|
|||
|
||||
return await self.stores.main.get_current_hosts_in_room_ordered(room_id)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_current_hosts_in_room_or_partial_state_approximation(
|
||||
self, room_id: str
|
||||
) -> Collection[str]:
|
||||
|
|
@ -582,6 +614,8 @@ class StateStorageController:
|
|||
|
||||
return hosts
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_users_in_room_with_profiles(
|
||||
self, room_id: str
|
||||
) -> Mapping[str, ProfileInfo]:
|
||||
|
|
|
|||
|
|
@ -13,8 +13,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from typing import TYPE_CHECKING, Collection, List, Tuple
|
||||
from typing import TYPE_CHECKING, Collection, Counter, List, Tuple
|
||||
|
||||
from synapse.api.errors import SynapseError
|
||||
from synapse.storage.database import LoggingTransaction
|
||||
|
|
|
|||
|
|
@ -565,9 +565,8 @@ class DatabasePool:
|
|||
# A set of tables that are not safe to use native upserts in.
|
||||
self._unsafe_to_upsert_tables = set(UNIQUE_INDEX_BACKGROUND_UPDATES.keys())
|
||||
|
||||
# We add the user_directory_search table to the blacklist on SQLite
|
||||
# because the existing search table does not have an index, making it
|
||||
# unsafe to use native upserts.
|
||||
# The user_directory_search table is unsafe to use native upserts
|
||||
# on SQLite because the existing search table does not have an index.
|
||||
if isinstance(self.engine, Sqlite3Engine):
|
||||
self._unsafe_to_upsert_tables.add("user_directory_search")
|
||||
|
||||
|
|
|
|||
|
|
@ -85,13 +85,10 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
|
|||
writers=hs.config.worker.writers.account_data,
|
||||
)
|
||||
else:
|
||||
# Multiple writers are not supported for SQLite.
|
||||
#
|
||||
# We shouldn't be running in worker mode with SQLite, but its useful
|
||||
# to support it for unit tests.
|
||||
#
|
||||
# If this process is the writer than we need to use
|
||||
# `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets
|
||||
# updated over replication. (Multiple writers are not supported for
|
||||
# SQLite).
|
||||
self._account_data_id_gen = StreamIdGenerator(
|
||||
db_conn,
|
||||
hs.get_replication_notifier(),
|
||||
|
|
|
|||
|
|
@ -274,11 +274,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore):
|
|||
async def invalidate_cache_and_stream(
|
||||
self, cache_name: str, keys: Tuple[Any, ...]
|
||||
) -> None:
|
||||
"""Invalidates the cache and adds it to the cache stream so slaves
|
||||
"""Invalidates the cache and adds it to the cache stream so other workers
|
||||
will know to invalidate their caches.
|
||||
|
||||
This should only be used to invalidate caches where slaves won't
|
||||
otherwise know from other replication streams that the cache should
|
||||
This should only be used to invalidate caches where other workers won't
|
||||
otherwise have known from other replication streams that the cache should
|
||||
be invalidated.
|
||||
"""
|
||||
cache_func = getattr(self, cache_name, None)
|
||||
|
|
@ -297,11 +297,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore):
|
|||
cache_func: CachedFunction,
|
||||
keys: Tuple[Any, ...],
|
||||
) -> None:
|
||||
"""Invalidates the cache and adds it to the cache stream so slaves
|
||||
"""Invalidates the cache and adds it to the cache stream so other workers
|
||||
will know to invalidate their caches.
|
||||
|
||||
This should only be used to invalidate caches where slaves won't
|
||||
otherwise know from other replication streams that the cache should
|
||||
This should only be used to invalidate caches where other workers won't
|
||||
otherwise have known from other replication streams that the cache should
|
||||
be invalidated.
|
||||
"""
|
||||
txn.call_after(cache_func.invalidate, keys)
|
||||
|
|
@ -310,7 +310,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore):
|
|||
def _invalidate_all_cache_and_stream(
|
||||
self, txn: LoggingTransaction, cache_func: CachedFunction
|
||||
) -> None:
|
||||
"""Invalidates the entire cache and adds it to the cache stream so slaves
|
||||
"""Invalidates the entire cache and adds it to the cache stream so other workers
|
||||
will know to invalidate their caches.
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -105,8 +105,6 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
|
|||
is_writer=hs.config.worker.worker_app is None,
|
||||
)
|
||||
|
||||
# Type-ignore: _device_list_id_gen is mixed in from either DataStore (as a
|
||||
# StreamIdGenerator) or SlavedDataStore (as a SlavedIdTracker).
|
||||
device_list_max = self._device_list_id_gen.get_current_token()
|
||||
device_list_prefill, min_device_list_id = self.db_pool.get_cache_dict(
|
||||
db_conn,
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ from synapse.storage.database import (
|
|||
from synapse.storage.databases.main.events_worker import EventsWorkerStore
|
||||
from synapse.storage.databases.main.signatures import SignatureWorkerStore
|
||||
from synapse.storage.engines import PostgresEngine, Sqlite3Engine
|
||||
from synapse.types import JsonDict
|
||||
from synapse.types import JsonDict, StrCollection
|
||||
from synapse.util import json_encoder
|
||||
from synapse.util.caches.descriptors import cached
|
||||
from synapse.util.caches.lrucache import LruCache
|
||||
|
|
@ -1583,6 +1583,35 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
|
|||
|
||||
txn.execute(sql, (room_id, event_id, 1, self._clock.time_msec(), cause))
|
||||
|
||||
@trace
|
||||
async def get_event_ids_with_failed_pull_attempts(
|
||||
self, event_ids: StrCollection
|
||||
) -> Set[str]:
|
||||
"""
|
||||
Filter the given list of `event_ids` and return events which have any failed
|
||||
pull attempts.
|
||||
|
||||
Args:
|
||||
event_ids: A list of events to filter down.
|
||||
|
||||
Returns:
|
||||
A filtered down list of `event_ids` that have previous failed pull attempts.
|
||||
"""
|
||||
|
||||
rows = await self.db_pool.simple_select_many_batch(
|
||||
table="event_failed_pull_attempts",
|
||||
column="event_id",
|
||||
iterable=event_ids,
|
||||
keyvalues={},
|
||||
retcols=("event_id",),
|
||||
desc="get_event_ids_with_failed_pull_attempts",
|
||||
)
|
||||
event_ids_with_failed_pull_attempts: Set[str] = {
|
||||
row["event_id"] for row in rows
|
||||
}
|
||||
|
||||
return event_ids_with_failed_pull_attempts
|
||||
|
||||
@trace
|
||||
async def get_event_ids_to_not_pull_from_backoff(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -289,179 +289,52 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
|
|||
unique=True,
|
||||
)
|
||||
|
||||
self.db_pool.updates.register_background_update_handler(
|
||||
"event_push_backfill_thread_id",
|
||||
self._background_backfill_thread_id,
|
||||
self.db_pool.updates.register_background_validate_constraint(
|
||||
"event_push_actions_staging_thread_id",
|
||||
constraint_name="event_push_actions_staging_thread_id",
|
||||
table="event_push_actions_staging",
|
||||
)
|
||||
|
||||
# Indexes which will be used to quickly make the thread_id column non-null.
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
"event_push_actions_thread_id_null",
|
||||
index_name="event_push_actions_thread_id_null",
|
||||
self.db_pool.updates.register_background_validate_constraint(
|
||||
"event_push_actions_thread_id",
|
||||
constraint_name="event_push_actions_thread_id",
|
||||
table="event_push_actions",
|
||||
columns=["thread_id"],
|
||||
where_clause="thread_id IS NULL",
|
||||
)
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
"event_push_summary_thread_id_null",
|
||||
index_name="event_push_summary_thread_id_null",
|
||||
self.db_pool.updates.register_background_validate_constraint(
|
||||
"event_push_summary_thread_id",
|
||||
constraint_name="event_push_summary_thread_id",
|
||||
table="event_push_summary",
|
||||
columns=["thread_id"],
|
||||
where_clause="thread_id IS NULL",
|
||||
)
|
||||
|
||||
# Check ASAP (and then later, every 1s) to see if we have finished
|
||||
# background updates the event_push_actions and event_push_summary tables.
|
||||
self._clock.call_later(0.0, self._check_event_push_backfill_thread_id)
|
||||
self._event_push_backfill_thread_id_done = False
|
||||
|
||||
@wrap_as_background_process("check_event_push_backfill_thread_id")
|
||||
async def _check_event_push_backfill_thread_id(self) -> None:
|
||||
"""
|
||||
Has thread_id finished backfilling?
|
||||
|
||||
If not, we need to just-in-time update it so the queries work.
|
||||
"""
|
||||
done = await self.db_pool.updates.has_completed_background_update(
|
||||
"event_push_backfill_thread_id"
|
||||
self.db_pool.updates.register_background_update_handler(
|
||||
"event_push_drop_null_thread_id_indexes",
|
||||
self._background_drop_null_thread_id_indexes,
|
||||
)
|
||||
|
||||
if done:
|
||||
self._event_push_backfill_thread_id_done = True
|
||||
else:
|
||||
# Reschedule to run.
|
||||
self._clock.call_later(15.0, self._check_event_push_backfill_thread_id)
|
||||
|
||||
async def _background_backfill_thread_id(
|
||||
async def _background_drop_null_thread_id_indexes(
|
||||
self, progress: JsonDict, batch_size: int
|
||||
) -> int:
|
||||
"""
|
||||
Fill in the thread_id field for event_push_actions and event_push_summary.
|
||||
|
||||
This is preparatory so that it can be made non-nullable in the future.
|
||||
|
||||
Because all current (null) data is done in an unthreaded manner this
|
||||
simply assumes it is on the "main" timeline. Since event_push_actions
|
||||
are periodically cleared it is not possible to correctly re-calculate
|
||||
the thread_id.
|
||||
Drop the indexes used to find null thread_ids for event_push_actions and
|
||||
event_push_summary.
|
||||
"""
|
||||
event_push_actions_done = progress.get("event_push_actions_done", False)
|
||||
|
||||
def add_thread_id_txn(
|
||||
txn: LoggingTransaction, start_stream_ordering: int
|
||||
) -> int:
|
||||
sql = """
|
||||
SELECT stream_ordering
|
||||
FROM event_push_actions
|
||||
WHERE
|
||||
thread_id IS NULL
|
||||
AND stream_ordering > ?
|
||||
ORDER BY stream_ordering
|
||||
LIMIT ?
|
||||
"""
|
||||
txn.execute(sql, (start_stream_ordering, batch_size))
|
||||
def drop_null_thread_id_indexes_txn(txn: LoggingTransaction) -> None:
|
||||
sql = "DROP INDEX IF EXISTS event_push_actions_thread_id_null"
|
||||
logger.debug("[SQL] %s", sql)
|
||||
txn.execute(sql)
|
||||
|
||||
# No more rows to process.
|
||||
rows = txn.fetchall()
|
||||
if not rows:
|
||||
progress["event_push_actions_done"] = True
|
||||
self.db_pool.updates._background_update_progress_txn(
|
||||
txn, "event_push_backfill_thread_id", progress
|
||||
)
|
||||
return 0
|
||||
sql = "DROP INDEX IF EXISTS event_push_summary_thread_id_null"
|
||||
logger.debug("[SQL] %s", sql)
|
||||
txn.execute(sql)
|
||||
|
||||
# Update the thread ID for any of those rows.
|
||||
max_stream_ordering = rows[-1][0]
|
||||
|
||||
sql = """
|
||||
UPDATE event_push_actions
|
||||
SET thread_id = 'main'
|
||||
WHERE ? < stream_ordering AND stream_ordering <= ? AND thread_id IS NULL
|
||||
"""
|
||||
txn.execute(
|
||||
sql,
|
||||
(
|
||||
start_stream_ordering,
|
||||
max_stream_ordering,
|
||||
),
|
||||
)
|
||||
|
||||
# Update progress.
|
||||
processed_rows = txn.rowcount
|
||||
progress["max_event_push_actions_stream_ordering"] = max_stream_ordering
|
||||
self.db_pool.updates._background_update_progress_txn(
|
||||
txn, "event_push_backfill_thread_id", progress
|
||||
)
|
||||
|
||||
return processed_rows
|
||||
|
||||
def add_thread_id_summary_txn(txn: LoggingTransaction) -> int:
|
||||
min_user_id = progress.get("max_summary_user_id", "")
|
||||
min_room_id = progress.get("max_summary_room_id", "")
|
||||
|
||||
# Slightly overcomplicated query for getting the Nth user ID / room
|
||||
# ID tuple, or the last if there are less than N remaining.
|
||||
sql = """
|
||||
SELECT user_id, room_id FROM (
|
||||
SELECT user_id, room_id FROM event_push_summary
|
||||
WHERE (user_id, room_id) > (?, ?)
|
||||
AND thread_id IS NULL
|
||||
ORDER BY user_id, room_id
|
||||
LIMIT ?
|
||||
) AS e
|
||||
ORDER BY user_id DESC, room_id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
|
||||
txn.execute(sql, (min_user_id, min_room_id, batch_size))
|
||||
row = txn.fetchone()
|
||||
if not row:
|
||||
return 0
|
||||
|
||||
max_user_id, max_room_id = row
|
||||
|
||||
sql = """
|
||||
UPDATE event_push_summary
|
||||
SET thread_id = 'main'
|
||||
WHERE
|
||||
(?, ?) < (user_id, room_id) AND (user_id, room_id) <= (?, ?)
|
||||
AND thread_id IS NULL
|
||||
"""
|
||||
txn.execute(sql, (min_user_id, min_room_id, max_user_id, max_room_id))
|
||||
processed_rows = txn.rowcount
|
||||
|
||||
progress["max_summary_user_id"] = max_user_id
|
||||
progress["max_summary_room_id"] = max_room_id
|
||||
self.db_pool.updates._background_update_progress_txn(
|
||||
txn, "event_push_backfill_thread_id", progress
|
||||
)
|
||||
|
||||
return processed_rows
|
||||
|
||||
# First update the event_push_actions table, then the event_push_summary table.
|
||||
#
|
||||
# Note that the event_push_actions_staging table is ignored since it is
|
||||
# assumed that items in that table will only exist for a short period of
|
||||
# time.
|
||||
if not event_push_actions_done:
|
||||
result = await self.db_pool.runInteraction(
|
||||
"event_push_backfill_thread_id",
|
||||
add_thread_id_txn,
|
||||
progress.get("max_event_push_actions_stream_ordering", 0),
|
||||
)
|
||||
else:
|
||||
result = await self.db_pool.runInteraction(
|
||||
"event_push_backfill_thread_id",
|
||||
add_thread_id_summary_txn,
|
||||
)
|
||||
|
||||
# Only done after the event_push_summary table is done.
|
||||
if not result:
|
||||
await self.db_pool.updates._end_background_update(
|
||||
"event_push_backfill_thread_id"
|
||||
)
|
||||
|
||||
return result
|
||||
await self.db_pool.runInteraction(
|
||||
"drop_null_thread_id_indexes_txn",
|
||||
drop_null_thread_id_indexes_txn,
|
||||
)
|
||||
await self.db_pool.updates._end_background_update(
|
||||
"event_push_drop_null_thread_id_indexes"
|
||||
)
|
||||
return 0
|
||||
|
||||
async def get_unread_counts_by_room_for_user(self, user_id: str) -> Dict[str, int]:
|
||||
"""Get the notification count by room for a user. Only considers notifications,
|
||||
|
|
@ -711,25 +584,6 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
|
|||
(ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE),
|
||||
)
|
||||
|
||||
# First ensure that the existing rows have an updated thread_id field.
|
||||
if not self._event_push_backfill_thread_id_done:
|
||||
txn.execute(
|
||||
"""
|
||||
UPDATE event_push_summary
|
||||
SET thread_id = ?
|
||||
WHERE room_id = ? AND user_id = ? AND thread_id is NULL
|
||||
""",
|
||||
(MAIN_TIMELINE, room_id, user_id),
|
||||
)
|
||||
txn.execute(
|
||||
"""
|
||||
UPDATE event_push_actions
|
||||
SET thread_id = ?
|
||||
WHERE room_id = ? AND user_id = ? AND thread_id is NULL
|
||||
""",
|
||||
(MAIN_TIMELINE, room_id, user_id),
|
||||
)
|
||||
|
||||
# First we pull the counts from the summary table.
|
||||
#
|
||||
# We check that `last_receipt_stream_ordering` matches the stream ordering of the
|
||||
|
|
@ -1545,25 +1399,6 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
|
|||
(room_id, user_id, stream_ordering, *thread_args),
|
||||
)
|
||||
|
||||
# First ensure that the existing rows have an updated thread_id field.
|
||||
if not self._event_push_backfill_thread_id_done:
|
||||
txn.execute(
|
||||
"""
|
||||
UPDATE event_push_summary
|
||||
SET thread_id = ?
|
||||
WHERE room_id = ? AND user_id = ? AND thread_id is NULL
|
||||
""",
|
||||
(MAIN_TIMELINE, room_id, user_id),
|
||||
)
|
||||
txn.execute(
|
||||
"""
|
||||
UPDATE event_push_actions
|
||||
SET thread_id = ?
|
||||
WHERE room_id = ? AND user_id = ? AND thread_id is NULL
|
||||
""",
|
||||
(MAIN_TIMELINE, room_id, user_id),
|
||||
)
|
||||
|
||||
# Fetch the notification counts between the stream ordering of the
|
||||
# latest receipt and what was previously summarised.
|
||||
unread_counts = self._get_notif_unread_count_for_user_room(
|
||||
|
|
@ -1698,19 +1533,6 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
|
|||
rotate_to_stream_ordering: The new maximum event stream ordering to summarise.
|
||||
"""
|
||||
|
||||
# Ensure that any new actions have an updated thread_id.
|
||||
if not self._event_push_backfill_thread_id_done:
|
||||
txn.execute(
|
||||
"""
|
||||
UPDATE event_push_actions
|
||||
SET thread_id = ?
|
||||
WHERE ? < stream_ordering AND stream_ordering <= ? AND thread_id IS NULL
|
||||
""",
|
||||
(MAIN_TIMELINE, old_rotate_stream_ordering, rotate_to_stream_ordering),
|
||||
)
|
||||
|
||||
# XXX Do we need to update summaries here too?
|
||||
|
||||
# Calculate the new counts that should be upserted into event_push_summary
|
||||
sql = """
|
||||
SELECT user_id, room_id, thread_id,
|
||||
|
|
@ -1773,20 +1595,6 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
|
|||
|
||||
logger.info("Rotating notifications, handling %d rows", len(summaries))
|
||||
|
||||
# Ensure that any updated threads have the proper thread_id.
|
||||
if not self._event_push_backfill_thread_id_done:
|
||||
txn.execute_batch(
|
||||
"""
|
||||
UPDATE event_push_summary
|
||||
SET thread_id = ?
|
||||
WHERE room_id = ? AND user_id = ? AND thread_id is NULL
|
||||
""",
|
||||
[
|
||||
(MAIN_TIMELINE, room_id, user_id)
|
||||
for user_id, room_id, _ in summaries
|
||||
],
|
||||
)
|
||||
|
||||
self.db_pool.simple_upsert_many_txn(
|
||||
txn,
|
||||
table="event_push_summary",
|
||||
|
|
|
|||
|
|
@ -213,13 +213,10 @@ class EventsWorkerStore(SQLBaseStore):
|
|||
writers=hs.config.worker.writers.events,
|
||||
)
|
||||
else:
|
||||
# Multiple writers are not supported for SQLite.
|
||||
#
|
||||
# We shouldn't be running in worker mode with SQLite, but its useful
|
||||
# to support it for unit tests.
|
||||
#
|
||||
# If this process is the writer than we need to use
|
||||
# `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets
|
||||
# updated over replication. (Multiple writers are not supported for
|
||||
# SQLite).
|
||||
self._stream_id_gen = StreamIdGenerator(
|
||||
db_conn,
|
||||
hs.get_replication_notifier(),
|
||||
|
|
@ -1976,12 +1973,6 @@ class EventsWorkerStore(SQLBaseStore):
|
|||
|
||||
return rows, to_token, True
|
||||
|
||||
async def is_event_after(self, event_id1: str, event_id2: str) -> bool:
|
||||
"""Returns True if event_id1 is after event_id2 in the stream"""
|
||||
to_1, so_1 = await self.get_event_ordering(event_id1)
|
||||
to_2, so_2 = await self.get_event_ordering(event_id2)
|
||||
return (to_1, so_1) > (to_2, so_2)
|
||||
|
||||
@cached(max_entries=5000)
|
||||
async def get_event_ordering(self, event_id: str) -> Tuple[int, int]:
|
||||
res = await self.db_pool.simple_select_one(
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from synapse.storage.database import (
|
|||
LoggingDatabaseConnection,
|
||||
LoggingTransaction,
|
||||
)
|
||||
from synapse.storage.engines import PostgresEngine
|
||||
from synapse.types import JsonDict, UserID
|
||||
from synapse.util.caches.descriptors import cached
|
||||
|
||||
|
|
@ -40,6 +41,8 @@ class FilteringWorkerStore(SQLBaseStore):
|
|||
hs: "HomeServer",
|
||||
):
|
||||
super().__init__(database, db_conn, hs)
|
||||
self.server_name: str = hs.hostname
|
||||
self.database_engine = database.engine
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
"full_users_filters_unique_idx",
|
||||
index_name="full_users_unique_idx",
|
||||
|
|
@ -48,6 +51,98 @@ class FilteringWorkerStore(SQLBaseStore):
|
|||
unique=True,
|
||||
)
|
||||
|
||||
self.db_pool.updates.register_background_update_handler(
|
||||
"populate_full_user_id_user_filters",
|
||||
self.populate_full_user_id_user_filters,
|
||||
)
|
||||
|
||||
async def populate_full_user_id_user_filters(
|
||||
self, progress: JsonDict, batch_size: int
|
||||
) -> int:
|
||||
"""
|
||||
Background update to populate the column `full_user_id` of the table
|
||||
user_filters from entries in the column `user_local_part` of the same table
|
||||
"""
|
||||
|
||||
lower_bound_id = progress.get("lower_bound_id", "")
|
||||
|
||||
def _get_last_id(txn: LoggingTransaction) -> Optional[str]:
|
||||
sql = """
|
||||
SELECT user_id FROM user_filters
|
||||
WHERE user_id > ?
|
||||
ORDER BY user_id
|
||||
LIMIT 1 OFFSET 50
|
||||
"""
|
||||
txn.execute(sql, (lower_bound_id,))
|
||||
res = txn.fetchone()
|
||||
if res:
|
||||
upper_bound_id = res[0]
|
||||
return upper_bound_id
|
||||
else:
|
||||
return None
|
||||
|
||||
def _process_batch(
|
||||
txn: LoggingTransaction, lower_bound_id: str, upper_bound_id: str
|
||||
) -> None:
|
||||
sql = """
|
||||
UPDATE user_filters
|
||||
SET full_user_id = '@' || user_id || ?
|
||||
WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL
|
||||
"""
|
||||
txn.execute(sql, (f":{self.server_name}", lower_bound_id, upper_bound_id))
|
||||
|
||||
def _final_batch(txn: LoggingTransaction, lower_bound_id: str) -> None:
|
||||
sql = """
|
||||
UPDATE user_filters
|
||||
SET full_user_id = '@' || user_id || ?
|
||||
WHERE ? < user_id AND full_user_id IS NULL
|
||||
"""
|
||||
txn.execute(
|
||||
sql,
|
||||
(
|
||||
f":{self.server_name}",
|
||||
lower_bound_id,
|
||||
),
|
||||
)
|
||||
|
||||
if isinstance(self.database_engine, PostgresEngine):
|
||||
sql = """
|
||||
ALTER TABLE user_filters VALIDATE CONSTRAINT full_user_id_not_null
|
||||
"""
|
||||
txn.execute(sql)
|
||||
|
||||
upper_bound_id = await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_user_filters", _get_last_id
|
||||
)
|
||||
|
||||
if upper_bound_id is None:
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_user_filters", _final_batch, lower_bound_id
|
||||
)
|
||||
|
||||
await self.db_pool.updates._end_background_update(
|
||||
"populate_full_user_id_user_filters"
|
||||
)
|
||||
return 1
|
||||
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_user_filters",
|
||||
_process_batch,
|
||||
lower_bound_id,
|
||||
upper_bound_id,
|
||||
)
|
||||
|
||||
progress["lower_bound_id"] = upper_bound_id
|
||||
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_user_filters",
|
||||
self.db_pool.updates._background_update_progress_txn,
|
||||
"populate_full_user_id_user_filters",
|
||||
progress,
|
||||
)
|
||||
|
||||
return 50
|
||||
|
||||
@cached(num_args=2)
|
||||
async def get_user_filter(
|
||||
self, user_localpart: str, filter_id: Union[int, str]
|
||||
|
|
|
|||
|
|
@ -248,89 +248,6 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore):
|
|||
(count,) = cast(Tuple[int], txn.fetchone())
|
||||
return count
|
||||
|
||||
async def count_r30_users(self) -> Dict[str, int]:
|
||||
"""
|
||||
Counts the number of 30 day retained users, defined as:-
|
||||
* Users who have created their accounts more than 30 days ago
|
||||
* Where last seen at most 30 days ago
|
||||
* Where account creation and last_seen are > 30 days apart
|
||||
|
||||
Returns:
|
||||
A mapping of counts globally as well as broken out by platform.
|
||||
"""
|
||||
|
||||
def _count_r30_users(txn: LoggingTransaction) -> Dict[str, int]:
|
||||
thirty_days_in_secs = 86400 * 30
|
||||
now = int(self._clock.time())
|
||||
thirty_days_ago_in_secs = now - thirty_days_in_secs
|
||||
|
||||
sql = """
|
||||
SELECT platform, COUNT(*) FROM (
|
||||
SELECT
|
||||
users.name, platform, users.creation_ts * 1000,
|
||||
MAX(uip.last_seen)
|
||||
FROM users
|
||||
INNER JOIN (
|
||||
SELECT
|
||||
user_id,
|
||||
last_seen,
|
||||
CASE
|
||||
WHEN user_agent LIKE '%%Android%%' THEN 'android'
|
||||
WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
|
||||
WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
|
||||
WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
|
||||
WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
|
||||
ELSE 'unknown'
|
||||
END
|
||||
AS platform
|
||||
FROM user_ips
|
||||
) uip
|
||||
ON users.name = uip.user_id
|
||||
AND users.appservice_id is NULL
|
||||
AND users.creation_ts < ?
|
||||
AND uip.last_seen/1000 > ?
|
||||
AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
|
||||
GROUP BY users.name, platform, users.creation_ts
|
||||
) u GROUP BY platform
|
||||
"""
|
||||
|
||||
results = {}
|
||||
txn.execute(sql, (thirty_days_ago_in_secs, thirty_days_ago_in_secs))
|
||||
|
||||
for row in txn:
|
||||
if row[0] == "unknown":
|
||||
pass
|
||||
results[row[0]] = row[1]
|
||||
|
||||
sql = """
|
||||
SELECT COUNT(*) FROM (
|
||||
SELECT users.name, users.creation_ts * 1000,
|
||||
MAX(uip.last_seen)
|
||||
FROM users
|
||||
INNER JOIN (
|
||||
SELECT
|
||||
user_id,
|
||||
last_seen
|
||||
FROM user_ips
|
||||
) uip
|
||||
ON users.name = uip.user_id
|
||||
AND appservice_id is NULL
|
||||
AND users.creation_ts < ?
|
||||
AND uip.last_seen/1000 > ?
|
||||
AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
|
||||
GROUP BY users.name, users.creation_ts
|
||||
) u
|
||||
"""
|
||||
|
||||
txn.execute(sql, (thirty_days_ago_in_secs, thirty_days_ago_in_secs))
|
||||
|
||||
(count,) = cast(Tuple[int], txn.fetchone())
|
||||
results["all"] = count
|
||||
|
||||
return results
|
||||
|
||||
return await self.db_pool.runInteraction("count_r30_users", _count_r30_users)
|
||||
|
||||
async def count_r30v2_users(self) -> Dict[str, int]:
|
||||
"""
|
||||
Counts the number of 30 day retained users, defined as users that:
|
||||
|
|
|
|||
|
|
@ -15,9 +15,14 @@ from typing import TYPE_CHECKING, Optional
|
|||
|
||||
from synapse.api.errors import StoreError
|
||||
from synapse.storage._base import SQLBaseStore
|
||||
from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
|
||||
from synapse.storage.database import (
|
||||
DatabasePool,
|
||||
LoggingDatabaseConnection,
|
||||
LoggingTransaction,
|
||||
)
|
||||
from synapse.storage.databases.main.roommember import ProfileInfo
|
||||
from synapse.types import UserID
|
||||
from synapse.storage.engines import PostgresEngine
|
||||
from synapse.types import JsonDict, UserID
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from synapse.server import HomeServer
|
||||
|
|
@ -31,6 +36,8 @@ class ProfileWorkerStore(SQLBaseStore):
|
|||
hs: "HomeServer",
|
||||
):
|
||||
super().__init__(database, db_conn, hs)
|
||||
self.server_name: str = hs.hostname
|
||||
self.database_engine = database.engine
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
"profiles_full_user_id_key_idx",
|
||||
index_name="profiles_full_user_id_key",
|
||||
|
|
@ -39,6 +46,97 @@ class ProfileWorkerStore(SQLBaseStore):
|
|||
unique=True,
|
||||
)
|
||||
|
||||
self.db_pool.updates.register_background_update_handler(
|
||||
"populate_full_user_id_profiles", self.populate_full_user_id_profiles
|
||||
)
|
||||
|
||||
async def populate_full_user_id_profiles(
|
||||
self, progress: JsonDict, batch_size: int
|
||||
) -> int:
|
||||
"""
|
||||
Background update to populate the column `full_user_id` of the table
|
||||
profiles from entries in the column `user_local_part` of the same table
|
||||
"""
|
||||
|
||||
lower_bound_id = progress.get("lower_bound_id", "")
|
||||
|
||||
def _get_last_id(txn: LoggingTransaction) -> Optional[str]:
|
||||
sql = """
|
||||
SELECT user_id FROM profiles
|
||||
WHERE user_id > ?
|
||||
ORDER BY user_id
|
||||
LIMIT 1 OFFSET 50
|
||||
"""
|
||||
txn.execute(sql, (lower_bound_id,))
|
||||
res = txn.fetchone()
|
||||
if res:
|
||||
upper_bound_id = res[0]
|
||||
return upper_bound_id
|
||||
else:
|
||||
return None
|
||||
|
||||
def _process_batch(
|
||||
txn: LoggingTransaction, lower_bound_id: str, upper_bound_id: str
|
||||
) -> None:
|
||||
sql = """
|
||||
UPDATE profiles
|
||||
SET full_user_id = '@' || user_id || ?
|
||||
WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL
|
||||
"""
|
||||
txn.execute(sql, (f":{self.server_name}", lower_bound_id, upper_bound_id))
|
||||
|
||||
def _final_batch(txn: LoggingTransaction, lower_bound_id: str) -> None:
|
||||
sql = """
|
||||
UPDATE profiles
|
||||
SET full_user_id = '@' || user_id || ?
|
||||
WHERE ? < user_id AND full_user_id IS NULL
|
||||
"""
|
||||
txn.execute(
|
||||
sql,
|
||||
(
|
||||
f":{self.server_name}",
|
||||
lower_bound_id,
|
||||
),
|
||||
)
|
||||
|
||||
if isinstance(self.database_engine, PostgresEngine):
|
||||
sql = """
|
||||
ALTER TABLE profiles VALIDATE CONSTRAINT full_user_id_not_null
|
||||
"""
|
||||
txn.execute(sql)
|
||||
|
||||
upper_bound_id = await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_profiles", _get_last_id
|
||||
)
|
||||
|
||||
if upper_bound_id is None:
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_profiles", _final_batch, lower_bound_id
|
||||
)
|
||||
|
||||
await self.db_pool.updates._end_background_update(
|
||||
"populate_full_user_id_profiles"
|
||||
)
|
||||
return 1
|
||||
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_profiles",
|
||||
_process_batch,
|
||||
lower_bound_id,
|
||||
upper_bound_id,
|
||||
)
|
||||
|
||||
progress["lower_bound_id"] = upper_bound_id
|
||||
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_full_user_id_profiles",
|
||||
self.db_pool.updates._background_update_progress_txn,
|
||||
"populate_full_user_id_profiles",
|
||||
progress,
|
||||
)
|
||||
|
||||
return 50
|
||||
|
||||
async def get_profileinfo(self, user_localpart: str) -> ProfileInfo:
|
||||
try:
|
||||
profile = await self.db_pool.simple_select_one(
|
||||
|
|
|
|||
|
|
@ -85,13 +85,10 @@ class ReceiptsWorkerStore(SQLBaseStore):
|
|||
else:
|
||||
self._can_write_to_receipts = True
|
||||
|
||||
# Multiple writers are not supported for SQLite.
|
||||
#
|
||||
# We shouldn't be running in worker mode with SQLite, but its useful
|
||||
# to support it for unit tests.
|
||||
#
|
||||
# If this process is the writer than we need to use
|
||||
# `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets
|
||||
# updated over replication. (Multiple writers are not supported for
|
||||
# SQLite).
|
||||
self._receipts_id_gen = StreamIdGenerator(
|
||||
db_conn,
|
||||
hs.get_replication_notifier(),
|
||||
|
|
|
|||
|
|
@ -1099,7 +1099,7 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore):
|
|||
# `get_joined_hosts` is called with the "current" state group for the
|
||||
# room, and so consecutive calls will be for consecutive state groups
|
||||
# which point to the previous state group.
|
||||
cache = await self._get_joined_hosts_cache(room_id) # type: ignore[misc]
|
||||
cache = await self._get_joined_hosts_cache(room_id)
|
||||
|
||||
# If the state group in the cache matches, we already have the data we need.
|
||||
if state_entry.state_group == cache.state_group:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import re
|
|||
import unicodedata
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Collection,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
|
|
@ -45,7 +46,7 @@ from synapse.util.stringutils import non_null_str_or_none
|
|||
if TYPE_CHECKING:
|
||||
from synapse.server import HomeServer
|
||||
|
||||
from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules
|
||||
from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules, UserTypes
|
||||
from synapse.storage.database import (
|
||||
DatabasePool,
|
||||
LoggingDatabaseConnection,
|
||||
|
|
@ -356,13 +357,30 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore):
|
|||
Add all local users to the user directory.
|
||||
"""
|
||||
|
||||
def _get_next_batch(txn: LoggingTransaction) -> Optional[List[str]]:
|
||||
sql = "SELECT user_id FROM %s LIMIT %s" % (
|
||||
TEMP_TABLE + "_users",
|
||||
str(batch_size),
|
||||
)
|
||||
txn.execute(sql)
|
||||
user_result = cast(List[Tuple[str]], txn.fetchall())
|
||||
def _populate_user_directory_process_users_txn(
|
||||
txn: LoggingTransaction,
|
||||
) -> Optional[int]:
|
||||
if self.database_engine.supports_returning:
|
||||
# Note: we use an ORDER BY in the SELECT to force usage of an
|
||||
# index. Otherwise, postgres does a sequential scan that is
|
||||
# surprisingly slow (I think due to the fact it will read/skip
|
||||
# over lots of already deleted rows).
|
||||
sql = f"""
|
||||
DELETE FROM {TEMP_TABLE + "_users"}
|
||||
WHERE user_id IN (
|
||||
SELECT user_id FROM {TEMP_TABLE + "_users"} ORDER BY user_id LIMIT ?
|
||||
)
|
||||
RETURNING user_id
|
||||
"""
|
||||
txn.execute(sql, (batch_size,))
|
||||
user_result = cast(List[Tuple[str]], txn.fetchall())
|
||||
else:
|
||||
sql = "SELECT user_id FROM %s ORDER BY user_id LIMIT %s" % (
|
||||
TEMP_TABLE + "_users",
|
||||
str(batch_size),
|
||||
)
|
||||
txn.execute(sql)
|
||||
user_result = cast(List[Tuple[str]], txn.fetchall())
|
||||
|
||||
if not user_result:
|
||||
return None
|
||||
|
|
@ -378,85 +396,81 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore):
|
|||
assert count_result is not None
|
||||
progress["remaining"] = count_result[0]
|
||||
|
||||
return users_to_work_on
|
||||
if not users_to_work_on:
|
||||
return None
|
||||
|
||||
users_to_work_on = await self.db_pool.runInteraction(
|
||||
"populate_user_directory_temp_read", _get_next_batch
|
||||
logger.debug(
|
||||
"Processing the next %d users of %d remaining",
|
||||
len(users_to_work_on),
|
||||
progress["remaining"],
|
||||
)
|
||||
|
||||
# First filter down to users we want to insert into the user directory.
|
||||
users_to_insert = self._filter_local_users_for_dir_txn(
|
||||
txn, users_to_work_on
|
||||
)
|
||||
|
||||
# Next fetch their profiles. Note that the `user_id` here is the
|
||||
# *localpart*, and that not all users have profiles.
|
||||
profile_rows = self.db_pool.simple_select_many_txn(
|
||||
txn,
|
||||
table="profiles",
|
||||
column="user_id",
|
||||
iterable=[get_localpart_from_id(u) for u in users_to_insert],
|
||||
retcols=(
|
||||
"user_id",
|
||||
"displayname",
|
||||
"avatar_url",
|
||||
),
|
||||
keyvalues={},
|
||||
)
|
||||
profiles = {
|
||||
f"@{row['user_id']}:{self.server_name}": _UserDirProfile(
|
||||
f"@{row['user_id']}:{self.server_name}",
|
||||
row["displayname"],
|
||||
row["avatar_url"],
|
||||
)
|
||||
for row in profile_rows
|
||||
}
|
||||
|
||||
profiles_to_insert = [
|
||||
profiles.get(user_id) or _UserDirProfile(user_id)
|
||||
for user_id in users_to_insert
|
||||
]
|
||||
|
||||
# Actually insert the users with their profiles into the directory.
|
||||
self._update_profiles_in_user_dir_txn(txn, profiles_to_insert)
|
||||
|
||||
# We've finished processing the users. Delete it from the table, if
|
||||
# we haven't already.
|
||||
if not self.database_engine.supports_returning:
|
||||
self.db_pool.simple_delete_many_txn(
|
||||
txn,
|
||||
table=TEMP_TABLE + "_users",
|
||||
column="user_id",
|
||||
values=users_to_work_on,
|
||||
keyvalues={},
|
||||
)
|
||||
|
||||
# Update the remaining counter.
|
||||
progress["remaining"] -= len(users_to_work_on)
|
||||
self.db_pool.updates._background_update_progress_txn(
|
||||
txn, "populate_user_directory_process_users", progress
|
||||
)
|
||||
return len(users_to_work_on)
|
||||
|
||||
processed_count = await self.db_pool.runInteraction(
|
||||
"populate_user_directory_temp", _populate_user_directory_process_users_txn
|
||||
)
|
||||
|
||||
# No more users -- complete the transaction.
|
||||
if not users_to_work_on:
|
||||
if not processed_count:
|
||||
await self.db_pool.updates._end_background_update(
|
||||
"populate_user_directory_process_users"
|
||||
)
|
||||
return 1
|
||||
|
||||
logger.debug(
|
||||
"Processing the next %d users of %d remaining"
|
||||
% (len(users_to_work_on), progress["remaining"])
|
||||
)
|
||||
|
||||
# First filter down to users we want to insert into the user directory.
|
||||
users_to_insert = [
|
||||
user_id
|
||||
for user_id in users_to_work_on
|
||||
if await self.should_include_local_user_in_dir(user_id)
|
||||
]
|
||||
|
||||
# Next fetch their profiles. Note that the `user_id` here is the
|
||||
# *localpart*, and that not all users have profiles.
|
||||
profile_rows = await self.db_pool.simple_select_many_batch(
|
||||
table="profiles",
|
||||
column="user_id",
|
||||
iterable=[get_localpart_from_id(u) for u in users_to_insert],
|
||||
retcols=(
|
||||
"user_id",
|
||||
"displayname",
|
||||
"avatar_url",
|
||||
),
|
||||
keyvalues={},
|
||||
desc="populate_user_directory_process_users_get_profiles",
|
||||
)
|
||||
profiles = {
|
||||
f"@{row['user_id']}:{self.server_name}": _UserDirProfile(
|
||||
f"@{row['user_id']}:{self.server_name}",
|
||||
row["displayname"],
|
||||
row["avatar_url"],
|
||||
)
|
||||
for row in profile_rows
|
||||
}
|
||||
|
||||
profiles_to_insert = [
|
||||
profiles.get(user_id) or _UserDirProfile(user_id)
|
||||
for user_id in users_to_insert
|
||||
]
|
||||
|
||||
# Actually insert the users with their profiles into the directory.
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_user_directory_process_users_insertion",
|
||||
self._update_profiles_in_user_dir_txn,
|
||||
profiles_to_insert,
|
||||
)
|
||||
|
||||
# We've finished processing the users. Delete it from the table.
|
||||
await self.db_pool.simple_delete_many(
|
||||
table=TEMP_TABLE + "_users",
|
||||
column="user_id",
|
||||
iterable=users_to_work_on,
|
||||
keyvalues={},
|
||||
desc="populate_user_directory_process_users_delete",
|
||||
)
|
||||
|
||||
# Update the remaining counter.
|
||||
progress["remaining"] -= len(users_to_work_on)
|
||||
await self.db_pool.runInteraction(
|
||||
"populate_user_directory",
|
||||
self.db_pool.updates._background_update_progress_txn,
|
||||
"populate_user_directory_process_users",
|
||||
progress,
|
||||
)
|
||||
|
||||
return len(users_to_work_on)
|
||||
return processed_count
|
||||
|
||||
async def should_include_local_user_in_dir(self, user: str) -> bool:
|
||||
"""Certain classes of local user are omitted from the user directory.
|
||||
|
|
@ -494,6 +508,30 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore):
|
|||
|
||||
return True
|
||||
|
||||
def _filter_local_users_for_dir_txn(
|
||||
self, txn: LoggingTransaction, users: Collection[str]
|
||||
) -> Collection[str]:
|
||||
"""A batched version of `should_include_local_user_in_dir`"""
|
||||
users = [
|
||||
user
|
||||
for user in users
|
||||
if self.get_app_service_by_user_id(user) is None # type: ignore[attr-defined]
|
||||
and not self.get_if_app_services_interested_in_user(user) # type: ignore[attr-defined]
|
||||
]
|
||||
|
||||
rows = self.db_pool.simple_select_many_txn(
|
||||
txn,
|
||||
table="users",
|
||||
column="name",
|
||||
iterable=users,
|
||||
keyvalues={
|
||||
"deactivated": 0,
|
||||
},
|
||||
retcols=("name", "user_type"),
|
||||
)
|
||||
|
||||
return [row["name"] for row in rows if row["user_type"] != UserTypes.SUPPORT]
|
||||
|
||||
async def is_room_world_readable_or_publicly_joinable(self, room_id: str) -> bool:
|
||||
"""Check if the room is either world_readable or publically joinable"""
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
import logging
|
||||
from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union
|
||||
|
||||
from synapse.logging.opentracing import tag_args, trace
|
||||
from synapse.storage._base import SQLBaseStore
|
||||
from synapse.storage.database import (
|
||||
DatabasePool,
|
||||
|
|
@ -40,6 +41,8 @@ class StateGroupBackgroundUpdateStore(SQLBaseStore):
|
|||
updates.
|
||||
"""
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
def _count_state_group_hops_txn(
|
||||
self, txn: LoggingTransaction, state_group: int
|
||||
) -> int:
|
||||
|
|
@ -83,6 +86,8 @@ class StateGroupBackgroundUpdateStore(SQLBaseStore):
|
|||
|
||||
return count
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
def _get_state_groups_from_groups_txn(
|
||||
self,
|
||||
txn: LoggingTransaction,
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import attr
|
|||
from synapse.api.constants import EventTypes
|
||||
from synapse.events import EventBase
|
||||
from synapse.events.snapshot import UnpersistedEventContext, UnpersistedEventContextBase
|
||||
from synapse.logging.opentracing import tag_args, trace
|
||||
from synapse.storage._base import SQLBaseStore
|
||||
from synapse.storage.database import (
|
||||
DatabasePool,
|
||||
|
|
@ -159,6 +160,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
|
|||
"get_state_group_delta", _get_state_group_delta_txn
|
||||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
@cancellable
|
||||
async def _get_state_groups_from_groups(
|
||||
self, groups: List[int], state_filter: StateFilter
|
||||
|
|
@ -187,6 +190,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
|
|||
|
||||
return results
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
def _get_state_for_group_using_cache(
|
||||
self,
|
||||
cache: DictionaryCache[int, StateKey, str],
|
||||
|
|
@ -239,6 +244,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
|
|||
|
||||
return state_filter.filter_state(state_dict_ids), not missing_types
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
@cancellable
|
||||
async def _get_state_for_groups(
|
||||
self, groups: Iterable[int], state_filter: Optional[StateFilter] = None
|
||||
|
|
@ -305,6 +312,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
|
|||
|
||||
return state
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
def _get_state_for_groups_using_cache(
|
||||
self,
|
||||
groups: Iterable[int],
|
||||
|
|
@ -403,6 +412,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
|
|||
fetched_keys=non_member_types,
|
||||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def store_state_deltas_for_batched(
|
||||
self,
|
||||
events_and_context: List[Tuple[EventBase, UnpersistedEventContextBase]],
|
||||
|
|
@ -520,6 +531,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
|
|||
prev_group,
|
||||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def store_state_group(
|
||||
self,
|
||||
event_id: str,
|
||||
|
|
@ -772,6 +785,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
|
|||
((sg,) for sg in state_groups_to_delete),
|
||||
)
|
||||
|
||||
@trace
|
||||
@tag_args
|
||||
async def get_previous_state_groups(
|
||||
self, state_groups: Iterable[int]
|
||||
) -> Dict[int, int]:
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
SCHEMA_VERSION = 76 # remember to update the list below when updating
|
||||
SCHEMA_VERSION = 77 # remember to update the list below when updating
|
||||
"""Represents the expectations made by the codebase about the database schema
|
||||
|
||||
This should be incremented whenever the codebase changes its requirements on the
|
||||
|
|
@ -100,13 +100,22 @@ Changes in SCHEMA_VERSION = 75:
|
|||
|
||||
Changes in SCHEMA_VERSION = 76:
|
||||
- Adds a full_user_id column to tables profiles and user_filters.
|
||||
|
||||
Changes in SCHEMA_VERSION = 77
|
||||
- (Postgres) Add NOT VALID CHECK (full_user_id IS NOT NULL) to tables profiles and user_filters
|
||||
"""
|
||||
|
||||
|
||||
SCHEMA_COMPAT_VERSION = (
|
||||
# Queries against `event_stream_ordering` columns in membership tables must
|
||||
# be disambiguated.
|
||||
74
|
||||
#
|
||||
# The threads_id column must written to with non-null values for the
|
||||
# event_push_actions, event_push_actions_staging, and event_push_summary tables.
|
||||
#
|
||||
# insertions to the column `full_user_id` of tables profiles and user_filters can no
|
||||
# longer be null
|
||||
76
|
||||
)
|
||||
"""Limit on how far the synapse codebase can be rolled back without breaking db compat
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from synapse.storage.prepare_database import get_statements
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# This stream is used to notify replication slaves that some caches have
|
||||
# This stream is used to notify workers over replication that some caches have
|
||||
# been invalidated that they cannot infer from the other streams.
|
||||
CREATE_TABLE = """
|
||||
CREATE TABLE cache_invalidation_stream (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
/* Copyright 2023 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
ALTER TABLE profiles ADD CONSTRAINT full_user_id_not_null CHECK (full_user_id IS NOT NULL) NOT VALID;
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/* Copyright 2023 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
ALTER TABLE user_filters ADD CONSTRAINT full_user_id_not_null CHECK (full_user_id IS NOT NULL) NOT VALID;
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/* Copyright 2023 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (7703, 'populate_full_user_id_profiles', '{}');
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/* Copyright 2023 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (7704, 'populate_full_user_id_user_filters', '{}');
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
/* Copyright 2023 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- Force the background updates from 06thread_notifications.sql to run in the
|
||||
-- foreground as code will now require those to be "done".
|
||||
|
||||
DELETE FROM background_updates WHERE update_name = 'event_push_backfill_thread_id';
|
||||
|
||||
-- Overwrite any null thread_id values.
|
||||
UPDATE event_push_actions_staging SET thread_id = 'main' WHERE thread_id IS NULL;
|
||||
UPDATE event_push_actions SET thread_id = 'main' WHERE thread_id IS NULL;
|
||||
UPDATE event_push_summary SET thread_id = 'main' WHERE thread_id IS NULL;
|
||||
|
||||
-- Drop the background updates to calculate the indexes used to find null thread_ids.
|
||||
DELETE FROM background_updates WHERE update_name = 'event_push_actions_thread_id_null';
|
||||
DELETE FROM background_updates WHERE update_name = 'event_push_summary_thread_id_null';
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
/* Copyright 2022 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- The thread_id columns can now be made non-nullable.
|
||||
--
|
||||
-- SQLite doesn't support modifying columns to an existing table, so it must
|
||||
-- be recreated.
|
||||
|
||||
-- Create the new tables.
|
||||
CREATE TABLE event_push_actions_staging_new (
|
||||
event_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
actions TEXT NOT NULL,
|
||||
notif SMALLINT NOT NULL,
|
||||
highlight SMALLINT NOT NULL,
|
||||
unread SMALLINT,
|
||||
thread_id TEXT,
|
||||
inserted_ts BIGINT,
|
||||
CONSTRAINT event_push_actions_staging_thread_id CHECK (thread_id is NOT NULL)
|
||||
);
|
||||
|
||||
CREATE TABLE event_push_actions_new (
|
||||
room_id TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
profile_tag VARCHAR(32),
|
||||
actions TEXT NOT NULL,
|
||||
topological_ordering BIGINT,
|
||||
stream_ordering BIGINT,
|
||||
notif SMALLINT,
|
||||
highlight SMALLINT,
|
||||
unread SMALLINT,
|
||||
thread_id TEXT,
|
||||
CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag),
|
||||
CONSTRAINT event_push_actions_thread_id CHECK (thread_id is NOT NULL)
|
||||
);
|
||||
|
||||
CREATE TABLE event_push_summary_new (
|
||||
user_id TEXT NOT NULL,
|
||||
room_id TEXT NOT NULL,
|
||||
notif_count BIGINT NOT NULL,
|
||||
stream_ordering BIGINT NOT NULL,
|
||||
unread_count BIGINT,
|
||||
last_receipt_stream_ordering BIGINT,
|
||||
thread_id TEXT,
|
||||
CONSTRAINT event_push_summary_thread_id CHECK (thread_id is NOT NULL)
|
||||
);
|
||||
|
||||
-- Copy the data.
|
||||
INSERT INTO event_push_actions_staging_new (event_id, user_id, actions, notif, highlight, unread, thread_id, inserted_ts)
|
||||
SELECT event_id, user_id, actions, notif, highlight, unread, thread_id, inserted_ts
|
||||
FROM event_push_actions_staging;
|
||||
|
||||
INSERT INTO event_push_actions_new (room_id, event_id, user_id, profile_tag, actions, topological_ordering, stream_ordering, notif, highlight, unread, thread_id)
|
||||
SELECT room_id, event_id, user_id, profile_tag, actions, topological_ordering, stream_ordering, notif, highlight, unread, thread_id
|
||||
FROM event_push_actions;
|
||||
|
||||
INSERT INTO event_push_summary_new (user_id, room_id, notif_count, stream_ordering, unread_count, last_receipt_stream_ordering, thread_id)
|
||||
SELECT user_id, room_id, notif_count, stream_ordering, unread_count, last_receipt_stream_ordering, thread_id
|
||||
FROM event_push_summary;
|
||||
|
||||
-- Drop the old tables.
|
||||
DROP TABLE event_push_actions_staging;
|
||||
DROP TABLE event_push_actions;
|
||||
DROP TABLE event_push_summary;
|
||||
|
||||
-- Rename the tables.
|
||||
ALTER TABLE event_push_actions_staging_new RENAME TO event_push_actions_staging;
|
||||
ALTER TABLE event_push_actions_new RENAME TO event_push_actions;
|
||||
ALTER TABLE event_push_summary_new RENAME TO event_push_summary;
|
||||
|
||||
-- Recreate the indexes.
|
||||
CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging(event_id);
|
||||
|
||||
CREATE INDEX event_push_actions_highlights_index ON event_push_actions (user_id, room_id, topological_ordering, stream_ordering);
|
||||
CREATE INDEX event_push_actions_rm_tokens on event_push_actions( user_id, room_id, topological_ordering, stream_ordering );
|
||||
CREATE INDEX event_push_actions_room_id_user_id on event_push_actions(room_id, user_id);
|
||||
CREATE INDEX event_push_actions_stream_ordering on event_push_actions( stream_ordering, user_id );
|
||||
CREATE INDEX event_push_actions_u_highlight ON event_push_actions (user_id, stream_ordering);
|
||||
|
||||
CREATE UNIQUE INDEX event_push_summary_unique_index2 ON event_push_summary (user_id, room_id, thread_id) ;
|
||||
|
||||
-- Recreate some indexes in the background, by re-running the background updates
|
||||
-- from 72/02event_push_actions_index.sql and 72/06thread_notifications.sql.
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
|
||||
(7706, 'event_push_summary_unique_index2', '{}')
|
||||
ON CONFLICT (update_name) DO UPDATE SET progress_json = '{}';
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
|
||||
(7706, 'event_push_actions_stream_highlight_index', '{}')
|
||||
ON CONFLICT (update_name) DO UPDATE SET progress_json = '{}';
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/* Copyright 2022 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- The thread_id columns can now be made non-nullable, this is done by using a
|
||||
-- constraint (and not altering the column) to avoid taking out a full table lock.
|
||||
--
|
||||
-- We initially add an invalid constraint which guards against new data (this
|
||||
-- doesn't lock the table).
|
||||
ALTER TABLE event_push_actions
|
||||
ADD CONSTRAINT event_push_actions_thread_id CHECK (thread_id IS NOT NULL) NOT VALID;
|
||||
|
||||
-- We then validate the constraint which doesn't need to worry about new data. It
|
||||
-- only needs a SHARE UPDATE EXCLUSIVE lock but can still take a while to complete.
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
|
||||
(7706, 'event_push_actions_thread_id', '{}', 'event_push_actions_staging_thread_id');
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/* Copyright 2022 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- The thread_id columns can now be made non-nullable, this is done by using a
|
||||
-- constraint (and not altering the column) to avoid taking out a full table lock.
|
||||
--
|
||||
-- We initially add an invalid constraint which guards against new data (this
|
||||
-- doesn't lock the table).
|
||||
ALTER TABLE event_push_actions_staging
|
||||
ADD CONSTRAINT event_push_actions_staging_thread_id CHECK (thread_id IS NOT NULL) NOT VALID;
|
||||
|
||||
-- We then validate the constraint which doesn't need to worry about new data. It
|
||||
-- only needs a SHARE UPDATE EXCLUSIVE lock but can still take a while to complete.
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
|
||||
(7706, 'event_push_actions_staging_thread_id', '{}');
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/* Copyright 2022 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
-- The thread_id columns can now be made non-nullable, this is done by using a
|
||||
-- constraint (and not altering the column) to avoid taking out a full table lock.
|
||||
--
|
||||
-- We initially add an invalid constraint which guards against new data (this
|
||||
-- doesn't lock the table).
|
||||
ALTER TABLE event_push_summary
|
||||
ADD CONSTRAINT event_push_summary_thread_id CHECK (thread_id IS NOT NULL) NOT VALID;
|
||||
|
||||
-- We then validate the constraint which doesn't need to worry about new data. It
|
||||
-- only needs a SHARE UPDATE EXCLUSIVE lock but can still take a while to complete.
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
|
||||
(7706, 'event_push_summary_thread_id', '{}', 'event_push_actions_thread_id'),
|
||||
-- Also clean-up the old indexes.
|
||||
(7706, 'event_push_drop_null_thread_id_indexes', '{}', 'event_push_summary_thread_id');
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue