maxmustermann/synapse
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'develop' into matrix-org-hotfixes

Browse source
This commit is contained in:
Richard van der Hoff 2019-05-21 16:29:25 +01:00
parent ca03f90ee7 17f6804837
commit 1c8f2c34ff
22 changed files with 229 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/5146.bugfix Normal file
View file

@ -0,0 +1 @@
Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error.

1
changelog.d/5200.bugfix
View file

@ -1 +0,0 @@
Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo.

1
changelog.d/5204.feature Normal file
View file

@ -0,0 +1 @@
Stick an expiration date to any registered user missing one at startup if account validity is enabled.

2
changelog.d/5210.feature
View file

@ -1 +1 @@
Add a new room version which uses a new event ID format.
Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002).

1
changelog.d/5217.feature Normal file
View file

@ -0,0 +1 @@
Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002).

1
changelog.d/5218.bugfix Normal file
View file

@ -0,0 +1 @@
Fix incompatibility between ACME support and Python 3.5.2.

1
changelog.d/5219.bugfix Normal file
View file

@ -0,0 +1 @@
Fix error handling for rooms whose versions are unknown.

8
docs/sample_config.yaml
View file

@ -747,6 +747,14 @@ uploads_path: "DATADIR/uploads"
# link. ``%(app)s`` can be used as a placeholder for the ``app_name`` parameter
# from the ``email`` section.
#
# Once this feature is enabled, Synapse will look for registered users without an
# expiration date at startup and will add one to every account it found using the
# current settings at that time.
# This means that, if a validity period is set, and Synapse is restarted (it will
# then derive an expiration date from the current validity period), and some time
# after that the validity period changes and Synapse is restarted, the users'
# expiration dates won't be updated unless their account is manually renewed.
#
#account_validity:
# enabled: True
# period: 6w

18
synapse/api/errors.py
View file

@ -328,9 +328,23 @@ class RoomKeysVersionError(SynapseError):
self.current_version = current_version
class IncompatibleRoomVersionError(SynapseError):
"""A server is trying to join a room whose version it does not support."""
class UnsupportedRoomVersionError(SynapseError):
"""The client's request to create a room used a room version that the server does
not support."""
def __init__(self):
super(UnsupportedRoomVersionError, self).__init__(
code=400,
msg="Homeserver does not support this room version",
errcode=Codes.UNSUPPORTED_ROOM_VERSION,
)
class IncompatibleRoomVersionError(SynapseError):
"""A server is trying to join a room whose version it does not support.
Unlike UnsupportedRoomVersionError, it is specific to the case of the make_join
failing.
"""
def __init__(self, room_version):
super(IncompatibleRoomVersionError, self).__init__(
code=400,

8
synapse/api/room_versions.py
View file

@ -77,9 +77,9 @@ class RoomVersions(object):
EventFormatVersions.V2,
StateResolutionVersions.V2,
)
EVENTID_NOSLASH_TEST = RoomVersion(
"eventid-noslash-test",
RoomDisposition.UNSTABLE,
V4 = RoomVersion(
"4",
RoomDisposition.STABLE,
EventFormatVersions.V3,
StateResolutionVersions.V2,
)
@ -95,6 +95,6 @@ KNOWN_ROOM_VERSIONS = {
RoomVersions.V2,
RoomVersions.V3,
RoomVersions.STATE_V2_TEST,
RoomVersions.EVENTID_NOSLASH_TEST,
RoomVersions.V4,
)
} # type: dict[str, RoomVersion]

2
synapse/app/client_reader.py
View file

@ -29,7 +29,6 @@ from synapse.http.server import JsonResource
from synapse.http.site import SynapseSite
from synapse.metrics import RegistryProxy
from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
from synapse.replication.slave.storage import SlavedProfileStore
from synapse.replication.slave.storage._base import BaseSlavedStore
from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@ -84,7 +83,6 @@ class ClientReaderSlavedStore(
SlavedTransactionStore,
SlavedClientIpStore,
BaseSlavedStore,
SlavedProfileStore,
):
pass

8
synapse/config/registration.py
View file

@ -123,6 +123,14 @@ class RegistrationConfig(Config):
# link. ``%%(app)s`` can be used as a placeholder for the ``app_name`` parameter
# from the ``email`` section.
#
# Once this feature is enabled, Synapse will look for registered users without an
# expiration date at startup and will add one to every account it found using the
# current settings at that time.
# This means that, if a validity period is set, and Synapse is restarted (it will
# then derive an expiration date from the current validity period), and some time
# after that the validity period changes and Synapse is restarted, the users'
# expiration dates won't be updated unless their account is manually renewed.
#
#account_validity:
# enabled: True
# period: 6w

8
synapse/events/__init__.py
View file

@ -21,6 +21,7 @@ import six
from unpaddedbase64 import encode_base64
from synapse.api.errors import UnsupportedRoomVersionError
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, EventFormatVersions
from synapse.util.caches import intern_dict
from synapse.util.frozenutils import freeze
@ -369,12 +370,15 @@ def room_version_to_event_format(room_version):
Returns:
int
Raises:
UnsupportedRoomVersionError if the room version is unknown
"""
v = KNOWN_ROOM_VERSIONS.get(room_version)
if not v:
# We should have already checked version, so this should not happen
raise RuntimeError("Unrecognized room version %s" % (room_version,))
# this can happen if support is withdrawn for a room version
raise UnsupportedRoomVersionError()
return v.event_format

6
synapse/events/builder.py
View file

@ -18,6 +18,7 @@ import attr
from twisted.internet import defer
from synapse.api.constants import MAX_DEPTH
from synapse.api.errors import UnsupportedRoomVersionError
from synapse.api.room_versions import (
KNOWN_EVENT_FORMAT_VERSIONS,
KNOWN_ROOM_VERSIONS,
@ -178,9 +179,8 @@ class EventBuilderFactory(object):
"""
v = KNOWN_ROOM_VERSIONS.get(room_version)
if not v:
raise Exception(
"No event format defined for version %r" % (room_version,)
)
# this can happen if support is withdrawn for a room version
raise UnsupportedRoomVersionError()
return self.for_room_version(v, key_values)
def for_room_version(self, room_version, key_values):

14
synapse/federation/federation_server.py
View file

@ -33,6 +33,7 @@ from synapse.api.errors import (
IncompatibleRoomVersionError,
NotFoundError,
SynapseError,
UnsupportedRoomVersionError,
)
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
from synapse.crypto.event_signing import compute_event_signature
@ -198,11 +199,22 @@ class FederationServer(FederationBase):
try:
room_version = yield self.store.get_room_version(room_id)
format_ver = room_version_to_event_format(room_version)
except NotFoundError:
logger.info("Ignoring PDU for unknown room_id: %s", room_id)
continue
try:
format_ver = room_version_to_event_format(room_version)
except UnsupportedRoomVersionError:
# this can happen if support for a given room version is withdrawn,
# so that we still get events for said room.
logger.info(
"Ignoring PDU for room %s with unknown version %s",
room_id,
room_version,
)
continue
event = event_from_pdu_json(p, format_ver)
pdus_by_room.setdefault(room_id, []).append(event)

7
synapse/handlers/federation.py
View file

@ -1916,6 +1916,11 @@ class FederationHandler(BaseHandler):
event.room_id, latest_event_ids=extrem_ids,
)
logger.debug(
"Doing soft-fail check for %s: state %s",
event.event_id, current_state_ids,
)
# Now check if event pass auth against said current state
auth_types = auth_types_for_event(event)
current_state_ids = [
@ -1932,7 +1937,7 @@ class FederationHandler(BaseHandler):
self.auth.check(room_version, event, auth_events=current_auth_events)
except AuthError as e:
logger.warn(
"Failed current state auth resolution for %r because %s",
"Soft-failing %r because %s",
event, e,
)
event.internal_metadata.soft_failed = True

40
synapse/python_dependencies.py
View file

@ -16,7 +16,12 @@
import logging
from pkg_resources import DistributionNotFound, VersionConflict, get_distribution
from pkg_resources import (
DistributionNotFound,
Requirement,
VersionConflict,
get_provider,
)
logger = logging.getLogger(__name__)
@ -91,7 +96,13 @@ CONDITIONAL_REQUIREMENTS = {
# ACME support is required to provision TLS certificates from authorities
# that use the protocol, such as Let's Encrypt.
"acme": ["txacme>=0.9.2"],
"acme": [
"txacme>=0.9.2",
# txacme depends on eliot. Eliot 1.8.0 is incompatible with
# python 3.5.2, as per https://github.com/itamarst/eliot/issues/418
'eliot<1.8.0;python_version<"3.5.3"',
],
"saml2": ["pysaml2>=4.5.0"],
"systemd": ["systemd-python>=231"],
@ -125,10 +136,10 @@ class DependencyException(Exception):
@property
def dependencies(self):
for i in self.args[0]:
yield '"' + i + '"'
yield "'" + i + "'"
def check_requirements(for_feature=None, _get_distribution=get_distribution):
def check_requirements(for_feature=None):
deps_needed = []
errors = []
@ -139,7 +150,7 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution):
for dependency in reqs:
try:
_get_distribution(dependency)
_check_requirement(dependency)
except VersionConflict as e:
deps_needed.append(dependency)
errors.append(
@ -157,7 +168,7 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution):
for dependency in OPTS:
try:
_get_distribution(dependency)
_check_requirement(dependency)
except VersionConflict as e:
deps_needed.append(dependency)
errors.append(
@ -175,6 +186,23 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution):
raise DependencyException(deps_needed)
def _check_requirement(dependency_string):
"""Parses a dependency string, and checks if the specified requirement is installed
Raises:
VersionConflict if the requirement is installed, but with the the wrong version
DistributionNotFound if nothing is found to provide the requirement
"""
req = Requirement.parse(dependency_string)
# first check if the markers specify that this requirement needs installing
if req.marker is not None and not req.marker.evaluate():
# not required for this environment
return
get_provider(req)
if __name__ == "__main__":
import sys

58
synapse/storage/_base.py
View file

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2017-2018 New Vector Ltd
# Copyright 2019 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -227,6 +229,8 @@ class SQLBaseStore(object):
# A set of tables that are not safe to use native upserts in.
self._unsafe_to_upsert_tables = set(UNIQUE_INDEX_BACKGROUND_UPDATES.keys())
self._account_validity = self.hs.config.account_validity
# We add the user_directory_search table to the blacklist on SQLite
# because the existing search table does not have an index, making it
# unsafe to use native upserts.
@ -243,6 +247,14 @@ class SQLBaseStore(object):
self._check_safe_to_upsert,
)
if self._account_validity.enabled:
self._clock.call_later(
0.0,
run_as_background_process,
"account_validity_set_expiration_dates",
self._set_expiration_date_when_missing,
)
@defer.inlineCallbacks
def _check_safe_to_upsert(self):
"""
@ -275,6 +287,52 @@ class SQLBaseStore(object):
self._check_safe_to_upsert,
)
@defer.inlineCallbacks
def _set_expiration_date_when_missing(self):
"""
Retrieves the list of registered users that don't have an expiration date, and
adds an expiration date for each of them.
"""
def select_users_with_no_expiration_date_txn(txn):
"""Retrieves the list of registered users with no expiration date from the
database.
"""
sql = (
"SELECT users.name FROM users"
" LEFT JOIN account_validity ON (users.name = account_validity.user_id)"
" WHERE account_validity.user_id is NULL;"
)
txn.execute(sql, [])
res = self.cursor_to_dict(txn)
if res:
for user in res:
self.set_expiration_date_for_user_txn(txn, user["name"])
yield self.runInteraction(
"get_users_with_no_expiration_date",
select_users_with_no_expiration_date_txn,
)
def set_expiration_date_for_user_txn(self, txn, user_id):
"""Sets an expiration date to the account with the given user ID.
Args:
user_id (str): User ID to set an expiration date for.
"""
now_ms = self._clock.time_msec()
expiration_ts = now_ms + self._account_validity.period
self._simple_insert_txn(
txn,
"account_validity",
values={
"user_id": user_id,
"expiration_ts_ms": expiration_ts,
"email_sent": False,
},
)
def start_profiling(self):
self._previous_loop_ts = self._clock.time_msec()

9
synapse/storage/events.py
View file

@ -575,10 +575,11 @@ class EventsStore(
def _get_events(txn, batch):
sql = """
SELECT prev_event_id
SELECT prev_event_id, internal_metadata
FROM event_edges
INNER JOIN events USING (event_id)
LEFT JOIN rejections USING (event_id)
LEFT JOIN event_json USING (event_id)
WHERE
prev_event_id IN (%s)
AND NOT events.outlier
@ -588,7 +589,11 @@ class EventsStore(
)
txn.execute(sql, batch)
results.extend(r[0] for r in txn)
results.extend(
r[0]
for r in txn
if not json.loads(r[1]).get("soft_failed")
)
for chunk in batch_iter(event_ids, 100):
yield self.runInteraction("_get_events_which_are_prevs", _get_events, chunk)

16
synapse/storage/registration.py
View file

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright 2014 - 2016 OpenMarket Ltd
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2017-2018 New Vector Ltd
# Copyright 2019 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -725,17 +727,7 @@ class RegistrationStore(
raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE)
if self._account_validity.enabled:
now_ms = self.clock.time_msec()
expiration_ts = now_ms + self._account_validity.period
self._simple_insert_txn(
txn,
"account_validity",
values={
"user_id": user_id,
"expiration_ts_ms": expiration_ts,
"email_sent": False,
}
)
self.set_expiration_date_for_user_txn(txn, user_id)
if token:
# it's possible for this to get a conflict, but only for a single user

55
tests/rest/client/v2_alpha/test_register.py
View file

@ -1,3 +1,20 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2017-2018 New Vector Ltd
# Copyright 2019 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import json
import os
@ -409,3 +426,41 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
self.assertEquals(channel.result["code"], b"200", channel.result)
self.assertEqual(len(self.email_attempts), 1)
class AccountValidityBackgroundJobTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
]
def make_homeserver(self, reactor, clock):
self.validity_period = 10
config = self.default_config()
config["enable_registration"] = True
config["account_validity"] = {
"enabled": False,
}
self.hs = self.setup_test_homeserver(config=config)
self.hs.config.account_validity.period = self.validity_period
self.store = self.hs.get_datastore()
return self.hs
def test_background_job(self):
"""
Tests whether the account validity startup background job does the right thing,
which is sticking an expiration date to every account that doesn't already have
one.
"""
user_id = self.register_user("kermit", "user")
now_ms = self.hs.clock.time_msec()
self.get_success(self.store._set_expiration_date_when_missing())
res = self.get_success(self.store.get_expiration_ts_for_user(user_id))
self.assertEqual(res, now_ms + self.validity_period)

2
tox.ini
View file

@ -94,7 +94,7 @@ commands =
# Make all greater-thans equals so we test the oldest version of our direct
# dependencies, but make the pyopenssl 17.0, which can work against an
# OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
/bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "s/psycopg2==2.6//" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs pip install'
/bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "s/psycopg2==2.6//" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs -d"\n" pip install'
# Add this so that coverage will run on subprocesses
/bin/sh -c 'echo "import coverage; coverage.process_startup()" > {envsitepackagesdir}/../sitecustomize.py'