maxmustermann/synapse
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into matrix-org-hotfixes

Browse source
This commit is contained in:
Richard van der Hoff 2021-06-07 16:57:58 +01:00
parent f36457dce2 beb251e3ee
commit 68d7704d19
17 changed files with 88 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/10105.misc Normal file
View file

@ -0,0 +1 @@
Add missing type hints to the admin API servlets.

1
changelog.d/10107.bugfix Normal file
View file

@ -0,0 +1 @@
Fixed a bug that could cause Synapse to stop notifying application services. Contributed by Willem Mulder.

1
changelog.d/10116.bugfix Normal file
View file

@ -0,0 +1 @@
Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel.

1
changelog.d/10124.misc Normal file
View file

@ -0,0 +1 @@
Work to improve the responsiveness of `/sync` requests.

1
changelog.d/10130.doc Normal file
View file

@ -0,0 +1 @@
Make a link in docs use HTTPS. Contributed by @RhnSharma.

1
changelog.d/10132.doc Normal file
View file

@ -0,0 +1 @@
Fix broken link in Docker docs.

2
docker/README.md
View file

@ -226,4 +226,4 @@ healthcheck:
## Using jemalloc
Jemalloc is embedded in the image and will be used instead of the default allocator.
You can read about jemalloc by reading the Synapse [README](../README.md).
You can read about jemalloc by reading the Synapse [README](../README.rst).

2
docs/turn-howto.md
View file

@ -4,7 +4,7 @@ This document explains how to enable VoIP relaying on your Home Server with
TURN.
The synapse Matrix Home Server supports integration with TURN server via the
[TURN server REST API](<http://tools.ietf.org/html/draft-uberti-behave-turn-rest-00>). This
[TURN server REST API](<https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00>). This
allows the Home Server to generate credentials that are valid for use on the
TURN server through the use of a secret shared between the Home Server and the
TURN server.

25
synapse/handlers/appservice.py
View file

@ -87,7 +87,8 @@ class ApplicationServicesHandler:
self.is_processing = True
try:
limit = 100
while True:
upper_bound = -1
while upper_bound < self.current_max:
(
upper_bound,
events,
@ -95,9 +96,6 @@ class ApplicationServicesHandler:
self.current_max, limit
)
if not events:
break
events_by_room = {} # type: Dict[str, List[EventBase]]
for event in events:
events_by_room.setdefault(event.room_id, []).append(event)
@ -153,9 +151,6 @@ class ApplicationServicesHandler:
await self.store.set_appservice_last_pos(upper_bound)
now = self.clock.time_msec()
ts = await self.store.get_received_ts(events[-1].event_id)
synapse.metrics.event_processing_positions.labels(
"appservice_sender"
).set(upper_bound)
@ -168,12 +163,16 @@ class ApplicationServicesHandler:
event_processing_loop_counter.labels("appservice_sender").inc()
synapse.metrics.event_processing_lag.labels(
"appservice_sender"
).set(now - ts)
synapse.metrics.event_processing_last_ts.labels(
"appservice_sender"
).set(ts)
if events:
now = self.clock.time_msec()
ts = await self.store.get_received_ts(events[-1].event_id)
synapse.metrics.event_processing_lag.labels(
"appservice_sender"
).set(now - ts)
synapse.metrics.event_processing_last_ts.labels(
"appservice_sender"
).set(ts)
finally:
self.is_processing = False

8
synapse/handlers/federation.py
View file

@ -178,6 +178,8 @@ class FederationHandler(BaseHandler):
self.room_queues = {} # type: Dict[str, List[Tuple[EventBase, str]]]
self._room_pdu_linearizer = Linearizer("fed_room_pdu")
self._room_backfill = Linearizer("room_backfill")
self.third_party_event_rules = hs.get_third_party_event_rules()
self._ephemeral_messages_enabled = hs.config.enable_ephemeral_messages
@ -1041,6 +1043,12 @@ class FederationHandler(BaseHandler):
return. This is used as part of the heuristic to decide if we
should back paginate.
"""
with (await self._room_backfill.queue(room_id)):
return await self._maybe_backfill_inner(room_id, current_depth, limit)
async def _maybe_backfill_inner(
self, room_id: str, current_depth: int, limit: int
) -> bool:
extremities = await self.store.get_oldest_events_with_depth_in_room(room_id)
if not extremities:

21
synapse/handlers/sync.py
View file

@ -316,6 +316,17 @@ class SyncHandler:
if context:
context.tag = sync_type
# if we have a since token, delete any to-device messages before that token
# (since we now know that the device has received them)
if since_token is not None:
since_stream_id = since_token.to_device_key
deleted = await self.store.delete_messages_for_device(
sync_config.user.to_string(), sync_config.device_id, since_stream_id
)
logger.debug(
"Deleted %d to-device messages up to %d", deleted, since_stream_id
)
if timeout == 0 or since_token is None or full_state:
# we are going to return immediately, so don't bother calling
# notifier.wait_for_events.
@ -1231,16 +1242,6 @@ class SyncHandler:
since_stream_id = int(sync_result_builder.since_token.to_device_key)
if since_stream_id != int(now_token.to_device_key):
# We only delete messages when a new message comes in, but that's
# fine so long as we delete them at some point.
deleted = await self.store.delete_messages_for_device(
user_id, device_id, since_stream_id
)
logger.debug(
"Deleted %d to-device messages up to %d", deleted, since_stream_id
)
messages, stream_id = await self.store.get_new_messages_for_device(
user_id, device_id, since_stream_id, now_token.to_device_key
)

45
synapse/rest/admin/__init__.py
View file

@ -17,11 +17,13 @@
import logging
import platform
from typing import TYPE_CHECKING, Optional, Tuple
import synapse
from synapse.api.errors import Codes, NotFoundError, SynapseError
from synapse.http.server import JsonResource
from synapse.http.server import HttpServer, JsonResource
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.http.site import SynapseRequest
from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin
from synapse.rest.admin.devices import (
DeleteDevicesRestServlet,
@ -66,22 +68,25 @@ from synapse.rest.admin.users import (
UserTokenRestServlet,
WhoisRestServlet,
)
from synapse.types import RoomStreamToken
from synapse.types import JsonDict, RoomStreamToken
from synapse.util.versionstring import get_version_string
if TYPE_CHECKING:
from synapse.server import HomeServer
logger = logging.getLogger(__name__)
class VersionServlet(RestServlet):
PATTERNS = admin_patterns("/server_version$")
def __init__(self, hs):
def __init__(self, hs: "HomeServer"):
self.res = {
"server_version": get_version_string(synapse),
"python_version": platform.python_version(),
}
def on_GET(self, request):
def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
return 200, self.res
@ -90,17 +95,14 @@ class PurgeHistoryRestServlet(RestServlet):
"/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?"
)
def __init__(self, hs):
"""
Args:
hs (synapse.server.HomeServer)
"""
def __init__(self, hs: "HomeServer"):
self.pagination_handler = hs.get_pagination_handler()
self.store = hs.get_datastore()
self.auth = hs.get_auth()
async def on_POST(self, request, room_id, event_id):
async def on_POST(
self, request: SynapseRequest, room_id: str, event_id: Optional[str]
) -> Tuple[int, JsonDict]:
await assert_requester_is_admin(self.auth, request)
body = parse_json_object_from_request(request, allow_empty_body=True)
@ -119,6 +121,8 @@ class PurgeHistoryRestServlet(RestServlet):
if event.room_id != room_id:
raise SynapseError(400, "Event is for wrong room.")
# RoomStreamToken expects [int] not Optional[int]
assert event.internal_metadata.stream_ordering is not None
room_token = RoomStreamToken(
event.depth, event.internal_metadata.stream_ordering
)
@ -173,16 +177,13 @@ class PurgeHistoryRestServlet(RestServlet):
class PurgeHistoryStatusRestServlet(RestServlet):
PATTERNS = admin_patterns("/purge_history_status/(?P<purge_id>[^/]+)")
def __init__(self, hs):
"""
Args:
hs (synapse.server.HomeServer)
"""
def __init__(self, hs: "HomeServer"):
self.pagination_handler = hs.get_pagination_handler()
self.auth = hs.get_auth()
async def on_GET(self, request, purge_id):
async def on_GET(
self, request: SynapseRequest, purge_id: str
) -> Tuple[int, JsonDict]:
await assert_requester_is_admin(self.auth, request)
purge_status = self.pagination_handler.get_purge_status(purge_id)
@ -203,12 +204,12 @@ class PurgeHistoryStatusRestServlet(RestServlet):
class AdminRestResource(JsonResource):
"""The REST resource which gets mounted at /_synapse/admin"""
def __init__(self, hs):
def __init__(self, hs: "HomeServer"):
JsonResource.__init__(self, hs, canonical_json=False)
register_servlets(hs, self)
def register_servlets(hs, http_server):
def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
"""
Register all the admin servlets.
"""
@ -242,7 +243,9 @@ def register_servlets(hs, http_server):
RateLimitRestServlet(hs).register(http_server)
def register_servlets_for_client_rest_resource(hs, http_server):
def register_servlets_for_client_rest_resource(
hs: "HomeServer", http_server: HttpServer
) -> None:
"""Register only the servlets which need to be exposed on /_matrix/client/xxx"""
WhoisRestServlet(hs).register(http_server)
PurgeHistoryStatusRestServlet(hs).register(http_server)

3
synapse/rest/admin/_base.py
View file

@ -13,6 +13,7 @@
# limitations under the License.
import re
from typing import Iterable, Pattern
from synapse.api.auth import Auth
from synapse.api.errors import AuthError
@ -20,7 +21,7 @@ from synapse.http.site import SynapseRequest
from synapse.types import UserID
def admin_patterns(path_regex: str, version: str = "v1"):
def admin_patterns(path_regex: str, version: str = "v1") -> Iterable[Pattern]:
"""Returns the list of patterns for an admin endpoint
Args:

12
synapse/rest/admin/groups.py
View file

@ -12,10 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import TYPE_CHECKING, Tuple
from synapse.api.errors import SynapseError
from synapse.http.servlet import RestServlet
from synapse.http.site import SynapseRequest
from synapse.rest.admin._base import admin_patterns, assert_user_is_admin
from synapse.types import JsonDict
if TYPE_CHECKING:
from synapse.server import HomeServer
logger = logging.getLogger(__name__)
@ -25,12 +31,14 @@ class DeleteGroupAdminRestServlet(RestServlet):
PATTERNS = admin_patterns("/delete_group/(?P<group_id>[^/]*)")
def __init__(self, hs):
def __init__(self, hs: "HomeServer"):
self.group_server = hs.get_groups_server_handler()
self.is_mine_id = hs.is_mine_id
self.auth = hs.get_auth()
async def on_POST(self, request, group_id):
async def on_POST(
self, request: SynapseRequest, group_id: str
) -> Tuple[int, JsonDict]:
requester = await self.auth.get_user_by_req(request)
await assert_user_is_admin(self.auth, requester.user)

12
synapse/rest/admin/media.py
View file

@ -17,6 +17,7 @@ import logging
from typing import TYPE_CHECKING, Tuple
from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
from synapse.http.server import HttpServer
from synapse.http.servlet import RestServlet, parse_boolean, parse_integer
from synapse.http.site import SynapseRequest
from synapse.rest.admin._base import (
@ -37,12 +38,11 @@ class QuarantineMediaInRoom(RestServlet):
this server.
"""
PATTERNS = (
admin_patterns("/room/(?P<room_id>[^/]+)/media/quarantine")
+
PATTERNS = [
*admin_patterns("/room/(?P<room_id>[^/]+)/media/quarantine"),
# This path kept around for legacy reasons
admin_patterns("/quarantine_media/(?P<room_id>[^/]+)")
)
*admin_patterns("/quarantine_media/(?P<room_id>[^/]+)"),
]
def __init__(self, hs: "HomeServer"):
self.store = hs.get_datastore()
@ -312,7 +312,7 @@ class DeleteMediaByDateSize(RestServlet):
return 200, {"deleted_media": deleted_media, "total": total}
def register_servlets_for_media_repo(hs: "HomeServer", http_server):
def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer) -> None:
"""
Media repo specific APIs.
"""

15
synapse/rest/admin/users.py
View file

@ -478,13 +478,12 @@ class UserRegisterServlet(RestServlet):
class WhoisRestServlet(RestServlet):
path_regex = "/whois/(?P<user_id>[^/]*)$"
PATTERNS = (
admin_patterns(path_regex)
+
PATTERNS = [
*admin_patterns(path_regex),
# URL for spec reason
# https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid
client_patterns("/admin" + path_regex, v1=True)
)
*client_patterns("/admin" + path_regex, v1=True),
]
def __init__(self, hs: "HomeServer"):
self.hs = hs
@ -553,11 +552,7 @@ class DeactivateAccountRestServlet(RestServlet):
class AccountValidityRenewServlet(RestServlet):
PATTERNS = admin_patterns("/account_validity/validity$")
def __init__(self, hs):
"""
Args:
hs (synapse.server.HomeServer): server
"""
def __init__(self, hs: "HomeServer"):
self.hs = hs
self.account_activity_handler = hs.get_account_validity_handler()
self.auth = hs.get_auth()

6
tests/handlers/test_appservice.py
View file

@ -57,10 +57,10 @@ class AppServiceHandlerTestCase(unittest.TestCase):
sender="@someone:anywhere", type="m.room.message", room_id="!foo:bar"
)
self.mock_store.get_new_events_for_appservice.side_effect = [
make_awaitable((0, [event])),
make_awaitable((0, [])),
make_awaitable((1, [event])),
]
self.handler.notify_interested_services(RoomStreamToken(None, 0))
self.handler.notify_interested_services(RoomStreamToken(None, 1))
self.mock_scheduler.submit_event_for_as.assert_called_once_with(
interested_service, event
@ -77,7 +77,6 @@ class AppServiceHandlerTestCase(unittest.TestCase):
self.mock_as_api.query_user.return_value = make_awaitable(True)
self.mock_store.get_new_events_for_appservice.side_effect = [
make_awaitable((0, [event])),
make_awaitable((0, [])),
]
self.handler.notify_interested_services(RoomStreamToken(None, 0))
@ -95,7 +94,6 @@ class AppServiceHandlerTestCase(unittest.TestCase):
self.mock_as_api.query_user.return_value = make_awaitable(True)
self.mock_store.get_new_events_for_appservice.side_effect = [
make_awaitable((0, [event])),
make_awaitable((0, [])),
]
self.handler.notify_interested_services(RoomStreamToken(None, 0))