From 21f135ba763a583ecf9ba2714b5151f6b14b61fd Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Dec 2015 16:26:08 +0000 Subject: [PATCH 001/294] Very first cut of calculating actions for events as they come in. Doesn't store them yet. Not very efficient. --- synapse/handlers/_base.py | 8 ++++++ synapse/handlers/federation.py | 15 +++++++++- synapse/push/action_generator.py | 47 ++++++++++++++++++++++++++++++++ synapse/storage/registration.py | 12 ++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 synapse/push/action_generator.py diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 5fd20285d..a8e8c4f5a 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -19,9 +19,12 @@ from synapse.api.errors import LimitExceededError, SynapseError, AuthError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes from synapse.types import UserID, RoomAlias +from synapse.push.action_generator import ActionGenerator from synapse.util.logcontext import PreserveLoggingContext +from synapse.events.utils import serialize_event + import logging @@ -264,6 +267,11 @@ class BaseHandler(object): event, context=context ) + action_generator = ActionGenerator(self.store) + yield action_generator.handle_event(serialize_event( + event, self.clock.time_msec() + )) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2855f2d7c..18289eb52 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -32,10 +32,12 @@ from synapse.crypto.event_signing import ( ) from synapse.types import UserID -from synapse.events.utils import prune_event +from synapse.events.utils import prune_event, serialize_event from synapse.util.retryutils import NotRetryingDestination +from synapse.push.action_generator import ActionGenerator + from twisted.internet import defer import itertools @@ -1113,6 +1115,11 @@ class FederationHandler(BaseHandler): current_state=current_state, ) + action_generator = ActionGenerator(self.store) + yield action_generator.handle_event(serialize_event( + event, self.clock.time_msec()) + ) + defer.returnValue((context, event_stream_id, max_stream_id)) @defer.inlineCallbacks @@ -1139,6 +1146,12 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) + for ev_info in event_infos: + action_generator = ActionGenerator(self.store) + yield action_generator.handle_event(serialize_event( + ev_info["event"], self.clock.time_msec()) + ) + @defer.inlineCallbacks def _persist_auth_tree(self, auth_events, state, event): """Checks the auth chain is valid (and passes auth checks) for the diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py new file mode 100644 index 000000000..508eeaed9 --- /dev/null +++ b/synapse/push/action_generator.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +import push_rule_evaluator + +import logging + + +logger = logging.getLogger(__name__) + + +class ActionGenerator: + def __init__(self, store): + self.store = store + # really we want to get all user ids and all profile tags too, + # since we want the actions for each profile tag for every user and + # also actions for a client with no profile tag for each user. + # Currently the event stream doesn't support profile tags on an + # event stream, so we just run the rules for a client with no profile + # tag (ie. we just need all the users). + + @defer.inlineCallbacks + def handle_event(self, event): + users = yield self.store.get_users_in_room(event['room_id']) + logger.error("users in room: %r", users) + + for uid in users: + evaluator = yield push_rule_evaluator.\ + evaluator_for_user_name_and_profile_tag( + uid, None, event['room_id'], self.store + ) + actions = yield evaluator.actions_for_event(event) + logger.info("actions for user %s: %s", uid, actions) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 2e5eddd25..f230faa25 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -291,6 +291,18 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(ret['user_id']) defer.returnValue(None) + @defer.inlineCallbacks + def get_all_user_ids(self): + """Returns all user ids registered on this homeserver""" + return self.runInteraction( + "get_all_user_ids", + self._get_all_user_ids_txn + ) + + def _get_all_user_ids_txn(self, txn): + txn.execute("SELECT name from users") + return [r[0] for r in txn.fetchall()] + @defer.inlineCallbacks def count_all_users(self): """Counts all users registered on the homeserver.""" From a84a6933274c0da64c41ac494b027ab5010a4801 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Dec 2015 17:18:46 +0000 Subject: [PATCH 002/294] Having consulted The Erikle, this should go at the end of on_receive_pdu, otherwise it will be triggered whenever we backfill too. --- synapse/handlers/federation.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 18289eb52..6a4269b50 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -244,6 +244,12 @@ class FederationHandler(BaseHandler): user = UserID.from_string(event.state_key) yield user_joined_room(self.distributor, user, event.room_id) + if not backfilled and not event.internal_metadata.is_outlier(): + action_generator = ActionGenerator(self.store) + yield action_generator.handle_event(serialize_event( + event, self.clock.time_msec()) + ) + @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): event_to_state = yield self.store.get_state_for_events( @@ -1115,11 +1121,6 @@ class FederationHandler(BaseHandler): current_state=current_state, ) - action_generator = ActionGenerator(self.store) - yield action_generator.handle_event(serialize_event( - event, self.clock.time_msec()) - ) - defer.returnValue((context, event_stream_id, max_stream_id)) @defer.inlineCallbacks @@ -1146,12 +1147,6 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) - for ev_info in event_infos: - action_generator = ActionGenerator(self.store) - yield action_generator.handle_event(serialize_event( - ev_info["event"], self.clock.time_msec()) - ) - @defer.inlineCallbacks def _persist_auth_tree(self, auth_events, state, event): """Checks the auth chain is valid (and passes auth checks) for the From aa667ee396c473f497b084655d47b2a9520a538a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Dec 2015 17:51:15 +0000 Subject: [PATCH 003/294] Save event actions to the db --- synapse/push/action_generator.py | 6 ++- synapse/storage/__init__.py | 2 + synapse/storage/event_actions.py | 41 +++++++++++++++++++ .../storage/schema/delta/27/event_actions.sql | 25 +++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 synapse/storage/event_actions.py create mode 100644 synapse/storage/schema/delta/27/event_actions.sql diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 508eeaed9..870c68a0c 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -19,7 +19,6 @@ import push_rule_evaluator import logging - logger = logging.getLogger(__name__) @@ -42,6 +41,9 @@ class ActionGenerator: evaluator = yield push_rule_evaluator.\ evaluator_for_user_name_and_profile_tag( uid, None, event['room_id'], self.store - ) + ) actions = yield evaluator.actions_for_event(event) logger.info("actions for user %s: %s", uid, actions) + self.store.set_actions_for_event( + event['event_id'], uid, None, actions + ) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c46b653f1..a112dd237 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -33,6 +33,7 @@ from .pusher import PusherStore from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore from .rejections import RejectionsStore +from .event_actions import EventActionsStore from .state import StateStore from .signatures import SignatureStore @@ -75,6 +76,7 @@ class DataStore(RoomMemberStore, RoomStore, SearchStore, TagsStore, AccountDataStore, + EventActionsStore ): def __init__(self, hs): diff --git a/synapse/storage/event_actions.py b/synapse/storage/event_actions.py new file mode 100644 index 000000000..593b1714c --- /dev/null +++ b/synapse/storage/event_actions.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore +from twisted.internet import defer + +import logging +import simplejson as json + +logger = logging.getLogger(__name__) + + +class EventActionsStore(SQLBaseStore): + @defer.inlineCallbacks + def set_actions_for_event(self, event_id, user_id, profile_tag, actions): + actionsJson = json.dumps(actions) + + ret = yield self.runInteraction( + "_set_actions_for_event", + self._simple_upsert_txn, + EventActionsTable.table_name, + {'event_id': event_id, 'user_id': user_id, 'profile_tag': profile_tag}, + {'actions': actionsJson} + ) + defer.returnValue(ret) + + +class EventActionsTable(object): + table_name = "event_actions" diff --git a/synapse/storage/schema/delta/27/event_actions.sql b/synapse/storage/schema/delta/27/event_actions.sql new file mode 100644 index 000000000..1246823a0 --- /dev/null +++ b/synapse/storage/schema/delta/27/event_actions.sql @@ -0,0 +1,25 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_actions( + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + profile_tag VARCHAR(32), + actions TEXT NOT NULL, + CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (event_id, user_id, profile_tag) +); + + +CREATE INDEX event_actions_event_id_user_id_profile_tag on event_actions(event_id, user_id, profile_tag); From 5e909c73d76cd56e3eac91635a99405182dcac3c Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Dec 2015 18:40:28 +0000 Subject: [PATCH 004/294] Store nothing instead of ['dont_notify'] for events with no notification required: much as it would be nice to be able to tell between the event not having been processed and there being no notification for it, this isn't worth filling up the table with ['dont_notify'] I think. Consequently treat the empty actions array as dont_notify and filter dont_notify out of the result. --- synapse/push/__init__.py | 18 +++--------------- synapse/push/action_generator.py | 8 ++++---- synapse/push/push_rule_evaluator.py | 9 +++++++-- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index e7c964bcd..d5928c1d2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -157,21 +157,7 @@ class Pusher(object): actions = yield rule_evaluator.actions_for_event(single_event) tweaks = rule_evaluator.tweaks_for_actions(actions) - if len(actions) == 0: - logger.warn("Empty actions! Using default action.") - actions = Pusher.DEFAULT_ACTIONS - - if 'notify' not in actions and 'dont_notify' not in actions: - logger.warn("Neither notify nor dont_notify in actions: adding default") - actions.extend(Pusher.DEFAULT_ACTIONS) - - if 'dont_notify' in actions: - logger.debug( - "%s for %s: dont_notify", - single_event['event_id'], self.user_name - ) - processed = True - else: + if 'notify' in actions: rejected = yield self.dispatch_push(single_event, tweaks) self.has_unread = True if isinstance(rejected, list) or isinstance(rejected, tuple): @@ -192,6 +178,8 @@ class Pusher(object): yield self.hs.get_pusherpool().remove_pusher( self.app_id, pk, self.user_name ) + else: + processed = True if not self.alive: return diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 870c68a0c..a72a7d703 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -35,7 +35,6 @@ class ActionGenerator: @defer.inlineCallbacks def handle_event(self, event): users = yield self.store.get_users_in_room(event['room_id']) - logger.error("users in room: %r", users) for uid in users: evaluator = yield push_rule_evaluator.\ @@ -44,6 +43,7 @@ class ActionGenerator: ) actions = yield evaluator.actions_for_event(event) logger.info("actions for user %s: %s", uid, actions) - self.store.set_actions_for_event( - event['event_id'], uid, None, actions - ) + if len(actions): + self.store.set_actions_for_event( + event['event_id'], uid, None, actions + ) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 92c7fd048..420476fd0 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -43,7 +43,7 @@ def evaluator_for_user_name_and_profile_tag(user_name, profile_tag, room_id, sto class PushRuleEvaluator: - DEFAULT_ACTIONS = ['dont_notify'] + DEFAULT_ACTIONS = [] INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") def __init__(self, user_name, profile_tag, raw_rules, enabled_map, room_id, @@ -85,7 +85,7 @@ class PushRuleEvaluator: """ if ev['user_id'] == self.user_name: # let's assume you probably know about messages you sent yourself - defer.returnValue(['dont_notify']) + defer.returnValue([]) room_id = ev['room_id'] @@ -131,6 +131,11 @@ class PushRuleEvaluator: "%s matches for user %s, event %s", r['rule_id'], self.user_name, ev['event_id'] ) + + # filter out dont_notify as we treat an empty actions list + # as dont_notify, and this doesn't take up a row in our database + actions = [x for x in actions if x != 'dont_notify'] + defer.returnValue(actions) logger.info( From 42ad49f5b75c2c645c4060026c21c5572f5b1063 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 16 Dec 2015 18:42:09 +0000 Subject: [PATCH 005/294] still very WIP, but now sends unread_notifications_count in the room object on sync (only actually corrrect in a full sync: hardcoded to 0 in incremental syncs). --- synapse/handlers/sync.py | 26 +++++++++ synapse/push/action_generator.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 1 + synapse/storage/event_actions.py | 53 ++++++++++++++++++- .../storage/schema/delta/27/event_actions.sql | 5 +- 5 files changed, 82 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 24c2b2fad..6d193a10c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -52,6 +52,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ "state", # dict[(str, str), FrozenEvent] "ephemeral", "account_data", + "unread_notification_count", ])): __slots__ = [] @@ -64,6 +65,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ or self.state or self.ephemeral or self.account_data + or self.unread_notification_count > 0 ) @@ -161,6 +163,18 @@ class SyncHandler(BaseHandler): else: return self.incremental_sync_with_gap(sync_config, since_token) + def last_read_event_id_for_room_and_user(self, room_id, user_id, ephemeral_by_room): + if room_id not in ephemeral_by_room: + return None + for e in ephemeral_by_room[room_id]: + if e['type'] != 'm.receipt': + continue + for receipt_event_id,val in e['content'].items(): + if 'm.read' in val: + if user_id in val['m.read']: + return receipt_event_id + return None + @defer.inlineCallbacks def full_state_sync(self, sync_config, timeline_since_token): """Get a sync for a client which is starting without any state. @@ -265,6 +279,16 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, since_token=timeline_since_token ) + last_unread_event_id = self.last_read_event_id_for_room_and_user( + room_id, sync_config.user.to_string(), ephemeral_by_room + ) + + notifs = [] + if last_unread_event_id: + notifs = yield self.store.get_unread_event_actions_by_room( + room_id, last_unread_event_id + ) + current_state = yield self.get_state_at(room_id, now_token) defer.returnValue(JoinedSyncResult( @@ -275,6 +299,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), + unread_notification_count=len(notifs) )) def account_data_for_user(self, account_data): @@ -509,6 +534,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), + unread_notification_count=0 ) logger.debug("Result for room %s: %r", room_id, room_sync) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index a72a7d703..1c7cd3166 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -45,5 +45,5 @@ class ActionGenerator: logger.info("actions for user %s: %s", uid, actions) if len(actions): self.store.set_actions_for_event( - event['event_id'], uid, None, actions + event, uid, None, actions ) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index f0a637a6d..4ca10732c 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -304,6 +304,7 @@ class SyncRestServlet(RestServlet): }, "state": {"events": serialized_state}, "account_data": {"events": account_data}, + "unread_notification_count": room.unread_notification_count } if joined: diff --git a/synapse/storage/event_actions.py b/synapse/storage/event_actions.py index 593b1714c..40ac8e2d2 100644 --- a/synapse/storage/event_actions.py +++ b/synapse/storage/event_actions.py @@ -24,18 +24,67 @@ logger = logging.getLogger(__name__) class EventActionsStore(SQLBaseStore): @defer.inlineCallbacks - def set_actions_for_event(self, event_id, user_id, profile_tag, actions): + def set_actions_for_event(self, event, user_id, profile_tag, actions): actionsJson = json.dumps(actions) ret = yield self.runInteraction( "_set_actions_for_event", self._simple_upsert_txn, EventActionsTable.table_name, - {'event_id': event_id, 'user_id': user_id, 'profile_tag': profile_tag}, + { + 'room_id': event['room_id'], + 'event_id': event['event_id'], + 'user_id': user_id, + 'profile_tag': profile_tag + }, {'actions': actionsJson} ) defer.returnValue(ret) + @defer.inlineCallbacks + def get_unread_event_actions_by_room(self, room_id, last_read_event_id): + #events = yield self._get_events( + # [last_read_event_id], + # check_redacted=False + #) + + def _get_unread_event_actions_by_room(txn): + sql = ( + "SELECT stream_ordering, topological_ordering" + " FROM events" + " WHERE room_id = ? AND event_id = ?" + ) + txn.execute( + sql, (room_id, last_read_event_id) + ) + results = txn.fetchall() + if len(results) == 0: + return [] + + stream_ordering = results[0][0] + topological_ordering = results[0][1] + + sql = ( + "SELECT ea.actions" + " FROM event_actions ea, events e" + " WHERE ea.room_id = e.room_id" + " AND ea.event_id = e.event_id" + " AND ea.room_id = ?" + " AND (" + " e.topological_ordering > ?" + " OR (e.topological_ordering == ? AND e.stream_ordering > ?)" + ")" + ) + txn.execute(sql, + (room_id, topological_ordering, topological_ordering, stream_ordering) + ) + return txn.fetchall() + + ret = yield self.runInteraction( + "get_unread_event_actions_by_room", + _get_unread_event_actions_by_room + ) + defer.returnValue(ret) class EventActionsTable(object): table_name = "event_actions" diff --git a/synapse/storage/schema/delta/27/event_actions.sql b/synapse/storage/schema/delta/27/event_actions.sql index 1246823a0..bbdaee990 100644 --- a/synapse/storage/schema/delta/27/event_actions.sql +++ b/synapse/storage/schema/delta/27/event_actions.sql @@ -14,12 +14,13 @@ */ CREATE TABLE IF NOT EXISTS event_actions( + room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, profile_tag VARCHAR(32), actions TEXT NOT NULL, - CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (event_id, user_id, profile_tag) + CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag) ); -CREATE INDEX event_actions_event_id_user_id_profile_tag on event_actions(event_id, user_id, profile_tag); +CREATE INDEX event_actions_room_id_event_id_user_id_profile_tag on event_actions(room_id, event_id, user_id, profile_tag); From 413d0d6a2404c579b1fa39ece9a698f9df8349db Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Dec 2015 17:47:00 +0000 Subject: [PATCH 006/294] Make unread notification count sending work: put the correct count in incremental syncs too, where necessary, and fix silly bugs like only select the event actions for that user... --- synapse/handlers/sync.py | 48 ++++++++++++++++++++++++-------- synapse/storage/event_actions.py | 21 ++++++++------ 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6d193a10c..44420a063 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -65,7 +65,8 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ or self.state or self.ephemeral or self.account_data - or self.unread_notification_count > 0 + # nb the notification count does not, er, count: if there's nothing + # else in the result, we don't need to send it. ) @@ -279,15 +280,12 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, since_token=timeline_since_token ) - last_unread_event_id = self.last_read_event_id_for_room_and_user( - room_id, sync_config.user.to_string(), ephemeral_by_room + notifs = yield self.unread_notifs_for_room_id( + room_id, sync_config, ephemeral_by_room ) - - notifs = [] - if last_unread_event_id: - notifs = yield self.store.get_unread_event_actions_by_room( - room_id, last_unread_event_id - ) + notif_count = None + if notifs is not None: + notif_count = len(notifs) current_state = yield self.get_state_at(room_id, now_token) @@ -299,7 +297,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), - unread_notification_count=len(notifs) + unread_notification_count=notif_count )) def account_data_for_user(self, account_data): @@ -441,6 +439,10 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("presence_key", presence_key) + _, all_ephemeral_by_room = yield self.ephemeral_by_room( + sync_config, now_token + ) + now_token, ephemeral_by_room = yield self.ephemeral_by_room( sync_config, now_token, since_token ) @@ -514,6 +516,13 @@ class SyncHandler(BaseHandler): else: prev_batch = now_token + notifs = yield self.unread_notifs_for_room_id( + room_id, sync_config, all_ephemeral_by_room + ) + notif_count = None + if notifs is not None: + notif_count = len(notifs) + just_joined = yield self.check_joined_room(sync_config, state) if just_joined: logger.debug("User has just joined %s: needs full state", @@ -534,7 +543,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), - unread_notification_count=0 + unread_notification_count=notif_count ) logger.debug("Result for room %s: %r", room_id, room_sync) @@ -805,3 +814,20 @@ class SyncHandler(BaseHandler): if join_event.content["membership"] == Membership.JOIN: return True return False + + @defer.inlineCallbacks + def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): + last_unread_event_id = self.last_read_event_id_for_room_and_user( + room_id, sync_config.user.to_string(), ephemeral_by_room + ) + + notifs = [] + if last_unread_event_id: + notifs = yield self.store.get_unread_event_actions_by_room_for_user( + room_id, sync_config.user.to_string(), last_unread_event_id + ) + else: + # There is no new information in this period, so your notification + # count is whatever it was last time. + defer.returnValue(None) + defer.returnValue(notifs) \ No newline at end of file diff --git a/synapse/storage/event_actions.py b/synapse/storage/event_actions.py index 40ac8e2d2..f7fe78e55 100644 --- a/synapse/storage/event_actions.py +++ b/synapse/storage/event_actions.py @@ -42,12 +42,9 @@ class EventActionsStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def get_unread_event_actions_by_room(self, room_id, last_read_event_id): - #events = yield self._get_events( - # [last_read_event_id], - # check_redacted=False - #) - + def get_unread_event_actions_by_room_for_user( + self, room_id, user_id, last_read_event_id + ): def _get_unread_event_actions_by_room(txn): sql = ( "SELECT stream_ordering, topological_ordering" @@ -65,10 +62,11 @@ class EventActionsStore(SQLBaseStore): topological_ordering = results[0][1] sql = ( - "SELECT ea.actions" + "SELECT ea.event_id, ea.actions" " FROM event_actions ea, events e" " WHERE ea.room_id = e.room_id" " AND ea.event_id = e.event_id" + " AND ea.user_id = ?" " AND ea.room_id = ?" " AND (" " e.topological_ordering > ?" @@ -76,9 +74,14 @@ class EventActionsStore(SQLBaseStore): ")" ) txn.execute(sql, - (room_id, topological_ordering, topological_ordering, stream_ordering) + ( + user_id, room_id, + topological_ordering, topological_ordering, stream_ordering + ) ) - return txn.fetchall() + return [ + { "event_id": row[0], "actions": row[1] } for row in txn.fetchall() + ] ret = yield self.runInteraction( "get_unread_event_actions_by_room", From b131fb1fe26da50ce30656cadbb24e72bd7ecdf9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Dec 2015 18:04:45 +0000 Subject: [PATCH 007/294] add list of things I want to fix with this branch --- problems_with_this_branch | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 problems_with_this_branch diff --git a/problems_with_this_branch b/problems_with_this_branch new file mode 100644 index 000000000..9c5df5521 --- /dev/null +++ b/problems_with_this_branch @@ -0,0 +1,5 @@ + * processing push by bluntly running the rules for each user + * processing push twice because the pushers still run the rules themselves rather than pulling out of the event_actions table + * consequently converting events back & forth between objects & dicts when processing + * fetching all the current ephemeral events for each room on an incremental sync to get the user's read receipt + * doing 2 more db queries per room to get the unread notif count From 091c545c4fb38f662b61cb46779a813f70971e4f Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 21 Dec 2015 10:14:57 +0000 Subject: [PATCH 008/294] pep8 --- synapse/handlers/sync.py | 6 +++--- synapse/storage/event_actions.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 44420a063..20b2a2595 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -170,7 +170,7 @@ class SyncHandler(BaseHandler): for e in ephemeral_by_room[room_id]: if e['type'] != 'm.receipt': continue - for receipt_event_id,val in e['content'].items(): + for receipt_event_id, val in e['content'].items(): if 'm.read' in val: if user_id in val['m.read']: return receipt_event_id @@ -281,7 +281,7 @@ class SyncHandler(BaseHandler): ) notifs = yield self.unread_notifs_for_room_id( - room_id, sync_config, ephemeral_by_room + room_id, sync_config, ephemeral_by_room ) notif_count = None if notifs is not None: @@ -830,4 +830,4 @@ class SyncHandler(BaseHandler): # There is no new information in this period, so your notification # count is whatever it was last time. defer.returnValue(None) - defer.returnValue(notifs) \ No newline at end of file + defer.returnValue(notifs) diff --git a/synapse/storage/event_actions.py b/synapse/storage/event_actions.py index f7fe78e55..fbd0a4227 100644 --- a/synapse/storage/event_actions.py +++ b/synapse/storage/event_actions.py @@ -73,14 +73,13 @@ class EventActionsStore(SQLBaseStore): " OR (e.topological_ordering == ? AND e.stream_ordering > ?)" ")" ) - txn.execute(sql, - ( - user_id, room_id, - topological_ordering, topological_ordering, stream_ordering - ) + txn.execute(sql, ( + user_id, room_id, + topological_ordering, topological_ordering, stream_ordering + ) ) return [ - { "event_id": row[0], "actions": row[1] } for row in txn.fetchall() + {"event_id": row[0], "actions": row[1]} for row in txn.fetchall() ] ret = yield self.runInteraction( @@ -89,5 +88,6 @@ class EventActionsStore(SQLBaseStore): ) defer.returnValue(ret) + class EventActionsTable(object): table_name = "event_actions" From f73f154ec2c8ffdc49270d3ccaf3053f915800f3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 21 Dec 2015 15:28:54 +0000 Subject: [PATCH 009/294] Only run pushers for users on this hs! --- synapse/handlers/_base.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/push/action_generator.py | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index a8e8c4f5a..24c4c6269 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -267,7 +267,7 @@ class BaseHandler(object): event, context=context ) - action_generator = ActionGenerator(self.store) + action_generator = ActionGenerator(self.hs, self.store) yield action_generator.handle_event(serialize_event( event, self.clock.time_msec() )) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6a4269b50..6525bde43 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -245,7 +245,7 @@ class FederationHandler(BaseHandler): yield user_joined_room(self.distributor, user, event.room_id) if not backfilled and not event.internal_metadata.is_outlier(): - action_generator = ActionGenerator(self.store) + action_generator = ActionGenerator(self.hs, self.store) yield action_generator.handle_event(serialize_event( event, self.clock.time_msec()) ) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 1c7cd3166..6e107ca79 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from synapse.types import UserID + import push_rule_evaluator import logging @@ -23,7 +25,8 @@ logger = logging.getLogger(__name__) class ActionGenerator: - def __init__(self, store): + def __init__(self, hs, store): + self.hs = hs self.store = store # really we want to get all user ids and all profile tags too, # since we want the actions for each profile tag for every user and @@ -37,6 +40,9 @@ class ActionGenerator: users = yield self.store.get_users_in_room(event['room_id']) for uid in users: + if not self.hs.is_mine(UserID.from_string(uid)): + continue + evaluator = yield push_rule_evaluator.\ evaluator_for_user_name_and_profile_tag( uid, None, event['room_id'], self.store From 65c451cb3878fb41f28a2adecd638894e18f5343 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 15:19:34 +0000 Subject: [PATCH 010/294] Add bulk push rule evaluator which actually still evaluates rules one by one, but does far fewer db queries to fetch the rules --- synapse/push/action_generator.py | 26 +++---- synapse/push/bulk_push_rule_evaluator.py | 99 ++++++++++++++++++++++++ synapse/push/push_rule_evaluator.py | 13 ++-- synapse/storage/push_rule.py | 41 ++++++++++ 4 files changed, 158 insertions(+), 21 deletions(-) create mode 100644 synapse/push/bulk_push_rule_evaluator.py diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 6e107ca79..2ad5f82da 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -15,9 +15,7 @@ from twisted.internet import defer -from synapse.types import UserID - -import push_rule_evaluator +import bulk_push_rule_evaluator import logging @@ -39,17 +37,13 @@ class ActionGenerator: def handle_event(self, event): users = yield self.store.get_users_in_room(event['room_id']) - for uid in users: - if not self.hs.is_mine(UserID.from_string(uid)): - continue + bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( + event['room_id'], self.hs, self.store + ) - evaluator = yield push_rule_evaluator.\ - evaluator_for_user_name_and_profile_tag( - uid, None, event['room_id'], self.store - ) - actions = yield evaluator.actions_for_event(event) - logger.info("actions for user %s: %s", uid, actions) - if len(actions): - self.store.set_actions_for_event( - event, uid, None, actions - ) + actions_by_user = bulk_evaluator.action_for_event_by_user(event) + + for uid,actions in actions_by_user.items(): + self.store.set_actions_for_event( + event, uid, None, actions + ) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py new file mode 100644 index 000000000..f531d2edc --- /dev/null +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import simplejson as json + +from twisted.internet import defer + +from synapse.types import UserID + +import baserules +from push_rule_evaluator import PushRuleEvaluator + +logger = logging.getLogger(__name__) + + +def decode_rule_json(rule): + rule['conditions'] = json.loads(rule['conditions']) + rule['actions'] = json.loads(rule['actions']) + return rule + + +@defer.inlineCallbacks +def evaluator_for_room_id(room_id, hs, store): + users = yield store.get_users_in_room(room_id) + rules_by_user = yield store.bulk_get_push_rules(users) + rules_by_user = { + uid: baserules.list_with_base_rules( + [decode_rule_json(rule_list) for rule_list in rules_by_user[uid]] + if uid in rules_by_user else [], + UserID.from_string(uid) + ) + for uid in users + } + member_events = yield store.get_current_state( + room_id=room_id, + event_type='m.room.member', + ) + display_names = {} + for ev in member_events: + if ev.content.get("displayname"): + display_names[ev.state_key] = ev.content.get("displayname") + + defer.returnValue(BulkPushRuleEvaluator( + room_id, rules_by_user, display_names, users + )) + + +class BulkPushRuleEvaluator: + def __init__(self, room_id, rules_by_user, display_names, users_in_room): + self.room_id = room_id + self.rules_by_user = rules_by_user + self.display_names = display_names + self.users_in_room = users_in_room + + def action_for_event_by_user(self, event): + actions_by_user = {} + + for uid, rules in self.rules_by_user.items(): + display_name = None + if uid in self.display_names: + display_name = self.display_names[uid] + + for rule in rules: + if 'enabled' in rule and not rule['enabled']: + continue + + # XXX: profile tags + if BulkPushRuleEvaluator.event_matches_rule( + event, rule, + display_name, len(self.users_in_room), None + ): + actions = [x for x in rule['actions'] if x != 'dont_notify'] + if len(actions) > 0: + actions_by_user[uid] = actions + break + return actions_by_user + + @staticmethod + def event_matches_rule(event, rule, + display_name, room_member_count, profile_tag): + matches = True + for cond in rule['conditions']: + matches &= PushRuleEvaluator._event_fulfills_condition( + event, cond, display_name, room_member_count, profile_tag + ) + return matches \ No newline at end of file diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 420476fd0..40c7622ec 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -113,7 +113,8 @@ class PushRuleEvaluator: for c in conditions: matches &= self._event_fulfills_condition( ev, c, display_name=my_display_name, - room_member_count=room_member_count + room_member_count=room_member_count, + profile_tag=self.profile_tag ) logger.debug( "Rule %s %s", @@ -156,16 +157,18 @@ class PushRuleEvaluator: re.sub(r'\\\-', '-', x.group(2)))), r) return r - def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): + @staticmethod + def _event_fulfills_condition(ev, condition, + display_name, room_member_count, profile_tag): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") return False # XXX: optimisation: cache our pattern regexps if condition['key'] == 'content.body': - r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) + r = r'\b%s\b' % PushRuleEvaluator._glob_to_regexp(condition['pattern']) else: - r = r'^%s$' % self._glob_to_regexp(condition['pattern']) + r = r'^%s$' % PushRuleEvaluator._glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False @@ -174,7 +177,7 @@ class PushRuleEvaluator: elif condition['kind'] == 'device': if 'profile_tag' not in condition: return True - return condition['profile_tag'] == self.profile_tag + return condition['profile_tag'] == profile_tag elif condition['kind'] == 'contains_display_name': # This is special because display names can be different diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 5305b7e12..9dec4aa68 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -55,6 +55,47 @@ class PushRuleStore(SQLBaseStore): r['rule_id']: False if r['enabled'] == 0 else True for r in results }) + @defer.inlineCallbacks + def bulk_get_push_rules(self, user_ids): + batch_size = 100 + + def f(txn, user_ids_to_fetch): + sql = ( + "SELECT " + + ",".join(map(lambda x: "pr."+x, PushRuleTable.fields)) + + " FROM " + PushRuleTable.table_name + " pr " + + " LEFT JOIN " + PushRuleEnableTable.table_name + " pre " + + " ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " + + " WHERE pr.user_name " + + " IN (" + ",".join(["?" for _ in user_ids_to_fetch]) + ")" + " AND (pre.enabled is null or pre.enabled = 1)" + " ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC" + ) + txn.execute(sql, user_ids_to_fetch) + return txn.fetchall() + + results = {} + + batch_start = 0 + while batch_start < len(user_ids): + batch_end = max(len(user_ids), batch_size) + batch_user_ids = user_ids[batch_start:batch_end] + batch_start = batch_end + + rows = yield self.runInteraction( + "bulk_get_push_rules", f, batch_user_ids + ) + + for r in rows: + rawdict = { + PushRuleTable.fields[i]: r[i] for i in range(len(r)) + } + + if rawdict['user_name'] not in results: + results[rawdict['user_name']] = [] + results[rawdict['user_name']].append(rawdict) + defer.returnValue(results) + @defer.inlineCallbacks def add_push_rule(self, before, after, **kwargs): vals = kwargs From 77f06856b612e2905b5b69f6f2de75ddd348adfd Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 15:22:26 +0000 Subject: [PATCH 011/294] clarify problems --- problems_with_this_branch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/problems_with_this_branch b/problems_with_this_branch index 9c5df5521..baed6f5c6 100644 --- a/problems_with_this_branch +++ b/problems_with_this_branch @@ -2,4 +2,4 @@ * processing push twice because the pushers still run the rules themselves rather than pulling out of the event_actions table * consequently converting events back & forth between objects & dicts when processing * fetching all the current ephemeral events for each room on an incremental sync to get the user's read receipt - * doing 2 more db queries per room to get the unread notif count + * doing 2 more db queries per room at sync time to get the unread notif count From 4c8f6a7e427cc0e22ff1a19c3f1d9da0f9438f18 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 17:04:31 +0000 Subject: [PATCH 012/294] Insert push actions in a single db query rather than one per user/profile_tag --- synapse/push/action_generator.py | 10 ++++++---- synapse/storage/event_actions.py | 31 ++++++++++++++++++------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 2ad5f82da..148b1bda8 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -43,7 +43,9 @@ class ActionGenerator: actions_by_user = bulk_evaluator.action_for_event_by_user(event) - for uid,actions in actions_by_user.items(): - self.store.set_actions_for_event( - event, uid, None, actions - ) + yield self.store.set_actions_for_event_and_users( + event, + [ + (uid, None, actions) for uid, actions in actions_by_user.items() + ] + ) diff --git a/synapse/storage/event_actions.py b/synapse/storage/event_actions.py index fbd0a4227..3efa445c1 100644 --- a/synapse/storage/event_actions.py +++ b/synapse/storage/event_actions.py @@ -24,22 +24,27 @@ logger = logging.getLogger(__name__) class EventActionsStore(SQLBaseStore): @defer.inlineCallbacks - def set_actions_for_event(self, event, user_id, profile_tag, actions): - actionsJson = json.dumps(actions) - - ret = yield self.runInteraction( - "_set_actions_for_event", - self._simple_upsert_txn, - EventActionsTable.table_name, - { + def set_actions_for_event_and_users(self, event, tuples): + """ + :param event: the event set actions for + :param tuples: list of tuples of (user_id, profile_tag, actions) + """ + values = [] + for uid, profile_tag, actions in tuples: + values.append({ 'room_id': event['room_id'], 'event_id': event['event_id'], - 'user_id': user_id, - 'profile_tag': profile_tag - }, - {'actions': actionsJson} + 'user_id': uid, + 'profile_tag': profile_tag, + 'actions': json.dumps(actions) + }) + + yield self.runInteraction( + "set_actions_for_event_and_users", + self._simple_insert_many_txn, + EventActionsTable.table_name, + values ) - defer.returnValue(ret) @defer.inlineCallbacks def get_unread_event_actions_by_room_for_user( From 3fbb0317453f00f4d84ebc52145d8afb6490909f Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 17:13:47 +0000 Subject: [PATCH 013/294] Remove the list of problems (moved to jira issues) --- problems_with_this_branch | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 problems_with_this_branch diff --git a/problems_with_this_branch b/problems_with_this_branch deleted file mode 100644 index baed6f5c6..000000000 --- a/problems_with_this_branch +++ /dev/null @@ -1,5 +0,0 @@ - * processing push by bluntly running the rules for each user - * processing push twice because the pushers still run the rules themselves rather than pulling out of the event_actions table - * consequently converting events back & forth between objects & dicts when processing - * fetching all the current ephemeral events for each room on an incremental sync to get the user's read receipt - * doing 2 more db queries per room at sync time to get the unread notif count From 5645d9747b17e9d119cc7badd7c2abe3c157a1a6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 17:19:22 +0000 Subject: [PATCH 014/294] Add some comments to areas that could be optimised. --- synapse/handlers/sync.py | 3 +++ synapse/push/__init__.py | 4 +++- synapse/push/bulk_push_rule_evaluator.py | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 4cbb43a31..fa5e954e0 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -447,6 +447,9 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("presence_key", presence_key) + # We now fetch all ephemeral events for this room in order to get + # this users current read receipt. This could almost certainly be + # optimised. _, all_ephemeral_by_room = yield self.ephemeral_by_room( sync_config, now_token ) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index d5928c1d2..635dedd52 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -26,7 +26,9 @@ import random logger = logging.getLogger(__name__) - +# Pushers could now be moved to pull out of the event_actions table instead +# of listening on the event stream: this would avoid them having to run the +# rules again. class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index f531d2edc..1c0fa72b2 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -59,6 +59,14 @@ def evaluator_for_room_id(room_id, hs, store): class BulkPushRuleEvaluator: + """ + Runs push rules for all users in a room. + This is faster than running PushRuleEvaluator for each user because it + fetches all the rules for all the users in one (batched) db query + rarher than doing multiple queries per-user. It currently uses + the same logic to run the actual rules, but could be optimised further + (see https://matrix.org/jira/browse/SYN-562) + """ def __init__(self, room_id, rules_by_user, display_names, users_in_room): self.room_id = room_id self.rules_by_user = rules_by_user From 9b4cd0cd0f31803657018bf0ac9178787d796912 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 17:25:09 +0000 Subject: [PATCH 015/294] pep8 & unused variable --- synapse/push/__init__.py | 1 + synapse/push/action_generator.py | 2 -- synapse/push/bulk_push_rule_evaluator.py | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 635dedd52..250f22a16 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -26,6 +26,7 @@ import random logger = logging.getLogger(__name__) + # Pushers could now be moved to pull out of the event_actions table instead # of listening on the event stream: this would avoid them having to run the # rules again. diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 148b1bda8..00f518f60 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -35,8 +35,6 @@ class ActionGenerator: @defer.inlineCallbacks def handle_event(self, event): - users = yield self.store.get_users_in_room(event['room_id']) - bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( event['room_id'], self.hs, self.store ) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 1c0fa72b2..c489bfc8d 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -54,7 +54,7 @@ def evaluator_for_room_id(room_id, hs, store): display_names[ev.state_key] = ev.content.get("displayname") defer.returnValue(BulkPushRuleEvaluator( - room_id, rules_by_user, display_names, users + room_id, rules_by_user, display_names, users )) @@ -104,4 +104,4 @@ class BulkPushRuleEvaluator: matches &= PushRuleEvaluator._event_fulfills_condition( event, cond, display_name, room_member_count, profile_tag ) - return matches \ No newline at end of file + return matches From d79e90f078c83314de3dc469770750dd2585e255 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 17:56:56 +0000 Subject: [PATCH 016/294] Add mocks to make tests work again --- tests/handlers/test_federation.py | 7 +++++++ tests/handlers/test_room.py | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index d392c2301..a4758c03d 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -49,6 +49,10 @@ class FederationTestCase(unittest.TestCase): "get_destination_retry_timings", "set_destination_retry_timings", "have_events", + "get_users_in_room", + "bulk_get_push_rules", + "get_current_state", + "set_actions_for_event_and_users", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -85,6 +89,9 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed((1,1)) self.datastore.get_room.return_value = defer.succeed(True) + self.datastore.get_users_in_room.return_value = ["@a:b"] + self.datastore.bulk_get_push_rules.return_value = {} + self.datastore.get_current_state.return_value = {} self.auth.check_host_in_room.return_value = defer.succeed(True) retry_timings_res = { diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 2a7553f98..ba20b3194 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -43,6 +43,10 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "store_room", "get_latest_events_in_room", "add_event_hashes", + "get_users_in_room", + "bulk_get_push_rules", + "get_current_state", + "set_actions_for_event_and_users", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -90,6 +94,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.persist_event.return_value = (1,1) self.datastore.add_event_hashes.return_value = [] + self.datastore.get_users_in_room.return_value = ["@bob:red"] + self.datastore.bulk_get_push_rules.return_value = {} @defer.inlineCallbacks def test_invite(self): @@ -109,6 +115,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_latest_events_in_room.return_value = ( defer.succeed([]) ) + self.datastore.get_current_state.return_value = {} def annotate(_): ctx = Mock() @@ -190,6 +197,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_latest_events_in_room.return_value = ( defer.succeed([]) ) + self.datastore.get_current_state.return_value = {} def annotate(_): ctx = Mock() @@ -265,6 +273,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_latest_events_in_room.return_value = ( defer.succeed([]) ) + self.datastore.get_current_state.return_value = {} def annotate(_): ctx = Mock() From d2a92c6bdeff51136a930ccddda6e734a2a0dc25 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 Dec 2015 18:25:04 +0000 Subject: [PATCH 017/294] Fix merge fail with anon access stuff --- synapse/handlers/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b1bfdce85..f63c073a2 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -520,11 +520,11 @@ class SyncHandler(BaseHandler): # this users current read receipt. This could almost certainly be # optimised. _, all_ephemeral_by_room = yield self.ephemeral_by_room( - sync_config, now_token + sync_config, now_token, room_ids ) now_token, ephemeral_by_room = yield self.ephemeral_by_room( - sync_config, now_token, since_token + sync_config, now_token, room_ids, since_token ) rm_handler = self.hs.get_handlers().room_member_handler From 3051c9d002a467643d1ab32bc36974d2e3f84c12 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Jan 2016 13:39:29 +0000 Subject: [PATCH 018/294] Address minor PR issues --- synapse/handlers/_base.py | 4 ++-- synapse/handlers/federation.py | 4 ++-- synapse/push/action_generator.py | 7 +++---- synapse/push/bulk_push_rule_evaluator.py | 2 +- synapse/storage/event_actions.py | 2 +- synapse/storage/push_rule.py | 6 +++--- synapse/storage/registration.py | 12 ------------ 7 files changed, 12 insertions(+), 25 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 24c4c6269..938eb29de 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -267,8 +267,8 @@ class BaseHandler(object): event, context=context ) - action_generator = ActionGenerator(self.hs, self.store) - yield action_generator.handle_event(serialize_event( + action_generator = ActionGenerator(self.store) + yield action_generator.handle_push_actions_for_event(serialize_event( event, self.clock.time_msec() )) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0b1221deb..764709b42 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -245,8 +245,8 @@ class FederationHandler(BaseHandler): yield user_joined_room(self.distributor, user, event.room_id) if not backfilled and not event.internal_metadata.is_outlier(): - action_generator = ActionGenerator(self.hs, self.store) - yield action_generator.handle_event(serialize_event( + action_generator = ActionGenerator(self.store) + yield action_generator.handle_push_actions_for_event(serialize_event( event, self.clock.time_msec()) ) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 00f518f60..4ab5d9e1b 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -23,8 +23,7 @@ logger = logging.getLogger(__name__) class ActionGenerator: - def __init__(self, hs, store): - self.hs = hs + def __init__(self, store): self.store = store # really we want to get all user ids and all profile tags too, # since we want the actions for each profile tag for every user and @@ -34,9 +33,9 @@ class ActionGenerator: # tag (ie. we just need all the users). @defer.inlineCallbacks - def handle_event(self, event): + def handle_push_actions_for_event(self, event): bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( - event['room_id'], self.hs, self.store + event['room_id'], self.store ) actions_by_user = bulk_evaluator.action_for_event_by_user(event) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index c489bfc8d..1c4e54ba4 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -33,7 +33,7 @@ def decode_rule_json(rule): @defer.inlineCallbacks -def evaluator_for_room_id(room_id, hs, store): +def evaluator_for_room_id(room_id, store): users = yield store.get_users_in_room(room_id) rules_by_user = yield store.bulk_get_push_rules(users) rules_by_user = { diff --git a/synapse/storage/event_actions.py b/synapse/storage/event_actions.py index 3efa445c1..fa9cbe71e 100644 --- a/synapse/storage/event_actions.py +++ b/synapse/storage/event_actions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 9dec4aa68..7c5123d64 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -62,12 +62,12 @@ class PushRuleStore(SQLBaseStore): def f(txn, user_ids_to_fetch): sql = ( "SELECT " + - ",".join(map(lambda x: "pr."+x, PushRuleTable.fields)) + + ",".join("pr."+x for x in PushRuleTable.fields) + " FROM " + PushRuleTable.table_name + " pr " + " LEFT JOIN " + PushRuleEnableTable.table_name + " pre " + " ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " + " WHERE pr.user_name " + - " IN (" + ",".join(["?" for _ in user_ids_to_fetch]) + ")" + " IN (" + ",".join("?" for _ in user_ids_to_fetch) + ")" " AND (pre.enabled is null or pre.enabled = 1)" " ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC" ) @@ -78,7 +78,7 @@ class PushRuleStore(SQLBaseStore): batch_start = 0 while batch_start < len(user_ids): - batch_end = max(len(user_ids), batch_size) + batch_end = min(len(user_ids), batch_size) batch_user_ids = user_ids[batch_start:batch_end] batch_start = batch_end diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 4676f225b..09a05b08e 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -291,18 +291,6 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(ret['user_id']) defer.returnValue(None) - @defer.inlineCallbacks - def get_all_user_ids(self): - """Returns all user ids registered on this homeserver""" - return self.runInteraction( - "get_all_user_ids", - self._get_all_user_ids_txn - ) - - def _get_all_user_ids_txn(self, txn): - txn.execute("SELECT name from users") - return [r[0] for r in txn.fetchall()] - @defer.inlineCallbacks def count_all_users(self): """Counts all users registered on the homeserver.""" From c914d67cda9682331639b78190db367974e4fb8b Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Jan 2016 14:05:37 +0000 Subject: [PATCH 019/294] Rename event-actions to event_push_actions as per PR request --- synapse/handlers/sync.py | 2 +- synapse/push/__init__.py | 2 +- synapse/push/action_generator.py | 2 +- synapse/storage/__init__.py | 4 ++-- ...event_actions.py => event_push_actions.py} | 20 +++++++++---------- ...ent_actions.sql => event_push_actions.sql} | 4 ++-- 6 files changed, 17 insertions(+), 17 deletions(-) rename synapse/storage/{event_actions.py => event_push_actions.py} (84%) rename synapse/storage/schema/delta/27/{event_actions.sql => event_push_actions.sql} (82%) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f63c073a2..64556c5eb 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -896,7 +896,7 @@ class SyncHandler(BaseHandler): notifs = [] if last_unread_event_id: - notifs = yield self.store.get_unread_event_actions_by_room_for_user( + notifs = yield self.store.get_unread_event_push_actions_by_room_for_user( room_id, sync_config.user.to_string(), last_unread_event_id ) else: diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 250f22a16..3ab6da062 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -27,7 +27,7 @@ import random logger = logging.getLogger(__name__) -# Pushers could now be moved to pull out of the event_actions table instead +# Pushers could now be moved to pull out of the event_push_actions table instead # of listening on the event stream: this would avoid them having to run the # rules again. class Pusher(object): diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 4ab5d9e1b..5526324a6 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -40,7 +40,7 @@ class ActionGenerator: actions_by_user = bulk_evaluator.action_for_event_by_user(event) - yield self.store.set_actions_for_event_and_users( + yield self.store.set_push_actions_for_event_and_users( event, [ (uid, None, actions) for uid, actions in actions_by_user.items() diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a112dd237..43e05f144 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -33,7 +33,7 @@ from .pusher import PusherStore from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore from .rejections import RejectionsStore -from .event_actions import EventActionsStore +from .event_push_actions import EventPushActionsStore from .state import StateStore from .signatures import SignatureStore @@ -76,7 +76,7 @@ class DataStore(RoomMemberStore, RoomStore, SearchStore, TagsStore, AccountDataStore, - EventActionsStore + EventPushActionsStore ): def __init__(self, hs): diff --git a/synapse/storage/event_actions.py b/synapse/storage/event_push_actions.py similarity index 84% rename from synapse/storage/event_actions.py rename to synapse/storage/event_push_actions.py index fa9cbe71e..016c0adf8 100644 --- a/synapse/storage/event_actions.py +++ b/synapse/storage/event_push_actions.py @@ -22,9 +22,9 @@ import simplejson as json logger = logging.getLogger(__name__) -class EventActionsStore(SQLBaseStore): +class EventPushActionsStore(SQLBaseStore): @defer.inlineCallbacks - def set_actions_for_event_and_users(self, event, tuples): + def set_push_actions_for_event_and_users(self, event, tuples): """ :param event: the event set actions for :param tuples: list of tuples of (user_id, profile_tag, actions) @@ -42,15 +42,15 @@ class EventActionsStore(SQLBaseStore): yield self.runInteraction( "set_actions_for_event_and_users", self._simple_insert_many_txn, - EventActionsTable.table_name, + EventPushActionsTable.table_name, values ) @defer.inlineCallbacks - def get_unread_event_actions_by_room_for_user( + def get_unread_event_push_actions_by_room_for_user( self, room_id, user_id, last_read_event_id ): - def _get_unread_event_actions_by_room(txn): + def _get_unread_event_push_actions_by_room(txn): sql = ( "SELECT stream_ordering, topological_ordering" " FROM events" @@ -68,7 +68,7 @@ class EventActionsStore(SQLBaseStore): sql = ( "SELECT ea.event_id, ea.actions" - " FROM event_actions ea, events e" + " FROM event_push_actions ea, events e" " WHERE ea.room_id = e.room_id" " AND ea.event_id = e.event_id" " AND ea.user_id = ?" @@ -88,11 +88,11 @@ class EventActionsStore(SQLBaseStore): ] ret = yield self.runInteraction( - "get_unread_event_actions_by_room", - _get_unread_event_actions_by_room + "get_unread_event_push_actions_by_room", + _get_unread_event_push_actions_by_room ) defer.returnValue(ret) -class EventActionsTable(object): - table_name = "event_actions" +class EventPushActionsTable(object): + table_name = "event_push_actions" diff --git a/synapse/storage/schema/delta/27/event_actions.sql b/synapse/storage/schema/delta/27/event_push_actions.sql similarity index 82% rename from synapse/storage/schema/delta/27/event_actions.sql rename to synapse/storage/schema/delta/27/event_push_actions.sql index bbdaee990..bdf6ae3f2 100644 --- a/synapse/storage/schema/delta/27/event_actions.sql +++ b/synapse/storage/schema/delta/27/event_push_actions.sql @@ -13,7 +13,7 @@ * limitations under the License. */ -CREATE TABLE IF NOT EXISTS event_actions( +CREATE TABLE IF NOT EXISTS event_push_actions( room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, @@ -23,4 +23,4 @@ CREATE TABLE IF NOT EXISTS event_actions( ); -CREATE INDEX event_actions_room_id_event_id_user_id_profile_tag on event_actions(room_id, event_id, user_id, profile_tag); +CREATE INDEX event_push_actions_room_id_event_id_user_id_profile_tag on event_push_actions(room_id, event_id, user_id, profile_tag); From 92a1e74b202757b0f4b577ccbd3e31d8dd4d6460 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Jan 2016 14:17:35 +0000 Subject: [PATCH 020/294] fix tests --- tests/handlers/test_federation.py | 2 +- tests/handlers/test_room.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index a4758c03d..6acc4ebad 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -52,7 +52,7 @@ class FederationTestCase(unittest.TestCase): "get_users_in_room", "bulk_get_push_rules", "get_current_state", - "set_actions_for_event_and_users", + "set_push_actions_for_event_and_users", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index ba20b3194..ff2b59712 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -46,7 +46,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "get_users_in_room", "bulk_get_push_rules", "get_current_state", - "set_actions_for_event_and_users", + "set_push_actions_for_event_and_users", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), From f1b67730fa085755a7ab459b0239608bd3585a67 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Jan 2016 14:50:36 +0000 Subject: [PATCH 021/294] Add unread_notif_count in incremental_sync_with_gap --- synapse/handlers/sync.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 64556c5eb..93c48d167 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -749,6 +749,13 @@ class SyncHandler(BaseHandler): if just_joined: state = yield self.get_state_at(room_id, now_token) + notifs = yield self.unread_notifs_for_room_id( + room_id, sync_config, ephemeral_by_room + ) + notif_count = None + if notifs is not None: + notif_count = len(notifs) + room_sync = JoinedSyncResult( room_id=room_id, timeline=batch, @@ -757,6 +764,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), + unread_notification_count=notif_count, ) logging.debug("Room sync: %r", room_sync) From d74c6ace247c31524dacf795108300e53bdbff55 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Jan 2016 15:32:00 +0000 Subject: [PATCH 022/294] comma --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 93c48d167..2ec42ee50 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -346,7 +346,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), - unread_notification_count=notif_count + unread_notification_count=notif_count, )) def account_data_for_user(self, account_data): From c77e7e60fc5d29e8a57bb82dd5aa8e72ae570d84 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Jan 2016 15:49:06 +0000 Subject: [PATCH 023/294] Only joined rooms have unread_notif_count --- synapse/rest/client/v2_alpha/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index cd3aef9e0..095f96e21 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -311,12 +311,12 @@ class SyncRestServlet(RestServlet): }, "state": {"events": serialized_state}, "account_data": {"events": account_data}, - "unread_notification_count": room.unread_notification_count } if joined: ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) result["ephemeral"] = {"events": ephemeral_events} + result["unread_notification_count"] = room.unread_notification_count return result From 4eb7b950c829dd8463df8ccd1095772452293a15 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Jan 2016 18:11:17 +0000 Subject: [PATCH 024/294] = not == in sql --- synapse/storage/event_push_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 016c0adf8..3075d0225 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -75,7 +75,7 @@ class EventPushActionsStore(SQLBaseStore): " AND ea.room_id = ?" " AND (" " e.topological_ordering > ?" - " OR (e.topological_ordering == ? AND e.stream_ordering > ?)" + " OR (e.topological_ordering = ? AND e.stream_ordering > ?)" ")" ) txn.execute(sql, ( From 1b5642604b9e5c662506d0bd2608d867287b6dfb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 Jan 2016 11:56:21 +0000 Subject: [PATCH 025/294] Support inviting 3pids in /createRoom --- synapse/handlers/room.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13f66e0df..2bdc76819 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -115,6 +115,8 @@ class RoomCreationHandler(BaseHandler): except: raise SynapseError(400, "Invalid user_id: %s" % (i,)) + invite_3pid_list = config.get("invite_3pid", []) + is_public = config.get("visibility", None) == "public" if room_id: @@ -220,6 +222,20 @@ class RoomCreationHandler(BaseHandler): "content": {"membership": Membership.INVITE}, }, ratelimit=False) + for invite_3pid in invite_3pid_list: + id_server = invite_3pid["id_server"] + address = invite_3pid["address"] + medium = invite_3pid["medium"] + yield self.hs.get_handlers().room_member_handler.do_3pid_invite( + room_id, + user, + medium, + address, + id_server, + None, + None, + ) + result = {"room_id": room_id} if room_alias: From c3ea36304bef9dc643e28d1aa9cd91d99f4434d9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 Jan 2016 12:57:45 +0000 Subject: [PATCH 026/294] Use named args --- synapse/handlers/room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 2bdc76819..0cfeda10d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -232,8 +232,8 @@ class RoomCreationHandler(BaseHandler): medium, address, id_server, - None, - None, + token_id=None, + txn_id=None, ) result = {"room_id": room_id} From 85ca8cb90c706ad40034a05282d751887d9bf05f Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 5 Jan 2016 13:32:39 +0000 Subject: [PATCH 027/294] comment typo --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 1c4e54ba4..c00acfd87 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -63,7 +63,7 @@ class BulkPushRuleEvaluator: Runs push rules for all users in a room. This is faster than running PushRuleEvaluator for each user because it fetches all the rules for all the users in one (batched) db query - rarher than doing multiple queries per-user. It currently uses + rather than doing multiple queries per-user. It currently uses the same logic to run the actual rules, but could be optimised further (see https://matrix.org/jira/browse/SYN-562) """ From acb19068d00903e8613b50e5d64e1daf31bbe9a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 Jan 2016 14:49:06 +0000 Subject: [PATCH 028/294] Return /sync when something under the 'leave' key has changed --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index feea407ea..9796f2a57 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -115,7 +115,7 @@ class SyncResult(collections.namedtuple("SyncResult", [ events. """ return bool( - self.presence or self.joined or self.invited + self.presence or self.joined or self.invited or self.archived ) GuestRoom = collections.namedtuple("GuestRoom", ("room_id", "membership")) From 8737ead008d5b5ff17839b1aa0e4f0889d339770 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 Jan 2016 17:30:30 +0000 Subject: [PATCH 029/294] Use larger thumbnail rather than smaller. --- synapse/rest/media/v1/thumbnail_resource.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index e506dad93..8b8fba3dc 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -248,6 +248,7 @@ class ThumbnailResource(BaseMediaResource): if desired_method.lower() == "crop": info_list = [] + info_list2 = [] for info in thumbnail_infos: t_w = info["thumbnail_width"] t_h = info["thumbnail_height"] @@ -258,12 +259,20 @@ class ThumbnailResource(BaseMediaResource): size_quality = abs((d_w - t_w) * (d_h - t_h)) type_quality = desired_type != info["thumbnail_type"] length_quality = info["thumbnail_length"] - info_list.append(( - aspect_quality, min_quality, size_quality, type_quality, - length_quality, info - )) + if t_w >= d_w or t_h >= d_h: + info_list.append(( + aspect_quality, min_quality, size_quality, type_quality, + length_quality, info + )) + else: + info_list2.append(( + aspect_quality, min_quality, size_quality, type_quality, + length_quality, info + )) if info_list: return min(info_list)[-1] + else: + return min(info_list2)[-1] else: info_list = [] info_list2 = [] From cfd07aafff71b452a01265f304172f56b2c49759 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 5 Jan 2016 18:01:18 +0000 Subject: [PATCH 030/294] Allow guests to upgrade their accounts --- synapse/api/auth.py | 6 +-- synapse/handlers/auth.py | 6 +-- synapse/handlers/register.py | 37 ++++++++++++++----- synapse/handlers/room.py | 2 +- synapse/handlers/sync.py | 2 +- synapse/rest/client/v2_alpha/register.py | 12 ++++-- synapse/rest/media/v1/thumbnail_resource.py | 2 +- synapse/storage/prepare_database.py | 4 +- synapse/storage/registration.py | 23 ++++++++---- .../storage/schema/delta/28/upgrade_times.sql | 21 +++++++++++ tests/api/test_auth.py | 18 ++++----- 11 files changed, 93 insertions(+), 40 deletions(-) create mode 100644 synapse/storage/schema/delta/28/upgrade_times.sql diff --git a/synapse/api/auth.py b/synapse/api/auth.py index adb7d6448..b86c6c839 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -583,7 +583,7 @@ class Auth(object): AuthError if no user by that token exists or the token is invalid. """ try: - ret = yield self._get_user_from_macaroon(token) + ret = yield self.get_user_from_macaroon(token) except AuthError: # TODO(daniel): Remove this fallback when all existing access tokens # have been re-issued as macaroons. @@ -591,7 +591,7 @@ class Auth(object): defer.returnValue(ret) @defer.inlineCallbacks - def _get_user_from_macaroon(self, macaroon_str): + def get_user_from_macaroon(self, macaroon_str): try: macaroon = pymacaroons.Macaroon.deserialize(macaroon_str) self.validate_macaroon(macaroon, "access", False) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e64b67cdf..62e82a257 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -408,7 +408,7 @@ class AuthHandler(BaseHandler): macaroon = pymacaroons.Macaroon.deserialize(login_token) auth_api = self.hs.get_auth() auth_api.validate_macaroon(macaroon, "login", True) - return self._get_user_from_macaroon(macaroon) + return self.get_user_from_macaroon(macaroon) except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError): raise AuthError(401, "Invalid token", errcode=Codes.UNKNOWN_TOKEN) @@ -421,7 +421,7 @@ class AuthHandler(BaseHandler): macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) return macaroon - def _get_user_from_macaroon(self, macaroon): + def get_user_from_macaroon(self, macaroon): user_prefix = "user_id = " for caveat in macaroon.caveats: if caveat.caveat_id.startswith(user_prefix): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index baf7c14e4..6f111ff63 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,12 +40,13 @@ class RegistrationHandler(BaseHandler): def __init__(self, hs): super(RegistrationHandler, self).__init__(hs) + self.auth = hs.get_auth() self.distributor = hs.get_distributor() self.distributor.declare("registered_user") self.captcha_client = CaptchaServerHttpClient(hs) @defer.inlineCallbacks - def check_username(self, localpart): + def check_username(self, localpart, guest_access_token=None): yield run_on_reactor() if urllib.quote(localpart) != localpart: @@ -62,14 +63,29 @@ class RegistrationHandler(BaseHandler): users = yield self.store.get_users_by_id_case_insensitive(user_id) if users: - raise SynapseError( - 400, - "User ID already taken.", - errcode=Codes.USER_IN_USE, - ) + if not guest_access_token: + raise SynapseError( + 400, + "User ID already taken.", + errcode=Codes.USER_IN_USE, + ) + user_data = yield self.auth.get_user_from_macaroon(guest_access_token) + if not user_data["is_guest"] or user_data["user"].localpart != localpart: + raise AuthError( + 403, + "Cannot register taken user ID without valid guest " + "credentials for that user.", + errcode=Codes.FORBIDDEN, + ) @defer.inlineCallbacks - def register(self, localpart=None, password=None, generate_token=True): + def register( + self, + localpart=None, + password=None, + generate_token=True, + guest_access_token=None + ): """Registers a new client on the server. Args: @@ -89,7 +105,7 @@ class RegistrationHandler(BaseHandler): password_hash = self.auth_handler().hash(password) if localpart: - yield self.check_username(localpart) + yield self.check_username(localpart, guest_access_token=guest_access_token) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() @@ -100,7 +116,8 @@ class RegistrationHandler(BaseHandler): yield self.store.register( user_id=user_id, token=token, - password_hash=password_hash + password_hash=password_hash, + was_guest=guest_access_token is not None, ) yield registered_user(self.distributor, user) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0cfeda10d..6186c37c7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9796f2a57..41a42418a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd +# Copyright 2015 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index b2b89652c..25389cede 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd +# Copyright 2015 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -119,8 +119,13 @@ class RegisterRestServlet(RestServlet): if self.hs.config.disable_registration: raise SynapseError(403, "Registration has been disabled") + guest_access_token = body.get("guest_access_token", None) + if desired_username is not None: - yield self.registration_handler.check_username(desired_username) + yield self.registration_handler.check_username( + desired_username, + guest_access_token=guest_access_token + ) if self.hs.config.enable_registration_captcha: flows = [ @@ -150,7 +155,8 @@ class RegisterRestServlet(RestServlet): (user_id, token) = yield self.registration_handler.register( localpart=desired_username, - password=new_password + password=new_password, + guest_access_token=guest_access_token, ) if result and LoginType.EMAIL_IDENTITY in result: diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 8b8fba3dc..c18160534 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 16eff6254..c1f5f9978 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 27 +SCHEMA_VERSION = 28 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 09a05b08e..f0fa0bd33 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -73,30 +73,39 @@ class RegistrationStore(SQLBaseStore): ) @defer.inlineCallbacks - def register(self, user_id, token, password_hash): + def register(self, user_id, token, password_hash, was_guest=False): """Attempts to register an account. Args: user_id (str): The desired user ID to register. token (str): The desired access token to use for this user. password_hash (str): Optional. The password hash for this user. + was_guest (bool): Optional. Whether this is a guest account being + upgraded to a non-guest account. Raises: StoreError if the user_id could not be registered. """ yield self.runInteraction( "register", - self._register, user_id, token, password_hash + self._register, user_id, token, password_hash, was_guest ) - def _register(self, txn, user_id, token, password_hash): + def _register(self, txn, user_id, token, password_hash, was_guest): now = int(self.clock.time()) next_id = self._access_tokens_id_gen.get_next_txn(txn) try: - txn.execute("INSERT INTO users(name, password_hash, creation_ts) " - "VALUES (?,?,?)", - [user_id, password_hash, now]) + if was_guest: + txn.execute("UPDATE users SET" + " password_hash = ?," + " upgrade_ts = ?" + " WHERE name = ?", + [password_hash, now, user_id]) + else: + txn.execute("INSERT INTO users(name, password_hash, creation_ts) " + "VALUES (?,?,?)", + [user_id, password_hash, now]) except self.database_engine.module.IntegrityError: raise StoreError( 400, "User ID already taken.", errcode=Codes.USER_IN_USE diff --git a/synapse/storage/schema/delta/28/upgrade_times.sql b/synapse/storage/schema/delta/28/upgrade_times.sql new file mode 100644 index 000000000..3e4a9ab45 --- /dev/null +++ b/synapse/storage/schema/delta/28/upgrade_times.sql @@ -0,0 +1,21 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Stores the timestamp when a user upgraded from a guest to a full user, if + * that happened. + */ + +ALTER TABLE users ADD COLUMN upgrade_ts BIGINT; diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 70d928def..5ff4c8a87 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd +# Copyright 2015 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -154,7 +154,7 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("gen = 1") macaroon.add_first_party_caveat("type = access") macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) - user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize()) + user_info = yield self.auth.get_user_from_macaroon(macaroon.serialize()) user = user_info["user"] self.assertEqual(UserID.from_string(user_id), user) @@ -171,7 +171,7 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("guest = true") serialized = macaroon.serialize() - user_info = yield self.auth._get_user_from_macaroon(serialized) + user_info = yield self.auth.get_user_from_macaroon(serialized) user = user_info["user"] is_guest = user_info["is_guest"] self.assertEqual(UserID.from_string(user_id), user) @@ -192,7 +192,7 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("type = access") macaroon.add_first_party_caveat("user_id = %s" % (user,)) with self.assertRaises(AuthError) as cm: - yield self.auth._get_user_from_macaroon(macaroon.serialize()) + yield self.auth.get_user_from_macaroon(macaroon.serialize()) self.assertEqual(401, cm.exception.code) self.assertIn("User mismatch", cm.exception.msg) @@ -212,7 +212,7 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("type = access") with self.assertRaises(AuthError) as cm: - yield self.auth._get_user_from_macaroon(macaroon.serialize()) + yield self.auth.get_user_from_macaroon(macaroon.serialize()) self.assertEqual(401, cm.exception.code) self.assertIn("No user caveat", cm.exception.msg) @@ -234,7 +234,7 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("user_id = %s" % (user,)) with self.assertRaises(AuthError) as cm: - yield self.auth._get_user_from_macaroon(macaroon.serialize()) + yield self.auth.get_user_from_macaroon(macaroon.serialize()) self.assertEqual(401, cm.exception.code) self.assertIn("Invalid macaroon", cm.exception.msg) @@ -257,7 +257,7 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("cunning > fox") with self.assertRaises(AuthError) as cm: - yield self.auth._get_user_from_macaroon(macaroon.serialize()) + yield self.auth.get_user_from_macaroon(macaroon.serialize()) self.assertEqual(401, cm.exception.code) self.assertIn("Invalid macaroon", cm.exception.msg) @@ -285,11 +285,11 @@ class AuthTestCase(unittest.TestCase): self.hs.clock.now = 5000 # seconds - yield self.auth._get_user_from_macaroon(macaroon.serialize()) + yield self.auth.get_user_from_macaroon(macaroon.serialize()) # TODO(daniel): Turn on the check that we validate expiration, when we # validate expiration (and remove the above line, which will start # throwing). # with self.assertRaises(AuthError) as cm: - # yield self.auth._get_user_from_macaroon(macaroon.serialize()) + # yield self.auth.get_user_from_macaroon(macaroon.serialize()) # self.assertEqual(401, cm.exception.code) # self.assertIn("Invalid macaroon", cm.exception.msg) From 2ef6de928d9f0d2761e0664d374b2bfe13682061 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 5 Jan 2016 18:12:37 +0000 Subject: [PATCH 031/294] Skip, rather than erroring, invalid guest requests Erroring causes problems when people make illegal requests, because they don't know what limit parameter they should pass. This is definitely buggy. It leaks message counts for rooms people don't have permission to see, via tokens. But apparently we already consciously decided to allow that as a team, so this preserves that behaviour. --- synapse/handlers/_base.py | 16 ++-------------- synapse/handlers/message.py | 4 ++-- synapse/handlers/room.py | 2 -- synapse/handlers/sync.py | 1 - synapse/notifier.py | 3 +-- 5 files changed, 5 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 5fd20285d..b474042e8 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -52,8 +52,7 @@ class BaseHandler(object): self.event_builder_factory = hs.get_event_builder_factory() @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, events, is_guest=False, - require_all_visible_for_guests=True): + def _filter_events_for_client(self, user_id, events, is_guest=False): # Assumes that user has at some point joined the room if not is_guest. def allowed(event, membership, visibility): @@ -114,17 +113,6 @@ class BaseHandler(object): if should_include: events_to_return.append(event) - if (require_all_visible_for_guests - and is_guest - and len(events_to_return) < len(events)): - # This indicates that some events in the requested range were not - # visible to guest users. To be safe, we reject the entire request, - # so that we don't have to worry about interpreting visibility - # boundaries. - raise AuthError(403, "User %s does not have permission" % ( - user_id - )) - defer.returnValue(events_to_return) def ratelimit(self, user_id): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index a1bed9b0d..5805190ce 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -685,7 +685,7 @@ class MessageHandler(BaseHandler): ).addErrback(unwrapFirstError) messages = yield self._filter_events_for_client( - user_id, messages, is_guest=is_guest, require_all_visible_for_guests=False + user_id, messages, is_guest=is_guest, ) start_token = now_token.copy_and_replace("room_key", token[0]) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6186c37c7..48a07e4e3 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -895,14 +895,12 @@ class RoomContextHandler(BaseHandler): user.to_string(), results["events_before"], is_guest=is_guest, - require_all_visible_for_guests=False ) results["events_after"] = yield self._filter_events_for_client( user.to_string(), results["events_after"], is_guest=is_guest, - require_all_visible_for_guests=False ) if results["events_after"]: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 41a42418a..b63a27b38 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -648,7 +648,6 @@ class SyncHandler(BaseHandler): sync_config.user.to_string(), loaded_recents, is_guest=sync_config.is_guest, - require_all_visible_for_guests=False ) loaded_recents.extend(recents) recents = loaded_recents diff --git a/synapse/notifier.py b/synapse/notifier.py index fd5257832..0a5653b8d 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -386,7 +386,6 @@ class Notifier(object): user.to_string(), new_events, is_guest=is_guest, - require_all_visible_for_guests=False ) events.extend(new_events) From 8ce5679813a49e5a575481efa3e7d3d191a0c8ef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 Jan 2016 11:19:51 +0000 Subject: [PATCH 032/294] Update readme to reflect change in signing key name --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 446cc6e7c..291a308b6 100644 --- a/README.rst +++ b/README.rst @@ -167,8 +167,7 @@ identify itself to other Home Servers, so don't lose or delete them. It would be wise to back them up somewhere safe. If, for whatever reason, you do need to change your Home Server's keys, you may find that other Home Servers have the old key cached. If you update the signing key, you should change the name of the -key in the .signing.key file (the second word, which by default is -, 'auto') to something different. +key in the .signing.key file (the second word) to something different. By default, registration of new users is disabled. You can either enable registration in the config by specifying ``enable_registration: true`` From c79f221192044203b9d32cfbd416a7fefeb34cd5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Jan 2016 11:38:09 +0000 Subject: [PATCH 033/294] Add is_guest flag to users db to track whether a user is a guest user or not. Use this so we can run _filter_events_for_client when calculating event_push_actions. --- synapse/handlers/_base.py | 8 ++-- synapse/handlers/federation.py | 6 +-- synapse/handlers/register.py | 4 +- synapse/push/action_generator.py | 6 +-- synapse/push/bulk_push_rule_evaluator.py | 27 ++++++++++--- synapse/rest/client/v2_alpha/register.py | 5 ++- synapse/storage/event_push_actions.py | 4 +- synapse/storage/registration.py | 40 ++++++++++++++----- .../delta/{27 => 28}/event_push_actions.sql | 0 9 files changed, 69 insertions(+), 31 deletions(-) rename synapse/storage/schema/delta/{27 => 28}/event_push_actions.sql (100%) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 3115a5065..66e35de6e 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -23,8 +23,6 @@ from synapse.push.action_generator import ActionGenerator from synapse.util.logcontext import PreserveLoggingContext -from synapse.events.utils import serialize_event - import logging @@ -256,9 +254,9 @@ class BaseHandler(object): ) action_generator = ActionGenerator(self.store) - yield action_generator.handle_push_actions_for_event(serialize_event( - event, self.clock.time_msec() - )) + yield action_generator.handle_push_actions_for_event( + event, self + ) destinations = set(extra_destinations) for k, s in context.current_state.items(): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 764709b42..075b9e21c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -32,7 +32,7 @@ from synapse.crypto.event_signing import ( ) from synapse.types import UserID -from synapse.events.utils import prune_event, serialize_event +from synapse.events.utils import prune_event from synapse.util.retryutils import NotRetryingDestination @@ -246,8 +246,8 @@ class FederationHandler(BaseHandler): if not backfilled and not event.internal_metadata.is_outlier(): action_generator = ActionGenerator(self.store) - yield action_generator.handle_push_actions_for_event(serialize_event( - event, self.clock.time_msec()) + yield action_generator.handle_push_actions_for_event( + event, self ) @defer.inlineCallbacks diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 6f111ff63..1799a668c 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -84,7 +84,8 @@ class RegistrationHandler(BaseHandler): localpart=None, password=None, generate_token=True, - guest_access_token=None + guest_access_token=None, + make_guest=False ): """Registers a new client on the server. @@ -118,6 +119,7 @@ class RegistrationHandler(BaseHandler): token=token, password_hash=password_hash, was_guest=guest_access_token is not None, + make_guest=make_guest ) yield registered_user(self.distributor, user) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 5526324a6..bcd40798f 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -33,12 +33,12 @@ class ActionGenerator: # tag (ie. we just need all the users). @defer.inlineCallbacks - def handle_push_actions_for_event(self, event): + def handle_push_actions_for_event(self, event, handler): bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( - event['room_id'], self.store + event.room_id, self.store ) - actions_by_user = bulk_evaluator.action_for_event_by_user(event) + actions_by_user = yield bulk_evaluator.action_for_event_by_user(event, handler) yield self.store.set_push_actions_for_event_and_users( event, diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index c00acfd87..63d65b446 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -23,6 +23,8 @@ from synapse.types import UserID import baserules from push_rule_evaluator import PushRuleEvaluator +from synapse.events.utils import serialize_event + logger = logging.getLogger(__name__) @@ -54,7 +56,7 @@ def evaluator_for_room_id(room_id, store): display_names[ev.state_key] = ev.content.get("displayname") defer.returnValue(BulkPushRuleEvaluator( - room_id, rules_by_user, display_names, users + room_id, rules_by_user, display_names, users, store )) @@ -67,13 +69,15 @@ class BulkPushRuleEvaluator: the same logic to run the actual rules, but could be optimised further (see https://matrix.org/jira/browse/SYN-562) """ - def __init__(self, room_id, rules_by_user, display_names, users_in_room): + def __init__(self, room_id, rules_by_user, display_names, users_in_room, store): self.room_id = room_id self.rules_by_user = rules_by_user self.display_names = display_names self.users_in_room = users_in_room + self.store = store - def action_for_event_by_user(self, event): + @defer.inlineCallbacks + def action_for_event_by_user(self, event, handler): actions_by_user = {} for uid, rules in self.rules_by_user.items(): @@ -81,6 +85,13 @@ class BulkPushRuleEvaluator: if uid in self.display_names: display_name = self.display_names[uid] + is_guest = yield self.store.is_guest(UserID.from_string(uid)) + filtered = yield handler._filter_events_for_client( + uid, [event], is_guest=is_guest + ) + if len(filtered) == 0: + continue + for rule in rules: if 'enabled' in rule and not rule['enabled']: continue @@ -94,14 +105,20 @@ class BulkPushRuleEvaluator: if len(actions) > 0: actions_by_user[uid] = actions break - return actions_by_user + defer.returnValue(actions_by_user) @staticmethod def event_matches_rule(event, rule, display_name, room_member_count, profile_tag): matches = True + + # passing the clock all the way into here is extremely awkward and push + # rules do not care about any of the relative timestamps, so we just + # pass 0 for the current time. + client_event = serialize_event(event, 0) + for cond in rule['conditions']: matches &= PushRuleEvaluator._event_fulfills_condition( - event, cond, display_name, room_member_count, profile_tag + client_event, cond, display_name, room_member_count, profile_tag ) return matches diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 25389cede..c4d025b46 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -259,7 +259,10 @@ class RegisterRestServlet(RestServlet): def _do_guest_registration(self): if not self.hs.config.allow_guest_access: defer.returnValue((403, "Guest access is disabled")) - user_id, _ = yield self.registration_handler.register(generate_token=False) + user_id, _ = yield self.registration_handler.register( + generate_token=False, + make_guest=True + ) access_token = self.auth_handler.generate_access_token(user_id, ["guest = true"]) defer.returnValue((200, { "user_id": user_id, diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 3075d0225..0634af6b6 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -32,8 +32,8 @@ class EventPushActionsStore(SQLBaseStore): values = [] for uid, profile_tag, actions in tuples: values.append({ - 'room_id': event['room_id'], - 'event_id': event['event_id'], + 'room_id': event.room_id, + 'event_id': event.event_id, 'user_id': uid, 'profile_tag': profile_tag, 'actions': json.dumps(actions) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f0fa0bd33..c79066f77 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError, Codes from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cached +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks class RegistrationStore(SQLBaseStore): @@ -73,7 +73,8 @@ class RegistrationStore(SQLBaseStore): ) @defer.inlineCallbacks - def register(self, user_id, token, password_hash, was_guest=False): + def register(self, user_id, token, password_hash, + was_guest=False, make_guest=False): """Attempts to register an account. Args: @@ -82,15 +83,18 @@ class RegistrationStore(SQLBaseStore): password_hash (str): Optional. The password hash for this user. was_guest (bool): Optional. Whether this is a guest account being upgraded to a non-guest account. + make_guest (boolean): True if the the new user should be guest, + false to add a regular user account. Raises: StoreError if the user_id could not be registered. """ yield self.runInteraction( "register", - self._register, user_id, token, password_hash, was_guest + self._register, user_id, token, password_hash, was_guest, make_guest ) + self.is_guest.invalidate((user_id,)) - def _register(self, txn, user_id, token, password_hash, was_guest): + def _register(self, txn, user_id, token, password_hash, was_guest, make_guest): now = int(self.clock.time()) next_id = self._access_tokens_id_gen.get_next_txn(txn) @@ -100,12 +104,14 @@ class RegistrationStore(SQLBaseStore): txn.execute("UPDATE users SET" " password_hash = ?," " upgrade_ts = ?" + " is_guest = ?" " WHERE name = ?", - [password_hash, now, user_id]) + [password_hash, now, make_guest, user_id]) else: - txn.execute("INSERT INTO users(name, password_hash, creation_ts) " - "VALUES (?,?,?)", - [user_id, password_hash, now]) + txn.execute("INSERT INTO users " + "(name, password_hash, creation_ts, is_guest) " + "VALUES (?,?,?,?)", + [user_id, password_hash, now, make_guest]) except self.database_engine.module.IntegrityError: raise StoreError( 400, "User ID already taken.", errcode=Codes.USER_IN_USE @@ -126,7 +132,7 @@ class RegistrationStore(SQLBaseStore): keyvalues={ "name": user_id, }, - retcols=["name", "password_hash"], + retcols=["name", "password_hash", "is_guest"], allow_none=True, ) @@ -136,7 +142,7 @@ class RegistrationStore(SQLBaseStore): """ def f(txn): sql = ( - "SELECT name, password_hash FROM users" + "SELECT name, password_hash, is_guest FROM users" " WHERE lower(name) = lower(?)" ) txn.execute(sql, (user_id,)) @@ -249,9 +255,21 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(res if res else False) + @cachedInlineCallbacks() + def is_guest(self, user): + res = yield self._simple_select_one_onecol( + table="users", + keyvalues={"name": user.to_string()}, + retcol="is_guest", + allow_none=True, + desc="is_guest", + ) + + defer.returnValue(res if res else False) + def _query_for_auth(self, txn, token): sql = ( - "SELECT users.name, access_tokens.id as token_id" + "SELECT users.name, users.is_guest, access_tokens.id as token_id" " FROM users" " INNER JOIN access_tokens on users.name = access_tokens.user_id" " WHERE token = ?" diff --git a/synapse/storage/schema/delta/27/event_push_actions.sql b/synapse/storage/schema/delta/28/event_push_actions.sql similarity index 100% rename from synapse/storage/schema/delta/27/event_push_actions.sql rename to synapse/storage/schema/delta/28/event_push_actions.sql From ae1262a241aa816b9e0f19e628afcc83229af64f Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Jan 2016 11:58:20 +0000 Subject: [PATCH 034/294] Add schema change file for is_guest flag --- .../schema/delta/28/users_is_guest.sql | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 synapse/storage/schema/delta/28/users_is_guest.sql diff --git a/synapse/storage/schema/delta/28/users_is_guest.sql b/synapse/storage/schema/delta/28/users_is_guest.sql new file mode 100644 index 000000000..80792e85d --- /dev/null +++ b/synapse/storage/schema/delta/28/users_is_guest.sql @@ -0,0 +1,22 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE users ADD is_guest admin SMALLINT DEFAULT 0 NOT NULL; +/* + * NB: any guest users created between 27 and 28 will be incorrectly + * marked as not guests: we don't bother to fill these in correctly + * because guest access is not really complete in 27 anyway so it's + * very unlikley there will be any guest users created. + */ From 992928304f3c95f87a3297799965159d295432ea Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Jan 2016 11:58:46 +0000 Subject: [PATCH 035/294] Delete notifications for redacted events --- synapse/push/action_generator.py | 7 +++++++ synapse/storage/event_push_actions.py | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index bcd40798f..4cf94f6c6 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -19,6 +19,8 @@ import bulk_push_rule_evaluator import logging +from synapse.api.constants import EventTypes + logger = logging.getLogger(__name__) @@ -34,6 +36,11 @@ class ActionGenerator: @defer.inlineCallbacks def handle_push_actions_for_event(self, event, handler): + if event.type == EventTypes.Redaction and event.redacts is not None: + yield self.store.remove_push_actions_for_event_id( + event.room_id, event.redacts + ) + bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( event.room_id, self.store ) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 0634af6b6..5b44431ab 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -93,6 +93,18 @@ class EventPushActionsStore(SQLBaseStore): ) defer.returnValue(ret) + @defer.inlineCallbacks + def remove_push_actions_for_event_id(self, room_id, event_id): + def f(txn): + txn.execute( + "DELETE FROM event_push_actions WHERE room_id = ? AND event_id = ?", + (room_id, event_id) + ) + yield self.runInteraction( + "remove_push_actions_for_event_id", + f + ) + class EventPushActionsTable(object): table_name = "event_push_actions" From 5880de186b2cd5708189c3a6e9a847174fedd495 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 6 Jan 2016 13:52:16 +0000 Subject: [PATCH 036/294] Log when we skip daily messages --- synapse/storage/events.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index fc5725097..ab500d728 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -936,6 +936,7 @@ class EventsStore(SQLBaseStore): ) now_reporting = self.cursor_to_dict(txn) if not now_reporting: + logger.info("Calculating daily messages skipped; no now_reporting") return None now_reporting = now_reporting[0]["stream_ordering"] @@ -948,11 +949,18 @@ class EventsStore(SQLBaseStore): ) if not last_reported: + logger.info("Calculating daily messages skipped; no last_reported") return None # Close enough to correct for our purposes. yesterday = (now - 24 * 60 * 60) - if math.fabs(yesterday - last_reported[0]["reported_time"]) > 60 * 60: + since_yesterday_seconds = yesterday - last_reported[0]["reported_time"] + any_since_yesterday = math.fabs(since_yesterday_seconds) > 60 * 60 + if any_since_yesterday: + logger.info( + "Calculating daily messages skipped; since_yesterday_seconds: %d" % + (since_yesterday_seconds,) + ) return None txn.execute( @@ -968,6 +976,7 @@ class EventsStore(SQLBaseStore): ) rows = self.cursor_to_dict(txn) if not rows: + logger.info("Calculating daily messages skipped; messages count missing") return None return rows[0]["messages"] From 797691f9085cb4ed5f4575592d25ba6b5376edcf Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 6 Jan 2016 14:04:27 +0000 Subject: [PATCH 037/294] Log on stats scheduling --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0807def6c..f0f92d66b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -718,6 +718,7 @@ def run(hs): if hs.config.report_stats: phone_home_task = task.LoopingCall(phone_stats_home) + logger.info("Scheduling stats reporting for 24 hour intervals") phone_home_task.start(60 * 60 * 24, now=False) def in_thread(): From bf32922e5a4903b16792a3aa7186e46555c7d1a0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 6 Jan 2016 14:13:34 +0000 Subject: [PATCH 038/294] Log when starting stats reporting --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f0f92d66b..49173e95b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -688,6 +688,7 @@ def run(hs): @defer.inlineCallbacks def phone_stats_home(): + logger.info("Gathering stats for reporting") now = int(hs.get_clock().time()) uptime = int(now - start_time) if uptime < 0: From 392773ccb271c8e456c497eec3d3b714f3d0d674 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 6 Jan 2016 16:44:13 +0000 Subject: [PATCH 039/294] Guest users must be joined to a room to see it in /sync --- synapse/api/errors.py | 16 --- synapse/api/filtering.py | 12 -- synapse/handlers/sync.py | 164 +++++++++------------------ synapse/rest/client/v2_alpha/sync.py | 7 +- 4 files changed, 56 insertions(+), 143 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 8bc7b9e6d..d4037b3d5 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -120,22 +120,6 @@ class AuthError(SynapseError): super(AuthError, self).__init__(*args, **kwargs) -class GuestAccessError(AuthError): - """An error raised when a there is a problem with a guest user accessing - a room""" - - def __init__(self, rooms, *args, **kwargs): - self.rooms = rooms - super(GuestAccessError, self).__init__(*args, **kwargs) - - def error_dict(self): - return cs_error( - self.msg, - self.errcode, - rooms=self.rooms, - ) - - class EventSizeError(SynapseError): """An error raised when an event is too big.""" diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 5287aaa75..4390d01e3 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -149,9 +149,6 @@ class FilterCollection(object): "include_leave", False ) - def list_rooms(self): - return self.room_filter.list_rooms() - def timeline_limit(self): return self.room_timeline_filter.limit() @@ -184,15 +181,6 @@ class Filter(object): def __init__(self, filter_json): self.filter_json = filter_json - def list_rooms(self): - """The list of room_id strings this filter restricts the output to - or None if the this filter doesn't list the room ids. - """ - if "rooms" in self.filter_json: - return list(set(self.filter_json["rooms"])) - else: - return None - def check(self, event): """Checks whether the filter matches the given event. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b63a27b38..fdccb5223 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -15,8 +15,8 @@ from ._base import BaseHandler +from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes -from synapse.api.errors import GuestAccessError from synapse.util import unwrapFirstError from twisted.internet import defer @@ -29,8 +29,8 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "is_guest", "filter", + "is_guest", ]) @@ -118,8 +118,6 @@ class SyncResult(collections.namedtuple("SyncResult", [ self.presence or self.joined or self.invited or self.archived ) -GuestRoom = collections.namedtuple("GuestRoom", ("room_id", "membership")) - class SyncHandler(BaseHandler): @@ -138,18 +136,6 @@ class SyncHandler(BaseHandler): A Deferred SyncResult. """ - if sync_config.is_guest: - bad_rooms = [] - for room_id in sync_config.filter.list_rooms(): - world_readable = yield self._is_world_readable(room_id) - if not world_readable: - bad_rooms.append(room_id) - - if bad_rooms: - raise GuestAccessError( - bad_rooms, 403, "Guest access not allowed" - ) - if timeout == 0 or since_token is None or full_state: # we are going to return immediately, so don't bother calling # notifier.wait_for_events. @@ -166,17 +152,6 @@ class SyncHandler(BaseHandler): ) defer.returnValue(result) - @defer.inlineCallbacks - def _is_world_readable(self, room_id): - state = yield self.hs.get_state_handler().get_current_state( - room_id, - EventTypes.RoomHistoryVisibility - ) - if state and "history_visibility" in state.content: - defer.returnValue(state.content["history_visibility"] == "world_readable") - else: - defer.returnValue(False) - def current_sync_for_user(self, sync_config, since_token=None, full_state=False): """Get the sync for client needed to match what the server has now. @@ -200,52 +175,37 @@ class SyncHandler(BaseHandler): """ now_token = yield self.event_sources.get_current_token() - if sync_config.is_guest: - room_list = [ - GuestRoom(room_id, Membership.JOIN) - for room_id in sync_config.filter.list_rooms() - ] - - account_data = {} - account_data_by_room = {} - tags_by_room = {} - - else: - membership_list = (Membership.INVITE, Membership.JOIN) - if sync_config.filter.include_leave: - membership_list += (Membership.LEAVE, Membership.BAN) - - room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=sync_config.user.to_string(), - membership_list=membership_list - ) - - account_data, account_data_by_room = ( - yield self.store.get_account_data_for_user( - sync_config.user.to_string() - ) - ) - - tags_by_room = yield self.store.get_tags_for_user( - sync_config.user.to_string() - ) - - presence_stream = self.event_sources.sources["presence"] - - joined_room_ids = [ - room.room_id for room in room_list - if room.membership == Membership.JOIN - ] - - presence, _ = yield presence_stream.get_new_events( - from_key=0, - user=sync_config.user, - room_ids=joined_room_ids, - is_guest=sync_config.is_guest, + now_token, ephemeral_by_room = yield self.ephemeral_by_room( + sync_config, now_token ) - now_token, ephemeral_by_room = yield self.ephemeral_by_room( - sync_config, now_token, joined_room_ids + presence_stream = self.event_sources.sources["presence"] + # TODO (mjark): This looks wrong, shouldn't we be getting the presence + # UP to the present rather than after the present? + pagination_config = PaginationConfig(from_token=now_token) + presence, _ = yield presence_stream.get_pagination_rows( + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None + ) + + membership_list = (Membership.INVITE, Membership.JOIN) + if sync_config.filter.include_leave: + membership_list += (Membership.LEAVE, Membership.BAN) + + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=sync_config.user.to_string(), + membership_list=membership_list + ) + + account_data, account_data_by_room = ( + yield self.store.get_account_data_for_user( + sync_config.user.to_string() + ) + ) + + tags_by_room = yield self.store.get_tags_for_user( + sync_config.user.to_string() ) joined = [] @@ -356,13 +316,11 @@ class SyncHandler(BaseHandler): return account_data_events @defer.inlineCallbacks - def ephemeral_by_room(self, sync_config, now_token, room_ids, - since_token=None): + def ephemeral_by_room(self, sync_config, now_token, since_token=None): """Get the ephemeral events for each room the user is in Args: sync_config (SyncConfig): The flags, filters and user for the sync. now_token (StreamToken): Where the server is currently up to. - room_ids (list): List of room id strings to get data for. since_token (StreamToken): Where the server was when the client last synced. Returns: @@ -373,6 +331,9 @@ class SyncHandler(BaseHandler): typing_key = since_token.typing_key if since_token else "0" + rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string()) + room_ids = [room.room_id for room in rooms] + typing_source = self.event_sources.sources["typing"] typing, typing_key = yield typing_source.get_new_events( user=sync_config.user, @@ -450,38 +411,8 @@ class SyncHandler(BaseHandler): """ now_token = yield self.event_sources.get_current_token() - if sync_config.is_guest: - room_ids = sync_config.filter.list_rooms() - - tags_by_room = {} - account_data = {} - account_data_by_room = {} - - else: - rooms = yield self.store.get_rooms_for_user( - sync_config.user.to_string() - ) - room_ids = [room.room_id for room in rooms] - - now_token, ephemeral_by_room = yield self.ephemeral_by_room( - sync_config, now_token, since_token - ) - - tags_by_room = yield self.store.get_updated_tags( - sync_config.user.to_string(), - since_token.account_data_key, - ) - - account_data, account_data_by_room = ( - yield self.store.get_updated_account_data_for_user( - sync_config.user.to_string(), - since_token.account_data_key, - ) - ) - - now_token, ephemeral_by_room = yield self.ephemeral_by_room( - sync_config, now_token, room_ids, since_token - ) + rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string()) + room_ids = [room.room_id for room in rooms] presence_source = self.event_sources.sources["presence"] presence, presence_key = yield presence_source.get_new_events( @@ -489,10 +420,15 @@ class SyncHandler(BaseHandler): from_key=since_token.presence_key, limit=sync_config.filter.presence_limit(), room_ids=room_ids, - is_guest=sync_config.is_guest, + # /sync doesn't support guest access, they can't get to this point in code + is_guest=False, ) now_token = now_token.copy_and_replace("presence_key", presence_key) + now_token, ephemeral_by_room = yield self.ephemeral_by_room( + sync_config, now_token, since_token + ) + rm_handler = self.hs.get_handlers().room_member_handler app_service = yield self.store.get_app_service_by_user_id( sync_config.user.to_string() @@ -512,8 +448,18 @@ class SyncHandler(BaseHandler): from_key=since_token.room_key, to_key=now_token.room_key, limit=timeline_limit + 1, - room_ids=room_ids if sync_config.is_guest else (), - is_guest=sync_config.is_guest, + ) + + tags_by_room = yield self.store.get_updated_tags( + sync_config.user.to_string(), + since_token.account_data_key, + ) + + account_data, account_data_by_room = ( + yield self.store.get_updated_account_data_for_user( + sync_config.user.to_string(), + since_token.account_data_key, + ) ) joined = [] diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 35a70ffad..dc3e6f70b 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -120,15 +120,10 @@ class SyncRestServlet(RestServlet): except: filter = FilterCollection({}) - if is_guest and filter.list_rooms() is None: - raise SynapseError( - 400, "Guest users must provide a list of rooms in the filter" - ) - sync_config = SyncConfig( user=user, - is_guest=is_guest, filter=filter, + is_guest=is_guest, ) if since is not None: From 0e48f7f2458f08341131b3b90c78b7034fe02d14 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Jan 2016 16:46:41 +0000 Subject: [PATCH 040/294] fix tests --- tests/handlers/test_federation.py | 4 ++++ tests/handlers/test_room.py | 5 +++++ tests/storage/test_registration.py | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 6acc4ebad..029c09411 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -53,6 +53,8 @@ class FederationTestCase(unittest.TestCase): "bulk_get_push_rules", "get_current_state", "set_push_actions_for_event_and_users", + "is_guest", + "get_state_for_events", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -73,6 +75,8 @@ class FederationTestCase(unittest.TestCase): self.handlers.federation_handler = FederationHandler(self.hs) + self.datastore.get_state_for_events.return_value = {"$a:b": {}} + @defer.inlineCallbacks def test_msg(self): pdu = FrozenEvent({ diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index ff2b59712..b1c8e6152 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -47,6 +47,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "bulk_get_push_rules", "get_current_state", "set_push_actions_for_event_and_users", + "get_state_for_events", + "is_guest", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -116,6 +118,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): defer.succeed([]) ) self.datastore.get_current_state.return_value = {} + self.datastore.get_state_for_events = lambda event_ids,types: {x: {} for x in event_ids} def annotate(_): ctx = Mock() @@ -198,6 +201,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): defer.succeed([]) ) self.datastore.get_current_state.return_value = {} + self.datastore.get_state_for_events = lambda event_ids,types: {x: {} for x in event_ids} def annotate(_): ctx = Mock() @@ -274,6 +278,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): defer.succeed([]) ) self.datastore.get_current_state.return_value = {} + self.datastore.get_state_for_events = lambda event_ids,types: {x: {} for x in event_ids} def annotate(_): ctx = Mock() diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 0cce6c37d..4760131f9 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -45,7 +45,7 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertEquals( # TODO(paul): Surely this field should be 'user_id', not 'name' # Additionally surely it shouldn't come in a 1-element list - {"name": self.user_id, "password_hash": self.pwhash}, + {"name": self.user_id, "password_hash": self.pwhash, "is_guest": 0}, (yield self.store.get_user_by_id(self.user_id)) ) From 4cec90a26061748602d244604b34b4df67087eb2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 6 Jan 2016 16:54:57 +0000 Subject: [PATCH 041/294] Pass whether the user was a guest to some of the event streams --- synapse/handlers/sync.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index fdccb5223..5428864bb 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -340,7 +340,7 @@ class SyncHandler(BaseHandler): from_key=typing_key, limit=sync_config.filter.ephemeral_limit(), room_ids=room_ids, - is_guest=False, + is_guest=sync_config.is_guest, ) now_token = now_token.copy_and_replace("typing_key", typing_key) @@ -363,8 +363,7 @@ class SyncHandler(BaseHandler): from_key=receipt_key, limit=sync_config.filter.ephemeral_limit(), room_ids=room_ids, - # /sync doesn't support guest access, they can't get to this point in code - is_guest=False, + is_guest=sync_config.is_guest, ) now_token = now_token.copy_and_replace("receipt_key", receipt_key) @@ -420,8 +419,7 @@ class SyncHandler(BaseHandler): from_key=since_token.presence_key, limit=sync_config.filter.presence_limit(), room_ids=room_ids, - # /sync doesn't support guest access, they can't get to this point in code - is_guest=False, + is_guest=sync_config.is_guest, ) now_token = now_token.copy_and_replace("presence_key", presence_key) From b6a585348ae8a07dc8105242e182435a240e6b8f Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Jan 2016 17:16:02 +0000 Subject: [PATCH 042/294] Adding is_guest here won't work because it just constructs a dict of uid -> password hash --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c79066f77..a52b67013 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -142,7 +142,7 @@ class RegistrationStore(SQLBaseStore): """ def f(txn): sql = ( - "SELECT name, password_hash, is_guest FROM users" + "SELECT name, password_hash FROM users" " WHERE lower(name) = lower(?)" ) txn.execute(sql, (user_id,)) From 09dc9854cd8f28b2e5ce90207cfc822d226ec2ad Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Jan 2016 17:44:10 +0000 Subject: [PATCH 043/294] comma style --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 1799a668c..ba26d13d4 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -119,7 +119,7 @@ class RegistrationHandler(BaseHandler): token=token, password_hash=password_hash, was_guest=guest_access_token is not None, - make_guest=make_guest + make_guest=make_guest, ) yield registered_user(self.distributor, user) From 11a974da21f5a8e815e647c212796755414001ca Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 6 Jan 2016 18:08:52 +0000 Subject: [PATCH 044/294] Add /_matrix/versions to report supported versions --- synapse/app/homeserver.py | 1 + synapse/rest/__init__.py | 6 ++++++ synapse/rest/client/versions.py | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 synapse/rest/client/versions.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 49173e95b..bbf8bee8e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -185,6 +185,7 @@ class SynapseHomeServer(HomeServer): "/_matrix/client/r0": client_resource, "/_matrix/client/unstable": client_resource, "/_matrix/client/v2_alpha": client_resource, + "/_matrix/client/versions": client_resource, }) if name == "federation": diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 7b67e9620..43ef29db7 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.rest.client import ( + versions, +) + from synapse.rest.client.v1 import ( room, events, @@ -53,6 +57,8 @@ class ClientRestResource(JsonResource): @staticmethod def register_servlets(client_resource, hs): + versions.register_servlets(client_resource) + # "v1" room.register_servlets(hs, client_resource) events.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py new file mode 100644 index 000000000..349ef6b39 --- /dev/null +++ b/synapse/rest/client/versions.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.http.servlet import RestServlet + +import logging +import re + +logger = logging.getLogger(__name__) + + +class VersionsRestServlet(RestServlet): + PATTERNS = [re.compile("^/_matrix/client/versions$")] + + def on_GET(self, request): + return (200, { + "versions": [ + "r0.0.1", + ] + }) + + +def register_servlets(http_server): + VersionsRestServlet().register(http_server) From 4106477e7f584e355ece46bae96d994e6ebc2b41 Mon Sep 17 00:00:00 2001 From: Robin Lambertz Date: Wed, 6 Jan 2016 23:19:33 +0100 Subject: [PATCH 045/294] Config Comment mixup in captcha public/private key --- synapse/config/captcha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index dd92fcd0d..f3300075b 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -29,10 +29,10 @@ class CaptchaConfig(Config): ## Captcha ## # This Home Server's ReCAPTCHA public key. - recaptcha_private_key: "YOUR_PRIVATE_KEY" + recaptcha_public_key: "YOUR_PUBLIC_KEY" # This Home Server's ReCAPTCHA private key. - recaptcha_public_key: "YOUR_PUBLIC_KEY" + recaptcha_private_key: "YOUR_PRIVATE_KEY" # Enables ReCaptcha checks when registering, preventing signup # unless a captcha is answered. Requires a valid ReCaptcha From 6c28ac260c2ce4bf93737e53ea3297bff08924c7 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 7 Jan 2016 04:26:29 +0000 Subject: [PATCH 046/294] copyrights --- contrib/cmdclient/console.py | 2 +- contrib/cmdclient/http.py | 2 +- contrib/experiments/cursesio.py | 2 +- contrib/experiments/test_messaging.py | 2 +- contrib/graph/graph.py | 2 +- contrib/graph/graph2.py | 2 +- scripts-dev/copyrighter-sql.pl | 4 ++-- scripts-dev/copyrighter.pl | 4 ++-- scripts/register_new_matrix_user | 2 +- scripts/synapse_port_db | 2 +- setup.py | 2 +- synapse/__init__.py | 2 +- synapse/api/__init__.py | 2 +- synapse/api/constants.py | 2 +- synapse/api/errors.py | 2 +- synapse/api/filtering.py | 2 +- synapse/api/ratelimiting.py | 2 +- synapse/api/urls.py | 2 +- synapse/app/__init__.py | 2 +- synapse/app/homeserver.py | 2 +- synapse/app/synctl.py | 2 +- synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 2 +- synapse/appservice/scheduler.py | 2 +- synapse/config/__init__.py | 2 +- synapse/config/__main__.py | 2 +- synapse/config/_base.py | 2 +- synapse/config/appservice.py | 2 +- synapse/config/captcha.py | 2 +- synapse/config/cas.py | 2 +- synapse/config/database.py | 2 +- synapse/config/homeserver.py | 2 +- synapse/config/key.py | 2 +- synapse/config/logger.py | 2 +- synapse/config/metrics.py | 2 +- synapse/config/password.py | 2 +- synapse/config/ratelimiting.py | 2 +- synapse/config/registration.py | 2 +- synapse/config/server.py | 2 +- synapse/config/tls.py | 2 +- synapse/config/voip.py | 2 +- synapse/crypto/__init__.py | 2 +- synapse/crypto/context_factory.py | 2 +- synapse/crypto/event_signing.py | 2 +- synapse/crypto/keyclient.py | 2 +- synapse/crypto/keyring.py | 2 +- synapse/events/__init__.py | 2 +- synapse/events/builder.py | 2 +- synapse/events/snapshot.py | 2 +- synapse/events/utils.py | 2 +- synapse/events/validator.py | 2 +- synapse/federation/__init__.py | 2 +- synapse/federation/federation_base.py | 2 +- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 2 +- synapse/federation/persistence.py | 2 +- synapse/federation/replication.py | 2 +- synapse/federation/transaction_queue.py | 2 +- synapse/federation/transport/__init__.py | 2 +- synapse/federation/transport/client.py | 2 +- synapse/federation/transport/server.py | 2 +- synapse/federation/units.py | 2 +- synapse/handlers/__init__.py | 2 +- synapse/handlers/account_data.py | 2 +- synapse/handlers/admin.py | 2 +- synapse/handlers/appservice.py | 2 +- synapse/handlers/directory.py | 2 +- synapse/handlers/events.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/handlers/identity.py | 2 +- synapse/handlers/presence.py | 2 +- synapse/handlers/profile.py | 2 +- synapse/handlers/receipts.py | 2 +- synapse/handlers/search.py | 2 +- synapse/handlers/typing.py | 2 +- synapse/http/__init__.py | 2 +- synapse/http/client.py | 2 +- synapse/http/endpoint.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/http/server.py | 2 +- synapse/http/servlet.py | 2 +- synapse/metrics/__init__.py | 2 +- synapse/metrics/metric.py | 2 +- synapse/metrics/resource.py | 2 +- synapse/push/__init__.py | 2 +- synapse/push/baserules.py | 2 +- synapse/push/httppusher.py | 2 +- synapse/push/push_rule_evaluator.py | 2 +- synapse/push/pusherpool.py | 2 +- synapse/push/rulekinds.py | 2 +- synapse/python_dependencies.py | 2 +- synapse/rest/__init__.py | 2 +- synapse/rest/client/__init__.py | 2 +- synapse/rest/client/v1/__init__.py | 2 +- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/base.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v1/events.py | 2 +- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/login.py | 2 +- synapse/rest/client/v1/presence.py | 2 +- synapse/rest/client/v1/profile.py | 2 +- synapse/rest/client/v1/push_rule.py | 2 +- synapse/rest/client/v1/pusher.py | 2 +- synapse/rest/client/v1/register.py | 2 +- synapse/rest/client/v1/room.py | 2 +- synapse/rest/client/v1/transactions.py | 2 +- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/client/v2_alpha/__init__.py | 2 +- synapse/rest/client/v2_alpha/_base.py | 2 +- synapse/rest/client/v2_alpha/account.py | 2 +- synapse/rest/client/v2_alpha/account_data.py | 2 +- synapse/rest/client/v2_alpha/auth.py | 2 +- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/rest/client/v2_alpha/keys.py | 2 +- synapse/rest/client/v2_alpha/receipts.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/client/v2_alpha/tags.py | 2 +- synapse/rest/client/v2_alpha/tokenrefresh.py | 2 +- synapse/rest/key/__init__.py | 2 +- synapse/rest/key/v1/__init__.py | 2 +- synapse/rest/key/v1/server_key_resource.py | 2 +- synapse/rest/key/v2/__init__.py | 2 +- synapse/rest/key/v2/local_key_resource.py | 2 +- synapse/rest/key/v2/remote_key_resource.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/__init__.py | 2 +- synapse/rest/media/v1/base_resource.py | 2 +- synapse/rest/media/v1/download_resource.py | 2 +- synapse/rest/media/v1/filepath.py | 2 +- synapse/rest/media/v1/identicon_resource.py | 2 +- synapse/rest/media/v1/media_repository.py | 2 +- synapse/rest/media/v1/thumbnailer.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- synapse/server.py | 2 +- synapse/state.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/account_data.py | 2 +- synapse/storage/appservice.py | 2 +- synapse/storage/background_updates.py | 2 +- synapse/storage/directory.py | 2 +- synapse/storage/end_to_end_keys.py | 2 +- synapse/storage/engines/__init__.py | 2 +- synapse/storage/engines/_base.py | 2 +- synapse/storage/engines/postgres.py | 2 +- synapse/storage/engines/sqlite3.py | 2 +- synapse/storage/event_federation.py | 2 +- synapse/storage/events.py | 2 +- synapse/storage/filtering.py | 2 +- synapse/storage/keys.py | 2 +- synapse/storage/media_repository.py | 2 +- synapse/storage/presence.py | 2 +- synapse/storage/profile.py | 2 +- synapse/storage/push_rule.py | 2 +- synapse/storage/pusher.py | 2 +- synapse/storage/receipts.py | 2 +- synapse/storage/rejections.py | 2 +- synapse/storage/room.py | 2 +- synapse/storage/roommember.py | 2 +- synapse/storage/schema/delta/11/v11.sql | 2 +- synapse/storage/schema/delta/12/v12.sql | 2 +- synapse/storage/schema/delta/13/v13.sql | 2 +- synapse/storage/schema/delta/14/upgrade_appservice_db.py | 2 +- synapse/storage/schema/delta/14/v14.sql | 2 +- synapse/storage/schema/delta/15/appservice_txns.sql | 2 +- synapse/storage/schema/delta/17/drop_indexes.sql | 2 +- synapse/storage/schema/delta/17/server_keys.sql | 2 +- synapse/storage/schema/delta/18/server_keys_bigger_ints.sql | 2 +- synapse/storage/schema/delta/19/event_index.sql | 2 +- synapse/storage/schema/delta/20/pushers.py | 2 +- synapse/storage/schema/delta/21/end_to_end_keys.sql | 2 +- synapse/storage/schema/delta/21/receipts.sql | 2 +- synapse/storage/schema/delta/22/receipts_index.sql | 2 +- synapse/storage/schema/delta/23/drop_state_index.sql | 2 +- synapse/storage/schema/delta/23/refresh_tokens.sql | 2 +- synapse/storage/schema/delta/24/stats_reporting.sql | 2 +- synapse/storage/schema/delta/25/00background_updates.sql | 2 +- synapse/storage/schema/delta/25/fts.py | 2 +- synapse/storage/schema/delta/25/guest_access.sql | 2 +- synapse/storage/schema/delta/25/history_visibility.sql | 2 +- synapse/storage/schema/delta/25/tags.sql | 2 +- synapse/storage/schema/delta/26/account_data.sql | 2 +- synapse/storage/schema/delta/27/account_data.sql | 2 +- synapse/storage/schema/delta/27/forgotten_memberships.sql | 2 +- synapse/storage/schema/delta/27/ts.py | 2 +- synapse/storage/schema/full_schemas/11/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/11/event_signatures.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 2 +- synapse/storage/schema/full_schemas/11/keys.sql | 2 +- synapse/storage/schema/full_schemas/11/media_repository.sql | 2 +- synapse/storage/schema/full_schemas/11/presence.sql | 2 +- synapse/storage/schema/full_schemas/11/profiles.sql | 2 +- synapse/storage/schema/full_schemas/11/redactions.sql | 2 +- synapse/storage/schema/full_schemas/11/room_aliases.sql | 2 +- synapse/storage/schema/full_schemas/11/state.sql | 2 +- synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- synapse/storage/schema/full_schemas/11/users.sql | 2 +- .../storage/schema/full_schemas/16/application_services.sql | 2 +- synapse/storage/schema/full_schemas/16/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/16/event_signatures.sql | 2 +- synapse/storage/schema/full_schemas/16/im.sql | 2 +- synapse/storage/schema/full_schemas/16/keys.sql | 2 +- synapse/storage/schema/full_schemas/16/media_repository.sql | 2 +- synapse/storage/schema/full_schemas/16/presence.sql | 2 +- synapse/storage/schema/full_schemas/16/profiles.sql | 2 +- synapse/storage/schema/full_schemas/16/push.sql | 2 +- synapse/storage/schema/full_schemas/16/redactions.sql | 2 +- synapse/storage/schema/full_schemas/16/room_aliases.sql | 2 +- synapse/storage/schema/full_schemas/16/state.sql | 2 +- synapse/storage/schema/full_schemas/16/transactions.sql | 2 +- synapse/storage/schema/full_schemas/16/users.sql | 2 +- synapse/storage/schema/schema_version.sql | 2 +- synapse/storage/search.py | 2 +- synapse/storage/signatures.py | 2 +- synapse/storage/state.py | 2 +- synapse/storage/stream.py | 2 +- synapse/storage/tags.py | 2 +- synapse/storage/transactions.py | 2 +- synapse/storage/util/__init__.py | 2 +- synapse/storage/util/id_generators.py | 2 +- synapse/streams/__init__.py | 2 +- synapse/streams/config.py | 2 +- synapse/streams/events.py | 2 +- synapse/types.py | 2 +- synapse/util/__init__.py | 2 +- synapse/util/async.py | 2 +- synapse/util/caches/__init__.py | 2 +- synapse/util/caches/descriptors.py | 2 +- synapse/util/caches/dictionary_cache.py | 2 +- synapse/util/caches/expiringcache.py | 2 +- synapse/util/caches/lrucache.py | 2 +- synapse/util/caches/snapshot_cache.py | 2 +- synapse/util/debug.py | 2 +- synapse/util/distributor.py | 2 +- synapse/util/frozenutils.py | 2 +- synapse/util/jsonobject.py | 2 +- synapse/util/logcontext.py | 2 +- synapse/util/logutils.py | 2 +- synapse/util/ratelimitutils.py | 2 +- synapse/util/retryutils.py | 2 +- synapse/util/stringutils.py | 2 +- tests/__init__.py | 2 +- tests/api/test_filtering.py | 2 +- tests/appservice/__init__.py | 2 +- tests/appservice/test_appservice.py | 2 +- tests/appservice/test_scheduler.py | 2 +- tests/crypto/__init__.py | 2 +- tests/crypto/test_event_signing.py | 2 +- tests/events/test_utils.py | 2 +- tests/federation/test_federation.py | 2 +- tests/handlers/test_appservice.py | 2 +- tests/handlers/test_auth.py | 2 +- tests/handlers/test_directory.py | 2 +- tests/handlers/test_federation.py | 2 +- tests/handlers/test_presence.py | 2 +- tests/handlers/test_presencelike.py | 2 +- tests/handlers/test_profile.py | 2 +- tests/handlers/test_room.py | 2 +- tests/handlers/test_typing.py | 2 +- tests/metrics/test_metric.py | 2 +- tests/rest/__init__.py | 2 +- tests/rest/client/__init__.py | 2 +- tests/rest/client/v1/__init__.py | 2 +- tests/rest/client/v1/test_events.py | 2 +- tests/rest/client/v1/test_presence.py | 2 +- tests/rest/client/v1/test_profile.py | 2 +- tests/rest/client/v1/test_rooms.py | 2 +- tests/rest/client/v1/test_typing.py | 2 +- tests/rest/client/v1/utils.py | 2 +- tests/rest/client/v2_alpha/__init__.py | 2 +- tests/rest/client/v2_alpha/test_filter.py | 2 +- tests/storage/event_injector.py | 2 +- tests/storage/test__base.py | 2 +- tests/storage/test_appservice.py | 2 +- tests/storage/test_base.py | 2 +- tests/storage/test_directory.py | 2 +- tests/storage/test_events.py | 2 +- tests/storage/test_presence.py | 2 +- tests/storage/test_profile.py | 2 +- tests/storage/test_redaction.py | 2 +- tests/storage/test_registration.py | 2 +- tests/storage/test_room.py | 2 +- tests/storage/test_roommember.py | 2 +- tests/storage/test_stream.py | 2 +- tests/test_distributor.py | 2 +- tests/test_state.py | 2 +- tests/test_test_utils.py | 2 +- tests/test_types.py | 2 +- tests/unittest.py | 2 +- tests/util/__init__.py | 2 +- tests/util/test_dict_cache.py | 2 +- tests/util/test_lrucache.py | 2 +- tests/util/test_snapshot_cache.py | 2 +- tests/utils.py | 2 +- 295 files changed, 297 insertions(+), 297 deletions(-) diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py index d9c6ec6a7..8bb03ce66 100755 --- a/contrib/cmdclient/console.py +++ b/contrib/cmdclient/console.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2014 OpenMarket Ltd +# Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py index 869f782ec..418689731 100644 --- a/contrib/cmdclient/http.py +++ b/contrib/cmdclient/http.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/contrib/experiments/cursesio.py b/contrib/experiments/cursesio.py index 95d87a1fd..44afe8100 100644 --- a/contrib/experiments/cursesio.py +++ b/contrib/experiments/cursesio.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/contrib/experiments/test_messaging.py b/contrib/experiments/test_messaging.py index fedf786ce..85c9c1198 100644 --- a/contrib/experiments/test_messaging.py +++ b/contrib/experiments/test_messaging.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index b2acadcf5..afd1d446b 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index d0d2cfe7c..1ccad6572 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts-dev/copyrighter-sql.pl b/scripts-dev/copyrighter-sql.pl index 890e51e58..13e630fc1 100755 --- a/scripts-dev/copyrighter-sql.pl +++ b/scripts-dev/copyrighter-sql.pl @@ -1,5 +1,5 @@ #!/usr/bin/perl -pi -# Copyright 2015 OpenMarket Ltd +# Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ # limitations under the License. $copyright = < Date: Thu, 7 Jan 2016 10:02:47 +0000 Subject: [PATCH 047/294] more commas --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 63d65b446..ce244fa95 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -42,7 +42,7 @@ def evaluator_for_room_id(room_id, store): uid: baserules.list_with_base_rules( [decode_rule_json(rule_list) for rule_list in rules_by_user[uid]] if uid in rules_by_user else [], - UserID.from_string(uid) + UserID.from_string(uid), ) for uid in users } From daadcf36c0bad93220d4fa422dce0c5740f63e3d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 7 Jan 2016 10:15:35 +0000 Subject: [PATCH 048/294] This comma is actually important --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a52b67013..ece71f2ee 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -103,7 +103,7 @@ class RegistrationStore(SQLBaseStore): if was_guest: txn.execute("UPDATE users SET" " password_hash = ?," - " upgrade_ts = ?" + " upgrade_ts = ?," " is_guest = ?" " WHERE name = ?", [password_hash, now, make_guest, user_id]) From c9ae1d1ee57648eb02ccb5c1585bf77a7ffa83ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Jan 2016 13:57:04 +0000 Subject: [PATCH 049/294] Change manhole to use ssh --- synapse/app/homeserver.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0807def6c..2c0b6d7b4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -42,6 +42,12 @@ from synapse.storage.prepare_database import UpgradeDatabaseException from synapse.server import HomeServer +from twisted.conch.manhole import ColoredManhole +from twisted.conch.insults import insults +from twisted.conch import manhole_ssh +from twisted.cred import checkers, portal + + from twisted.internet import reactor, task, defer from twisted.application import service from twisted.enterprise import adbapi @@ -67,7 +73,6 @@ from synapse.metrics.resource import MetricsResource, METRICS_PREFIX from synapse import events from daemonize import Daemonize -import twisted.manhole.telnet import synapse @@ -248,10 +253,21 @@ class SynapseHomeServer(HomeServer): if listener["type"] == "http": self._listener_http(config, listener) elif listener["type"] == "manhole": - f = twisted.manhole.telnet.ShellFactory() - f.username = "matrix" - f.password = "rabbithole" - f.namespace['hs'] = self + checker = checkers.InMemoryUsernamePasswordDatabaseDontUse( + matrix="rabbithole" + ) + + rlm = manhole_ssh.TerminalRealm() + rlm.chainedProtocolFactory = lambda: insults.ServerProtocol( + ColoredManhole, + { + "__name__": "__console__", + "hs": self, + } + ) + + f = manhole_ssh.ConchFactory(portal.Portal(rlm, [checker])) + reactor.listenTCP( listener["port"], f, From 5dc5e29b9cd688af5c71d6b465ccac3c8e1d3c14 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Jan 2016 14:02:57 +0000 Subject: [PATCH 050/294] s/telnet/ssh/ --- synapse/config/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 187edd516..ed0e8c9fe 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -200,7 +200,7 @@ class ServerConfig(Config): - names: [federation] compress: false - # Turn on the twisted telnet manhole service on localhost on the given + # Turn on the twisted ssh manhole service on localhost on the given # port. # - port: 9000 # bind_address: 127.0.0.1 From fcbe63eaade67ace5ffc54b6723bf055b54cd5d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Jan 2016 15:28:17 +0000 Subject: [PATCH 051/294] Use logger not logging --- synapse/handlers/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5428864bb..3bc18b433 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -660,7 +660,7 @@ class SyncHandler(BaseHandler): ), ) - logging.debug("Room sync: %r", room_sync) + logger.debug("Room sync: %r", room_sync) defer.returnValue(room_sync) @@ -709,7 +709,7 @@ class SyncHandler(BaseHandler): ), ) - logging.debug("Room sync: %r", room_sync) + logger.debug("Room sync: %r", room_sync) defer.returnValue(room_sync) From 33bef689c1c712776d4277f5a71009090b9d6029 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Jan 2016 15:28:40 +0000 Subject: [PATCH 052/294] Turn on core dumps --- synapse/app/homeserver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 14ce9cd80..58a4c812f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -375,10 +375,13 @@ def change_resource_limit(soft_file_no): soft_file_no = hard resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) - logger.info("Set file limit to: %d", soft_file_no) + + resource.setrlimit( + resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY) + ) except (ValueError, resource.error) as e: - logger.warn("Failed to set file limit: %s", e) + logger.warn("Failed to set file or core limit: %s", e) def setup(config_options): From 8677b7d698bf3d3b82e2bcfa783e3b55aa4246e2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 7 Jan 2016 18:57:15 +0000 Subject: [PATCH 053/294] Only use cropped thumbnails when asked for a cropped thumbnail. Even though ones cropped with scale might be technically valid. --- synapse/rest/media/v1/thumbnail_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index c18160534..ab5249978 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -253,7 +253,7 @@ class ThumbnailResource(BaseMediaResource): t_w = info["thumbnail_width"] t_h = info["thumbnail_height"] t_method = info["thumbnail_method"] - if t_method == "scale" or t_method == "crop": + if t_method == "crop": aspect_quality = abs(d_w * t_h - d_h * t_w) min_quality = 0 if d_w <= t_w and d_h <= t_h else 1 size_quality = abs((d_w - t_w) * (d_h - t_h)) From 8f8b884430d29e30688becf4bc38e48014a25851 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 Jan 2016 17:48:08 +0000 Subject: [PATCH 054/294] Don't log urlencoded access_tokens --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 58a4c812f..bafac884a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -496,8 +496,8 @@ class SynapseRequest(Request): def get_redacted_uri(self): return re.sub( - r'(\?.*access_token=)[^&]*(.*)$', - r'\1\2', + r'(\?.*accesss(_|%5[Ff])token=)[^&]*(.*)$', + r'\1\3', self.uri ) From fe56138142f386c927e0221414f223428d7c000d Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 11 Jan 2016 17:09:03 +0000 Subject: [PATCH 055/294] Remove rogue 'admin' --- synapse/storage/schema/delta/28/users_is_guest.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/28/users_is_guest.sql b/synapse/storage/schema/delta/28/users_is_guest.sql index 80792e85d..21d2b420b 100644 --- a/synapse/storage/schema/delta/28/users_is_guest.sql +++ b/synapse/storage/schema/delta/28/users_is_guest.sql @@ -13,7 +13,7 @@ * limitations under the License. */ -ALTER TABLE users ADD is_guest admin SMALLINT DEFAULT 0 NOT NULL; +ALTER TABLE users ADD is_guest SMALLINT DEFAULT 0 NOT NULL; /* * NB: any guest users created between 27 and 28 will be incorrectly * marked as not guests: we don't bother to fill these in correctly From b5d33a656f7045caa946d4bb25990a8c704dbb29 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 11 Jan 2016 17:13:52 +0000 Subject: [PATCH 056/294] Postgres doesn't like booleans --- synapse/storage/registration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index ece71f2ee..999b710fb 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -106,12 +106,12 @@ class RegistrationStore(SQLBaseStore): " upgrade_ts = ?," " is_guest = ?" " WHERE name = ?", - [password_hash, now, make_guest, user_id]) + [password_hash, now, 1 if make_guest else 0, user_id]) else: txn.execute("INSERT INTO users " "(name, password_hash, creation_ts, is_guest) " "VALUES (?,?,?,?)", - [user_id, password_hash, now, make_guest]) + [user_id, password_hash, now, 1 if make_guest else 0]) except self.database_engine.module.IntegrityError: raise StoreError( 400, "User ID already taken.", errcode=Codes.USER_IN_USE From 2110e35fd6e50aa7c44410601c7e5e938c912c3e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 11 Jan 2016 15:29:57 +0000 Subject: [PATCH 057/294] Introduce a Requester object This tracks data about the entity which made the request. This is instead of passing around a tuple, which requires call-site modifications every time a new piece of optional context is passed around. I tried to introduce a User object. I gave up. --- synapse/api/auth.py | 8 +- synapse/rest/client/v1/admin.py | 5 +- synapse/rest/client/v1/directory.py | 8 +- synapse/rest/client/v1/events.py | 19 +-- synapse/rest/client/v1/initial_sync.py | 4 +- synapse/rest/client/v1/presence.py | 16 +-- synapse/rest/client/v1/profile.py | 8 +- synapse/rest/client/v1/push_rule.py | 13 ++- synapse/rest/client/v1/pusher.py | 5 +- synapse/rest/client/v1/room.py | 116 +++++++++++-------- synapse/rest/client/v1/voip.py | 4 +- synapse/rest/client/v2_alpha/account.py | 20 ++-- synapse/rest/client/v2_alpha/account_data.py | 8 +- synapse/rest/client/v2_alpha/filter.py | 8 +- synapse/rest/client/v2_alpha/keys.py | 16 +-- synapse/rest/client/v2_alpha/receipts.py | 4 +- synapse/rest/client/v2_alpha/sync.py | 11 +- synapse/rest/client/v2_alpha/tags.py | 12 +- synapse/rest/media/v0/content_repository.py | 6 +- synapse/rest/media/v1/upload_resource.py | 4 +- synapse/types.py | 3 + tests/api/test_auth.py | 12 +- tests/rest/client/v1/test_presence.py | 5 +- tests/rest/client/v1/test_profile.py | 7 +- 24 files changed, 178 insertions(+), 144 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index b86c6c839..876869bb7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -22,7 +22,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError, EventSizeError -from synapse.types import RoomID, UserID, EventID +from synapse.types import Requester, RoomID, UserID, EventID from synapse.util.logutils import log_function from unpaddedbase64 import decode_base64 @@ -534,7 +534,9 @@ class Auth(object): request.authenticated_entity = user_id - defer.returnValue((UserID.from_string(user_id), "", False)) + defer.returnValue( + Requester(UserID.from_string(user_id), "", False) + ) return except KeyError: pass # normal users won't have the user_id query parameter set. @@ -564,7 +566,7 @@ class Auth(object): request.authenticated_entity = user.to_string() - defer.returnValue((user, token_id, is_guest,)) + defer.returnValue(Requester(user, token_id, is_guest)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 4d724dce7..e2f5eb7b2 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,8 +31,9 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, _, _ = yield self.auth.get_user_by_req(request) - is_admin = yield self.auth.is_server_admin(auth_user) + requester = yield self.auth.get_user_by_req(request) + auth_user = requester.user + is_admin = yield self.auth.is_server_admin(requester.user) if not is_admin and target_user != auth_user: raise AuthError(403, "You are not a server admin") diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 7eef6bf5d..74ec1e50e 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -69,9 +69,9 @@ class ClientDirectoryServer(ClientV1RestServlet): try: # try to auth as a user - user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) try: - user_id = user.to_string() + user_id = requester.user.to_string() yield dir_handler.create_association( user_id, room_alias, room_id, servers ) @@ -116,8 +116,8 @@ class ClientDirectoryServer(ClientV1RestServlet): # fallback to default user behaviour if they aren't an AS pass - user, _, _ = yield self.auth.get_user_by_req(request) - + requester = yield self.auth.get_user_by_req(request) + user = requester.user is_admin = yield self.auth.is_server_admin(user) if not is_admin: raise AuthError(403, "You need to be a server admin") diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index 631f2ca05..e89118b37 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,10 +34,11 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, _, is_guest = yield self.auth.get_user_by_req( + requester = yield self.auth.get_user_by_req( request, - allow_guest=True + allow_guest=True, ) + is_guest = requester.is_guest room_id = None if is_guest: if "room_id" not in request.args: @@ -56,9 +57,13 @@ class EventStreamRestServlet(ClientV1RestServlet): as_client_event = "raw" not in request.args chunk = yield handler.get_stream( - auth_user.to_string(), pagin_config, timeout=timeout, - as_client_event=as_client_event, affect_presence=(not is_guest), - room_id=room_id, is_guest=is_guest + requester.user.to_string(), + pagin_config, + timeout=timeout, + as_client_event=as_client_event, + affect_presence=(not is_guest), + room_id=room_id, + is_guest=is_guest, ) except: logger.exception("Event stream failed") @@ -80,9 +85,9 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler - event = yield handler.get_event(auth_user, event_id) + event = yield handler.get_event(requester.user, event_id) time_now = self.clock.time_msec() if event: diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 541319c35..ad161bdba 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,13 +25,13 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler include_archived = request.args.get("archived", None) == ["true"] content = yield handler.snapshot_all_rooms( - user_id=user.to_string(), + user_id=requester.user.to_string(), pagin_config=pagination_config, as_client_event=as_client_event, include_archived=include_archived, diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 855385ec1..a6f8754e3 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,17 +32,17 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( - target_user=user, auth_user=auth_user) + target_user=user, auth_user=requester.user) defer.returnValue((200, state)) @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -64,7 +64,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): raise SynapseError(400, "Unable to parse state") yield self.handlers.presence_handler.set_state( - target_user=user, auth_user=auth_user, state=state) + target_user=user, auth_user=requester.user, state=state) defer.returnValue((200, {})) @@ -77,13 +77,13 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") - if auth_user != user: + if requester.user != user: raise SynapseError(400, "Cannot get another user's presence list") presence = yield self.handlers.presence_handler.get_presence_list( @@ -97,13 +97,13 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") - if auth_user != user: + if requester.user != user: raise SynapseError( 400, "Cannot modify another user's presence list") diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index d4bc9e076..b15defdd0 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request, allow_guest=True) + requester = yield self.auth.get_user_by_req(request, allow_guest=True) user = UserID.from_string(user_id) try: @@ -47,7 +47,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name) + user, requester.user, new_name) defer.returnValue((200, {})) @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -80,7 +80,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_avatar_url( - user, auth_user, new_name) + user, requester.user, new_name) defer.returnValue((200, {})) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2aab28ae7..c0a21c0c1 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -43,7 +43,7 @@ class PushRuleRestServlet(ClientV1RestServlet): except InvalidRuleException as e: raise SynapseError(400, e.message) - user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) if '/' in spec['rule_id'] or '\\' in spec['rule_id']: raise SynapseError(400, "rule_id may not contain slashes") @@ -51,7 +51,7 @@ class PushRuleRestServlet(ClientV1RestServlet): content = _parse_json(request) if 'attr' in spec: - self.set_rule_attr(user.to_string(), spec, content) + self.set_rule_attr(requester.user, spec, content) defer.returnValue((200, {})) try: @@ -73,7 +73,7 @@ class PushRuleRestServlet(ClientV1RestServlet): try: yield self.hs.get_datastore().add_push_rule( - user_name=user.to_string(), + user_name=requester.user.to_string(), rule_id=_namespaced_rule_id_from_spec(spec), priority_class=priority_class, conditions=conditions, @@ -92,13 +92,13 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_DELETE(self, request): spec = _rule_spec_from_path(request.postpath) - user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) namespaced_rule_id = _namespaced_rule_id_from_spec(spec) try: yield self.hs.get_datastore().delete_push_rule( - user.to_string(), namespaced_rule_id + requester.user.to_string(), namespaced_rule_id ) defer.returnValue((200, {})) except StoreError as e: @@ -109,7 +109,8 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) + user = requester.user # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 81a8786ae..b162b210b 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -30,7 +30,8 @@ class PusherRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - user, token_id, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) + user = requester.user content = _parse_json(request) @@ -71,7 +72,7 @@ class PusherRestServlet(ClientV1RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), - access_token=token_id, + access_token=requester.access_token_id, profile_tag=content['profile_tag'], kind=content['kind'], app_id=content['app_id'], diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 926f77d1c..7496b2673 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -61,10 +61,14 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) - info = yield self.make_room(room_config, auth_user, None) + info = yield self.make_room( + room_config, + requester.user, + None, + ) room_config.update(info) defer.returnValue((200, info)) @@ -124,15 +128,15 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True) + requester = yield self.auth.get_user_by_req(request, allow_guest=True) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( - user_id=user.to_string(), + user_id=requester.user.to_string(), room_id=room_id, event_type=event_type, state_key=state_key, - is_guest=is_guest, + is_guest=requester.is_guest, ) if not data: @@ -143,7 +147,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key, txn_id=None): - user, token_id, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -151,7 +155,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): "type": event_type, "content": content, "room_id": room_id, - "sender": user.to_string(), + "sender": requester.user.to_string(), } if state_key is not None: @@ -159,7 +163,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): msg_handler = self.handlers.message_handler yield msg_handler.create_and_send_event( - event_dict, token_id=token_id, txn_id=txn_id, + event_dict, token_id=requester.access_token_id, txn_id=txn_id, ) defer.returnValue((200, {})) @@ -175,7 +179,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_type, txn_id=None): - user, token_id, _ = yield self.auth.get_user_by_req(request, allow_guest=True) + requester = yield self.auth.get_user_by_req(request, allow_guest=True) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -184,9 +188,9 @@ class RoomSendEventRestServlet(ClientV1RestServlet): "type": event_type, "content": content, "room_id": room_id, - "sender": user.to_string(), + "sender": requester.user.to_string(), }, - token_id=token_id, + token_id=requester.access_token_id, txn_id=txn_id, ) @@ -220,9 +224,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_identifier, txn_id=None): - user, token_id, is_guest = yield self.auth.get_user_by_req( + requester = yield self.auth.get_user_by_req( request, - allow_guest=True + allow_guest=True, ) # the identifier could be a room alias or a room id. Try one then the @@ -241,24 +245,27 @@ class JoinRoomAliasServlet(ClientV1RestServlet): if is_room_alias: handler = self.handlers.room_member_handler - ret_dict = yield handler.join_room_alias(user, identifier) + ret_dict = yield handler.join_room_alias( + requester.user, + identifier, + ) defer.returnValue((200, ret_dict)) else: # room id msg_handler = self.handlers.message_handler content = {"membership": Membership.JOIN} - if is_guest: + if requester.is_guest: content["kind"] = "guest" yield msg_handler.create_and_send_event( { "type": EventTypes.Member, "content": content, "room_id": identifier.to_string(), - "sender": user.to_string(), - "state_key": user.to_string(), + "sender": requester.user.to_string(), + "state_key": requester.user.to_string(), }, - token_id=token_id, + token_id=requester.access_token_id, txn_id=txn_id, - is_guest=is_guest, + is_guest=requester.is_guest, ) defer.returnValue((200, {"room_id": identifier.to_string()})) @@ -296,11 +303,11 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler events = yield handler.get_state_events( room_id=room_id, - user_id=user.to_string(), + user_id=requester.user.to_string(), ) chunk = [] @@ -315,7 +322,8 @@ class RoomMemberListRestServlet(ClientV1RestServlet): try: presence_handler = self.handlers.presence_handler presence_state = yield presence_handler.get_state( - target_user=target_user, auth_user=user + target_user=target_user, + auth_user=requester.user, ) event["content"].update(presence_state) except: @@ -332,7 +340,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True) + requester = yield self.auth.get_user_by_req(request, allow_guest=True) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -340,8 +348,8 @@ class RoomMessageListRestServlet(ClientV1RestServlet): handler = self.handlers.message_handler msgs = yield handler.get_messages( room_id=room_id, - user_id=user.to_string(), - is_guest=is_guest, + user_id=requester.user.to_string(), + is_guest=requester.is_guest, pagin_config=pagination_config, as_client_event=as_client_event ) @@ -355,13 +363,13 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True) + requester = yield self.auth.get_user_by_req(request, allow_guest=True) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( room_id=room_id, - user_id=user.to_string(), - is_guest=is_guest, + user_id=requester.user.to_string(), + is_guest=requester.is_guest, ) defer.returnValue((200, events)) @@ -372,13 +380,13 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True) + requester = yield self.auth.get_user_by_req(request, allow_guest=True) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, - user_id=user.to_string(), + user_id=requester.user.to_string(), pagin_config=pagination_config, - is_guest=is_guest, + is_guest=requester.is_guest, ) defer.returnValue((200, content)) @@ -394,12 +402,16 @@ class RoomEventContext(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_id): - user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True) + requester = yield self.auth.get_user_by_req(request, allow_guest=True) limit = int(request.args.get("limit", [10])[0]) results = yield self.handlers.room_context_handler.get_event_context( - user, room_id, event_id, limit, is_guest + requester.user, + room_id, + event_id, + limit, + requester.is_guest, ) time_now = self.clock.time_msec() @@ -429,14 +441,18 @@ class RoomMembershipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, membership_action, txn_id=None): - user, token_id, is_guest = yield self.auth.get_user_by_req( + requester = yield self.auth.get_user_by_req( request, - allow_guest=True + allow_guest=True, ) + user = requester.user effective_membership_action = membership_action - if is_guest and membership_action not in {Membership.JOIN, Membership.LEAVE}: + if requester.is_guest and membership_action not in { + Membership.JOIN, + Membership.LEAVE + }: raise AuthError(403, "Guest access not allowed") content = _parse_json(request) @@ -451,7 +467,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): content["medium"], content["address"], content["id_server"], - token_id, + requester.access_token_id, txn_id ) defer.returnValue((200, {})) @@ -473,7 +489,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): msg_handler = self.handlers.message_handler content = {"membership": unicode(effective_membership_action)} - if is_guest: + if requester.is_guest: content["kind"] = "guest" yield msg_handler.create_and_send_event( @@ -484,9 +500,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): "sender": user.to_string(), "state_key": state_key, }, - token_id=token_id, + token_id=requester.access_token_id, txn_id=txn_id, - is_guest=is_guest, + is_guest=requester.is_guest, ) if membership_action == "forget": @@ -524,7 +540,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_id, txn_id=None): - user, token_id, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -533,10 +549,10 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): "type": EventTypes.Redaction, "content": content, "room_id": room_id, - "sender": user.to_string(), + "sender": requester.user.to_string(), "redacts": event_id, }, - token_id=token_id, + token_id=requester.access_token_id, txn_id=txn_id, ) @@ -564,7 +580,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) @@ -576,14 +592,14 @@ class RoomTypingRestServlet(ClientV1RestServlet): if content["typing"]: yield typing_handler.started_typing( target_user=target_user, - auth_user=auth_user, + auth_user=requester.user, room_id=room_id, timeout=content.get("timeout", 30000), ) else: yield typing_handler.stopped_typing( target_user=target_user, - auth_user=auth_user, + auth_user=requester.user, room_id=room_id, ) @@ -597,12 +613,16 @@ class SearchRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) content = _parse_json(request) batch = request.args.get("next_batch", [None])[0] - results = yield self.handlers.search_handler.search(auth_user, content, batch) + results = yield self.handlers.search_handler.search( + requester.user, + content, + batch, + ) defer.returnValue((200, results)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 860cb0a64..ec4cf8db7 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret @@ -37,7 +37,7 @@ class VoipRestServlet(ClientV1RestServlet): defer.returnValue((200, {})) expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000 - username = "%d:%s" % (expiry, auth_user.to_string()) + username = "%d:%s" % (expiry, requester.user.to_string()) mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) # We need to use standard padded base64 encoding here diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index ddb6f041c..fa56249a6 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -55,10 +55,11 @@ class PasswordRestServlet(RestServlet): if LoginType.PASSWORD in result: # if using password, they should also be logged in - auth_user, _, _ = yield self.auth.get_user_by_req(request) - if auth_user.to_string() != result[LoginType.PASSWORD]: + requester = yield self.auth.get_user_by_req(request) + requester_user_id = requester.user.to_string() + if requester_user_id.to_string() != result[LoginType.PASSWORD]: raise LoginError(400, "", Codes.UNKNOWN) - user_id = auth_user.to_string() + user_id = requester_user_id elif LoginType.EMAIL_IDENTITY in result: threepid = result[LoginType.EMAIL_IDENTITY] if 'medium' not in threepid or 'address' not in threepid: @@ -102,10 +103,10 @@ class ThreepidRestServlet(RestServlet): def on_GET(self, request): yield run_on_reactor() - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) threepids = yield self.hs.get_datastore().user_get_threepids( - auth_user.to_string() + requester.user.to_string() ) defer.returnValue((200, {'threepids': threepids})) @@ -120,7 +121,8 @@ class ThreepidRestServlet(RestServlet): raise SynapseError(400, "Missing param", Codes.MISSING_PARAM) threePidCreds = body['threePidCreds'] - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) + user_id = requester.user.to_string() threepid = yield self.identity_handler.threepid_from_creds(threePidCreds) @@ -135,7 +137,7 @@ class ThreepidRestServlet(RestServlet): raise SynapseError(500, "Invalid response from ID Server") yield self.auth_handler.add_threepid( - auth_user.to_string(), + user_id, threepid['medium'], threepid['address'], threepid['validated_at'], @@ -144,10 +146,10 @@ class ThreepidRestServlet(RestServlet): if 'bind' in body and body['bind']: logger.debug( "Binding emails %s to %s", - threepid, auth_user.to_string() + threepid, user_id ) yield self.identity_handler.bind_threepid( - threePidCreds, auth_user.to_string() + threePidCreds, user_id ) defer.returnValue((200, {})) diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py index 629b04fe7..985efe2a6 100644 --- a/synapse/rest/client/v2_alpha/account_data.py +++ b/synapse/rest/client/v2_alpha/account_data.py @@ -43,8 +43,8 @@ class AccountDataServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id, account_data_type): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - if user_id != auth_user.to_string(): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add account data for other users.") try: @@ -82,8 +82,8 @@ class RoomAccountDataServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id, room_id, account_data_type): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - if user_id != auth_user.to_string(): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add account data for other users.") try: diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 2af7bfaf9..7695bebc2 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -40,9 +40,9 @@ class GetFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): target_user = UserID.from_string(user_id) - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) - if target_user != auth_user: + if target_user != requester.user: raise AuthError(403, "Cannot get filters for other users") if not self.hs.is_mine(target_user): @@ -76,9 +76,9 @@ class CreateFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) - if target_user != auth_user: + if target_user != requester.user: raise AuthError(403, "Cannot create filters for other users") if not self.hs.is_mine(target_user): diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 24c355483..f989b0861 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -64,8 +64,8 @@ class KeyUploadServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, device_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - user_id = auth_user.to_string() + requester = yield self.auth.get_user_by_req(request) + user_id = requester.user.to_string() # TODO: Check that the device_id matches that in the authentication # or derive the device_id from the authentication instead. try: @@ -78,8 +78,8 @@ class KeyUploadServlet(RestServlet): device_keys = body.get("device_keys", None) if device_keys: logger.info( - "Updating device_keys for device %r for user %r at %d", - device_id, auth_user, time_now + "Updating device_keys for device %r for user %s at %d", + device_id, user_id, time_now ) # TODO: Sign the JSON with the server key yield self.store.set_e2e_device_keys( @@ -109,8 +109,8 @@ class KeyUploadServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, device_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - user_id = auth_user.to_string() + requester = yield self.auth.get_user_by_req(request) + user_id = requester.user.to_string() result = yield self.store.count_e2e_one_time_keys(user_id, device_id) defer.returnValue((200, {"one_time_key_counts": result})) @@ -182,8 +182,8 @@ class KeyQueryServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, device_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - auth_user_id = auth_user.to_string() + requester = yield self.auth.get_user_by_req(request) + auth_user_id = requester.user.to_string() user_id = user_id if user_id else auth_user_id device_ids = [device_id] if device_id else [] result = yield self.handle_request( diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py index 43c23d609..eb4b369a3 100644 --- a/synapse/rest/client/v2_alpha/receipts.py +++ b/synapse/rest/client/v2_alpha/receipts.py @@ -40,7 +40,7 @@ class ReceiptRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, receipt_type, event_id): - user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) if receipt_type != "m.read": raise SynapseError(400, "Receipt type must be 'm.read'") @@ -48,7 +48,7 @@ class ReceiptRestServlet(RestServlet): yield self.receipts_handler.received_client_receipt( room_id, receipt_type, - user_id=user.to_string(), + user_id=requester.user.to_string(), event_id=event_id ) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index c05e7d50c..3867547ad 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -85,9 +85,10 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, token_id, is_guest = yield self.auth.get_user_by_req( + requester = yield self.auth.get_user_by_req( request, allow_guest=True ) + user = requester.user timeout = parse_integer(request, "timeout", default=0) since = parse_string(request, "since") @@ -123,7 +124,7 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, filter=filter, - is_guest=is_guest, + is_guest=requester.is_guest, ) if since is not None: @@ -146,15 +147,15 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() joined = self.encode_joined( - sync_result.joined, filter, time_now, token_id + sync_result.joined, filter, time_now, requester.access_token_id ) invited = self.encode_invited( - sync_result.invited, filter, time_now, token_id + sync_result.invited, filter, time_now, requester.access_token_id ) archived = self.encode_archived( - sync_result.archived, filter, time_now, token_id + sync_result.archived, filter, time_now, requester.access_token_id ) response_content = { diff --git a/synapse/rest/client/v2_alpha/tags.py b/synapse/rest/client/v2_alpha/tags.py index 1bfc36ab2..42f2203f3 100644 --- a/synapse/rest/client/v2_alpha/tags.py +++ b/synapse/rest/client/v2_alpha/tags.py @@ -42,8 +42,8 @@ class TagListServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, room_id): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - if user_id != auth_user.to_string(): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): raise AuthError(403, "Cannot get tags for other users.") tags = yield self.store.get_tags_for_room(user_id, room_id) @@ -68,8 +68,8 @@ class TagServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id, room_id, tag): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - if user_id != auth_user.to_string(): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add tags for other users.") try: @@ -88,8 +88,8 @@ class TagServlet(RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, user_id, room_id, tag): - auth_user, _, _ = yield self.auth.get_user_by_req(request) - if user_id != auth_user.to_string(): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add tags for other users.") max_id = yield self.store.remove_tag_from_room(user_id, room_id, tag) diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index dd7a1b2b3..dcf3eaee1 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,11 +66,11 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( - auth_user.to_string() + requester.user.to_string() ).replace('=', '') # use a random string for the main portion @@ -94,7 +94,7 @@ class ContentRepoResource(resource.Resource): file_name = prefix + main_part + suffix file_path = os.path.join(self.directory, file_name) logger.info("User %s is uploading a file to path %s", - auth_user.to_string(), + request.user.user_id.to_string(), file_path) # keep trying to make a non-clashing file, with a sensible max attempts diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index c1e895ee8..9c7ad4ae8 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -70,7 +70,7 @@ class UploadResource(BaseMediaResource): @request_handler @defer.inlineCallbacks def _async_render_POST(self, request): - auth_user, _, _ = yield self.auth.get_user_by_req(request) + requester = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") @@ -110,7 +110,7 @@ class UploadResource(BaseMediaResource): content_uri = yield self.create_content( media_type, upload_name, request.content.read(), - content_length, auth_user + content_length, requester.user ) respond_with_json( diff --git a/synapse/types.py b/synapse/types.py index 1ec7b3e10..2095837ba 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -18,6 +18,9 @@ from synapse.api.errors import SynapseError from collections import namedtuple +Requester = namedtuple("Requester", ["user", "access_token_id", "is_guest"]) + + class DomainSpecificString( namedtuple("DomainSpecificString", ("localpart", "domain")) ): diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 5ff4c8a87..474c5c418 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -51,8 +51,8 @@ class AuthTestCase(unittest.TestCase): request = Mock(args={}) request.args["access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = Mock(return_value=[""]) - (user, _, _) = yield self.auth.get_user_by_req(request) - self.assertEquals(user.to_string(), self.test_user) + requester = yield self.auth.get_user_by_req(request) + self.assertEquals(requester.user.to_string(), self.test_user) def test_get_user_by_req_user_bad_token(self): self.store.get_app_service_by_token = Mock(return_value=None) @@ -86,8 +86,8 @@ class AuthTestCase(unittest.TestCase): request = Mock(args={}) request.args["access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = Mock(return_value=[""]) - (user, _, _) = yield self.auth.get_user_by_req(request) - self.assertEquals(user.to_string(), self.test_user) + requester = yield self.auth.get_user_by_req(request) + self.assertEquals(requester.user.to_string(), self.test_user) def test_get_user_by_req_appservice_bad_token(self): self.store.get_app_service_by_token = Mock(return_value=None) @@ -121,8 +121,8 @@ class AuthTestCase(unittest.TestCase): request.args["access_token"] = [self.test_token] request.args["user_id"] = [masquerading_user_id] request.requestHeaders.getRawHeaders = Mock(return_value=[""]) - (user, _, _) = yield self.auth.get_user_by_req(request) - self.assertEquals(user.to_string(), masquerading_user_id) + requester = yield self.auth.get_user_by_req(request) + self.assertEquals(requester.user.to_string(), masquerading_user_id) def test_get_user_by_req_appservice_valid_token_bad_user_id(self): masquerading_user_id = "@doppelganger:matrix.org" diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index d782eadb6..90b911f87 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -14,7 +14,6 @@ # limitations under the License. """Tests REST events for /presence paths.""" - from tests import unittest from twisted.internet import defer @@ -26,7 +25,7 @@ from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler from synapse.rest.client.v1 import presence from synapse.rest.client.v1 import events -from synapse.types import UserID +from synapse.types import Requester, UserID from synapse.util.async import run_on_reactor from collections import namedtuple @@ -301,7 +300,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_clock().time_msec.return_value = 1000000 def _get_user_by_req(req=None, allow_guest=False): - return (UserID.from_string(myid), "", False) + return Requester(UserID.from_string(myid), "", False) hs.get_v1auth().get_user_by_req = _get_user_by_req diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 77b7b06c1..c1a3f5204 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -14,16 +14,15 @@ # limitations under the License. """Tests REST events for /profile paths.""" - from tests import unittest from twisted.internet import defer -from mock import Mock, NonCallableMock +from mock import Mock from ....utils import MockHttpResource, setup_test_homeserver from synapse.api.errors import SynapseError, AuthError -from synapse.types import UserID +from synapse.types import Requester, UserID from synapse.rest.client.v1 import profile @@ -53,7 +52,7 @@ class ProfileTestCase(unittest.TestCase): ) def _get_user_by_req(request=None, allow_guest=False): - return (UserID.from_string(myid), "", False) + return Requester(UserID.from_string(myid), "", False) hs.get_v1auth().get_user_by_req = _get_user_by_req From fd5c28dc52f591734b5f3eb465b283761e1116f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Jan 2016 10:55:11 +0000 Subject: [PATCH 058/294] Dont fire user_joined_room when guest hits /events Firing the 'user_joined_room' signal everytime a guest hits /events causes all presence for that room to be returned in the stream. This may sound helpful, but causes clients to tightloop calling /events. In general, guest users should get the initial presence from (room) intial sync and so we don't require presence to sbsequently come down the event stream. --- synapse/handlers/events.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 28c674730..c73eec2b9 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -36,10 +36,6 @@ def stopped_user_eventstream(distributor, user): return distributor.fire("stopped_user_eventstream", user) -def user_joined_room(distributor, user, room_id): - return distributor.fire("user_joined_room", user, room_id) - - class EventStreamHandler(BaseHandler): def __init__(self, hs): @@ -136,9 +132,6 @@ class EventStreamHandler(BaseHandler): # thundering herds on restart. timeout = random.randint(int(timeout*0.9), int(timeout*1.1)) - if is_guest: - yield user_joined_room(self.distributor, auth_user, room_id) - events, tokens = yield self.notifier.get_events_for( auth_user, pagin_config, timeout, only_room_events=only_room_events, From 31de2953a35ab9537b82e712eea78495cdd33712 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 12 Jan 2016 14:22:24 +0000 Subject: [PATCH 059/294] Remove the PushRuleTable and PushRuleEnableTable objects --- synapse/storage/push_rule.py | 83 ++++++++++++++---------------------- 1 file changed, 33 insertions(+), 50 deletions(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 0829262f4..a4dde1aac 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -27,11 +27,14 @@ class PushRuleStore(SQLBaseStore): @cachedInlineCallbacks() def get_push_rules_for_user(self, user_name): rows = yield self._simple_select_list( - table=PushRuleTable.table_name, + table="push_rules", keyvalues={ "user_name": user_name, }, - retcols=PushRuleTable.fields, + retcols=( + "user_name", "rule_id", "priority_class", "priority", + "conditions", "actions", + ), desc="get_push_rules_enabled_for_user", ) @@ -44,11 +47,13 @@ class PushRuleStore(SQLBaseStore): @cachedInlineCallbacks() def get_push_rules_enabled_for_user(self, user_name): results = yield self._simple_select_list( - table=PushRuleEnableTable.table_name, + table="push_rules_enable", keyvalues={ 'user_name': user_name }, - retcols=PushRuleEnableTable.fields, + retcols=( + "user_name", "rule_id", "enabled", + ), desc="get_push_rules_enabled_for_user", ) defer.returnValue({ @@ -61,14 +66,15 @@ class PushRuleStore(SQLBaseStore): def f(txn, user_ids_to_fetch): sql = ( - "SELECT " + - ",".join("pr."+x for x in PushRuleTable.fields) + - " FROM " + PushRuleTable.table_name + " pr " + - " LEFT JOIN " + PushRuleEnableTable.table_name + " pre " + - " ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " + - " WHERE pr.user_name " + + "SELECT" + " pr.user_name, pr.rule_id, priority_class, priority," + " conditions, actions" + " FROM push_rules AS pr" + " LEFT JOIN push_rules_enable AS pre" + " ON pr.user_name = pre.user_name AND pr.rule_id = pre.rule_id" + " WHERE pr.user_name" " IN (" + ",".join("?" for _ in user_ids_to_fetch) + ")" - " AND (pre.enabled is null or pre.enabled = 1)" + " AND (pre.enabled IS NULL OR pre.enabled = 1)" " ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC" ) txn.execute(sql, user_ids_to_fetch) @@ -86,14 +92,15 @@ class PushRuleStore(SQLBaseStore): "bulk_get_push_rules", f, batch_user_ids ) - for r in rows: - rawdict = { - PushRuleTable.fields[i]: r[i] for i in range(len(r)) - } + cols = ( + "user_name", "rule_id", "priority_class", "priority", + "conditions", "actions", + ) + + for row in rows: + rawdict = dict(zip(cols, rows)) + results.setdefault(rawdict["user_name"], []).append(rawdict) - if rawdict['user_name'] not in results: - results[rawdict['user_name']] = [] - results[rawdict['user_name']].append(rawdict) defer.returnValue(results) @defer.inlineCallbacks @@ -131,7 +138,7 @@ class PushRuleStore(SQLBaseStore): res = self._simple_select_one_txn( txn, - table=PushRuleTable.table_name, + table="push_rules", keyvalues={ "user_name": user_name, "rule_id": relative_to_rule, @@ -170,7 +177,7 @@ class PushRuleStore(SQLBaseStore): new_rule['priority'] = new_rule_priority sql = ( - "SELECT COUNT(*) FROM " + PushRuleTable.table_name + + "SELECT COUNT(*) FROM push_rules" " WHERE user_name = ? AND priority_class = ? AND priority = ?" ) txn.execute(sql, (user_name, priority_class, new_rule_priority)) @@ -179,7 +186,7 @@ class PushRuleStore(SQLBaseStore): # if there are conflicting rules, bump everything if num_conflicting: - sql = "UPDATE "+PushRuleTable.table_name+" SET priority = priority " + sql = "UPDATE push_rules SET priority = priority " if after: sql += "-1" else: @@ -202,7 +209,7 @@ class PushRuleStore(SQLBaseStore): self._simple_insert_txn( txn, - table=PushRuleTable.table_name, + table="push_rules", values=new_rule, ) @@ -210,7 +217,7 @@ class PushRuleStore(SQLBaseStore): priority_class, **kwargs): # find the highest priority rule in that class sql = ( - "SELECT COUNT(*), MAX(priority) FROM " + PushRuleTable.table_name + + "SELECT COUNT(*), MAX(priority) FROM push_rules" " WHERE user_name = ? and priority_class = ?" ) txn.execute(sql, (user_name, priority_class)) @@ -237,7 +244,7 @@ class PushRuleStore(SQLBaseStore): self._simple_insert_txn( txn, - table=PushRuleTable.table_name, + table="push_rules", values=new_rule, ) @@ -253,7 +260,7 @@ class PushRuleStore(SQLBaseStore): rule_id (str): The rule_id of the rule to be deleted """ yield self._simple_delete_one( - PushRuleTable.table_name, + "push_rules", {'user_name': user_name, 'rule_id': rule_id}, desc="delete_push_rule", ) @@ -274,7 +281,7 @@ class PushRuleStore(SQLBaseStore): new_id = self._push_rules_enable_id_gen.get_next_txn(txn) self._simple_upsert_txn( txn, - PushRuleEnableTable.table_name, + "push_rules_enable", {'user_name': user_name, 'rule_id': rule_id}, {'enabled': 1 if enabled else 0}, {'id': new_id}, @@ -293,27 +300,3 @@ class RuleNotFoundException(Exception): class InconsistentRuleException(Exception): pass - - -class PushRuleTable(object): - table_name = "push_rules" - - fields = [ - "id", - "user_name", - "rule_id", - "priority_class", - "priority", - "conditions", - "actions", - ] - - -class PushRuleEnableTable(object): - table_name = "push_rules_enable" - - fields = [ - "user_name", - "rule_id", - "enabled" - ] From a8e9e0b916b6d4e4ff739d28d9d036e8aab2fbbb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 12 Jan 2016 14:41:26 +0000 Subject: [PATCH 060/294] Remove the PushersTable and EventPushActionsTable objects --- synapse/storage/event_push_actions.py | 6 +----- synapse/storage/pusher.py | 14 +++++--------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 5b44431ab..d99171ee8 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -42,7 +42,7 @@ class EventPushActionsStore(SQLBaseStore): yield self.runInteraction( "set_actions_for_event_and_users", self._simple_insert_many_txn, - EventPushActionsTable.table_name, + "event_push_actions", values ) @@ -104,7 +104,3 @@ class EventPushActionsStore(SQLBaseStore): "remove_push_actions_for_event_id", f ) - - -class EventPushActionsTable(object): - table_name = "event_push_actions" diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index b9568dad2..2b90d6c62 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -86,7 +86,7 @@ class PusherStore(SQLBaseStore): try: next_id = yield self._pushers_id_gen.get_next() yield self._simple_upsert( - PushersTable.table_name, + "pushers", dict( app_id=app_id, pushkey=pushkey, @@ -114,7 +114,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def delete_pusher_by_app_id_pushkey_user_name(self, app_id, pushkey, user_name): yield self._simple_delete_one( - PushersTable.table_name, + "pushers", {"app_id": app_id, "pushkey": pushkey, 'user_name': user_name}, desc="delete_pusher_by_app_id_pushkey_user_name", ) @@ -122,7 +122,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def update_pusher_last_token(self, app_id, pushkey, user_name, last_token): yield self._simple_update_one( - PushersTable.table_name, + "pushers", {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'last_token': last_token}, desc="update_pusher_last_token", @@ -132,7 +132,7 @@ class PusherStore(SQLBaseStore): def update_pusher_last_token_and_success(self, app_id, pushkey, user_name, last_token, last_success): yield self._simple_update_one( - PushersTable.table_name, + "pushers", {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'last_token': last_token, 'last_success': last_success}, desc="update_pusher_last_token_and_success", @@ -142,12 +142,8 @@ class PusherStore(SQLBaseStore): def update_pusher_failing_since(self, app_id, pushkey, user_name, failing_since): yield self._simple_update_one( - PushersTable.table_name, + "pushers", {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'failing_since': failing_since}, desc="update_pusher_failing_since", ) - - -class PushersTable(object): - table_name = "pushers" From 70dfe4dc96519a20535dbd1b250d4359b6e93125 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Jan 2016 15:01:56 +0000 Subject: [PATCH 061/294] Don't include old left rooms --- synapse/handlers/sync.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 27fdbe28e..33c1a4512 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -585,7 +585,8 @@ class SyncHandler(BaseHandler): sync_config, leave_event, since_token, tags_by_room, account_data_by_room ) - archived.append(room_sync) + if room_sync: + archived.append(room_sync) invited = [ InvitedSyncResult(room_id=event.room_id, invite=event) @@ -726,6 +727,9 @@ class SyncHandler(BaseHandler): leave_token = since_token.copy_and_replace("room_key", stream_token) + if since_token.is_after(leave_token): + defer.returnValue(None) + batch = yield self.load_filtered_recents( leave_event.room_id, sync_config, leave_token, since_token, ) From 96e400fee5bf821174730e1421a14e4aa94e3c0d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 Jan 2016 11:07:32 +0000 Subject: [PATCH 062/294] Remove the RoomsTable object --- synapse/storage/room.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 390bd7865..dc09a3aab 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -49,7 +49,7 @@ class RoomStore(SQLBaseStore): """ try: yield self._simple_insert( - RoomsTable.table_name, + "rooms", { "room_id": room_id, "creator": room_creator_user_id, @@ -70,9 +70,9 @@ class RoomStore(SQLBaseStore): A namedtuple containing the room information, or an empty list. """ return self._simple_select_one( - table=RoomsTable.table_name, + table="rooms", keyvalues={"room_id": room_id}, - retcols=RoomsTable.fields, + retcols=("room_id", "is_public", "creator"), desc="get_room", allow_none=True, ) @@ -275,13 +275,3 @@ class RoomStore(SQLBaseStore): aliases.extend(e.content['aliases']) defer.returnValue((name, aliases)) - - -class RoomsTable(object): - table_name = "rooms" - - fields = [ - "room_id", - "is_public", - "creator" - ] From c0a279e808435d286ae254f51253d2adb3ee7858 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 Jan 2016 11:15:20 +0000 Subject: [PATCH 063/294] Delete the table objects from TransactionStore --- synapse/storage/transactions.py | 68 +++++---------------------------- tests/handlers/test_presence.py | 1 - tests/handlers/test_typing.py | 1 - 3 files changed, 10 insertions(+), 60 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index b40a070b6..4475c451c 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -16,8 +16,6 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached -from collections import namedtuple - from canonicaljson import encode_canonical_json import logging @@ -50,12 +48,15 @@ class TransactionStore(SQLBaseStore): def _get_received_txn_response(self, txn, transaction_id, origin): result = self._simple_select_one_txn( txn, - table=ReceivedTransactionsTable.table_name, + table="received_transactions", keyvalues={ "transaction_id": transaction_id, "origin": origin, }, - retcols=ReceivedTransactionsTable.fields, + retcols=( + "transaction_id", "origin", "ts", "response_code", "response_json", + "has_been_referenced", + ), allow_none=True, ) @@ -79,7 +80,7 @@ class TransactionStore(SQLBaseStore): """ return self._simple_insert( - table=ReceivedTransactionsTable.table_name, + table="received_transactions", values={ "transaction_id": transaction_id, "origin": origin, @@ -136,7 +137,7 @@ class TransactionStore(SQLBaseStore): self._simple_insert_txn( txn, - table=SentTransactions.table_name, + table="sent_transactions", values={ "id": next_id, "transaction_id": transaction_id, @@ -171,7 +172,7 @@ class TransactionStore(SQLBaseStore): code, response_json): self._simple_update_one_txn( txn, - table=SentTransactions.table_name, + table="sent_transactions", keyvalues={ "transaction_id": transaction_id, "destination": destination, @@ -229,11 +230,11 @@ class TransactionStore(SQLBaseStore): def _get_destination_retry_timings(self, txn, destination): result = self._simple_select_one_txn( txn, - table=DestinationsTable.table_name, + table="destinations", keyvalues={ "destination": destination, }, - retcols=DestinationsTable.fields, + retcols=("destination", "retry_last_ts", "retry_interval"), allow_none=True, ) @@ -304,52 +305,3 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (self._clock.time_msec(),)) return self.cursor_to_dict(txn) - - -class ReceivedTransactionsTable(object): - table_name = "received_transactions" - - fields = [ - "transaction_id", - "origin", - "ts", - "response_code", - "response_json", - "has_been_referenced", - ] - - -class SentTransactions(object): - table_name = "sent_transactions" - - fields = [ - "id", - "transaction_id", - "destination", - "ts", - "response_code", - "response_json", - ] - - EntryType = namedtuple("SentTransactionsEntry", fields) - - -class TransactionsToPduTable(object): - table_name = "transaction_id_to_pdu" - - fields = [ - "transaction_id", - "destination", - "pdu_id", - "pdu_origin", - ] - - -class DestinationsTable(object): - table_name = "destinations" - - fields = [ - "destination", - "retry_last_ts", - "retry_interval", - ] diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 15000aae0..447a22b5f 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -28,7 +28,6 @@ from synapse.api.constants import PresenceState from synapse.api.errors import SynapseError from synapse.handlers.presence import PresenceHandler, UserPresenceCache from synapse.streams.config import SourcePaginationConfig -from synapse.storage.transactions import DestinationsTable from synapse.types import UserID OFFLINE = PresenceState.OFFLINE diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 124bc10e0..763c04d66 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -27,7 +27,6 @@ from ..utils import ( from synapse.api.errors import AuthError from synapse.handlers.typing import TypingNotificationHandler -from synapse.storage.transactions import DestinationsTable from synapse.types import UserID From 8740e4e94a6de62680db2b6c7b9f9bfda244d8ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Jan 2016 11:37:17 +0000 Subject: [PATCH 064/294] bulk_get_push_rules should handle empty lists --- synapse/storage/push_rule.py | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 0829262f4..24d137f8c 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -57,43 +57,35 @@ class PushRuleStore(SQLBaseStore): @defer.inlineCallbacks def bulk_get_push_rules(self, user_ids): + if not user_ids: + defer.returnValue({}) + batch_size = 100 def f(txn, user_ids_to_fetch): sql = ( - "SELECT " + - ",".join("pr."+x for x in PushRuleTable.fields) + - " FROM " + PushRuleTable.table_name + " pr " + - " LEFT JOIN " + PushRuleEnableTable.table_name + " pre " + - " ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " + - " WHERE pr.user_name " + + "SELECT pr.*" + " FROM push_rules as pr " + " LEFT JOIN push_rules_enable as pre " + " ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " + " WHERE pr.user_name " " IN (" + ",".join("?" for _ in user_ids_to_fetch) + ")" " AND (pre.enabled is null or pre.enabled = 1)" " ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC" ) txn.execute(sql, user_ids_to_fetch) - return txn.fetchall() + return self.cursor_to_dict(txn) results = {} - batch_start = 0 - while batch_start < len(user_ids): - batch_end = min(len(user_ids), batch_size) - batch_user_ids = user_ids[batch_start:batch_end] - batch_start = batch_end - + chunks = [user_ids[i:i+batch_size] for i in xrange(0, len(user_ids), batch_size)] + for batch_user_ids in chunks: rows = yield self.runInteraction( "bulk_get_push_rules", f, batch_user_ids ) - for r in rows: - rawdict = { - PushRuleTable.fields[i]: r[i] for i in range(len(r)) - } - - if rawdict['user_name'] not in results: - results[rawdict['user_name']] = [] - results[rawdict['user_name']].append(rawdict) + for row in rows: + results.setdefault(row['user_name'], []).append(row) defer.returnValue(results) @defer.inlineCallbacks From 44b4fc5f50e9e77ec43e34b71041abdb272c6dea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Jan 2016 11:47:32 +0000 Subject: [PATCH 065/294] Use compiled regex --- synapse/app/homeserver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bafac884a..6928d9d3e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -88,6 +88,9 @@ import time logger = logging.getLogger("synapse.app.homeserver") +ACCESS_TOKEN_RE = re.compile(r'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$') + + def gz_wrap(r): return EncodingResourceWrapper(r, [GzipEncoderFactory()]) @@ -495,8 +498,7 @@ class SynapseRequest(Request): ) def get_redacted_uri(self): - return re.sub( - r'(\?.*accesss(_|%5[Ff])token=)[^&]*(.*)$', + return ACCESS_TOKEN_RE.sub( r'\1\3', self.uri ) From d9db819e23169dc73daee444641e79d5ded2f46f Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 13 Jan 2016 13:15:53 +0000 Subject: [PATCH 066/294] Change default pushrules back to notifying for all messages. --- synapse/push/baserules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 57de0e34b..f04f88b27 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -247,7 +247,6 @@ def make_base_append_underride_rules(user): }, { 'rule_id': 'global/underride/.m.rule.message', - 'enabled': False, 'conditions': [ { 'kind': 'event_match', From 7d09ab891528c16f66fc4adebbafb8134c51f484 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 13 Jan 2016 13:19:47 +0000 Subject: [PATCH 067/294] Require AS users to be registered before use --- synapse/api/auth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 876869bb7..e36313e2f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -528,6 +528,11 @@ class Auth(object): 403, "Application service cannot masquerade as this user." ) + if not (yield self.store.get_user_by_id(user_id)): + raise AuthError( + 403, + "Application service has not registered this user" + ) if not user_id: raise KeyError From 9c1f853d58440d1f924fa55bc242b248c410dd7c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 Jan 2016 13:08:59 +0000 Subject: [PATCH 068/294] Rename 'user_name' to 'user_id' in push to make it consistent with the rest of the code --- synapse/push/__init__.py | 36 +++++++++---------- synapse/push/baserules.py | 12 +++---- synapse/push/httppusher.py | 6 ++-- synapse/push/push_rule_evaluator.py | 24 ++++++------- synapse/push/pusherpool.py | 30 ++++++++-------- synapse/rest/client/v1/push_rule.py | 10 +++--- synapse/rest/client/v1/pusher.py | 4 +-- synapse/storage/push_rule.py | 54 ++++++++++++++--------------- synapse/storage/pusher.py | 22 ++++++------ 9 files changed, 99 insertions(+), 99 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index c5ddfb564..a5dc84160 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -35,7 +35,7 @@ class Pusher(object): MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, profile_tag, user_name, app_id, + def __init__(self, _hs, profile_tag, user_id, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): self.hs = _hs @@ -43,7 +43,7 @@ class Pusher(object): self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() self.profile_tag = profile_tag - self.user_name = user_name + self.user_id = user_id self.app_id = app_id self.app_display_name = app_display_name self.device_display_name = device_display_name @@ -92,15 +92,15 @@ class Pusher(object): # we fail to dispatch the push) config = PaginationConfig(from_token=None, limit='1') chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, timeout=0, affect_presence=False, + self.user_id, config, timeout=0, affect_presence=False, only_room_events=True ) self.last_token = chunk['end'] self.store.update_pusher_last_token( - self.app_id, self.pushkey, self.user_name, self.last_token + self.app_id, self.pushkey, self.user_id, self.last_token ) logger.info("Pusher %s for user %s starting from token %s", - self.pushkey, self.user_name, self.last_token) + self.pushkey, self.user_id, self.last_token) wait = 0 while self.alive: @@ -125,7 +125,7 @@ class Pusher(object): config = PaginationConfig(from_token=from_tok, limit='1') timeout = (300 + random.randint(-60, 60)) * 1000 chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, timeout=timeout, affect_presence=False, + self.user_id, config, timeout=timeout, affect_presence=False, only_room_events=True ) @@ -142,7 +142,7 @@ class Pusher(object): yield self.store.update_pusher_last_token( self.app_id, self.pushkey, - self.user_name, + self.user_id, self.last_token ) return @@ -153,8 +153,8 @@ class Pusher(object): processed = False rule_evaluator = yield \ - push_rule_evaluator.evaluator_for_user_name_and_profile_tag( - self.user_name, self.profile_tag, single_event['room_id'], self.store + push_rule_evaluator.evaluator_for_user_id_and_profile_tag( + self.user_id, self.profile_tag, single_event['room_id'], self.store ) actions = yield rule_evaluator.actions_for_event(single_event) @@ -179,7 +179,7 @@ class Pusher(object): pk ) yield self.hs.get_pusherpool().remove_pusher( - self.app_id, pk, self.user_name + self.app_id, pk, self.user_id ) else: processed = True @@ -193,7 +193,7 @@ class Pusher(object): yield self.store.update_pusher_last_token_and_success( self.app_id, self.pushkey, - self.user_name, + self.user_id, self.last_token, self.clock.time_msec() ) @@ -202,7 +202,7 @@ class Pusher(object): yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, - self.user_name, + self.user_id, self.failing_since) else: if not self.failing_since: @@ -210,7 +210,7 @@ class Pusher(object): yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, - self.user_name, + self.user_id, self.failing_since ) @@ -222,13 +222,13 @@ class Pusher(object): # of old notifications. logger.warn("Giving up on a notification to user %s, " "pushkey %s", - self.user_name, self.pushkey) + self.user_id, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] yield self.store.update_pusher_last_token( self.app_id, self.pushkey, - self.user_name, + self.user_id, self.last_token ) @@ -236,14 +236,14 @@ class Pusher(object): yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, - self.user_name, + self.user_id, self.failing_since ) else: logger.warn("Failed to dispatch push for user %s " "(failing for %dms)." "Trying again in %dms", - self.user_name, + self.user_id, self.clock.time_msec() - self.failing_since, self.backoff_delay) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) @@ -280,7 +280,7 @@ class Pusher(object): if last_active > self.last_last_active_time: self.last_last_active_time = last_active if self.has_unread: - logger.info("Resetting badge count for %s", self.user_name) + logger.info("Resetting badge count for %s", self.user_id) self.reset_badge_count() self.has_unread = False diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 57de0e34b..8bac7fd6a 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -15,27 +15,27 @@ from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP -def list_with_base_rules(rawrules, user_name): +def list_with_base_rules(rawrules, user_id): ruleslist = [] # shove the server default rules for each kind onto the end of each current_prio_class = PRIORITY_CLASS_INVERSE_MAP.keys()[-1] ruleslist.extend(make_base_prepend_rules( - user_name, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) for r in rawrules: if r['priority_class'] < current_prio_class: while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_append_rules( - user_name, + user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) current_prio_class -= 1 if current_prio_class > 0: ruleslist.extend(make_base_prepend_rules( - user_name, + user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) @@ -43,13 +43,13 @@ def list_with_base_rules(rawrules, user_name): while current_prio_class > 0: ruleslist.extend(make_base_append_rules( - user_name, + user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) current_prio_class -= 1 if current_prio_class > 0: ruleslist.extend(make_base_prepend_rules( - user_name, + user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 7866db6a2..28f1fab0e 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -23,13 +23,13 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, profile_tag, user_name, app_id, + def __init__(self, _hs, profile_tag, user_id, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, profile_tag, - user_name, + user_id, app_id, app_display_name, device_display_name, @@ -87,7 +87,7 @@ class HttpPusher(Pusher): } if event['type'] == 'm.room.member': d['notification']['membership'] = event['content']['membership'] - d['notification']['user_is_target'] = event['state_key'] == self.user_name + d['notification']['user_is_target'] = event['state_key'] == self.user_id if 'content' in event: d['notification']['content'] = event['content'] diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 705ab8c96..b0283743a 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -27,17 +27,17 @@ logger = logging.getLogger(__name__) @defer.inlineCallbacks -def evaluator_for_user_name_and_profile_tag(user_name, profile_tag, room_id, store): - rawrules = yield store.get_push_rules_for_user(user_name) - enabled_map = yield store.get_push_rules_enabled_for_user(user_name) +def evaluator_for_user_id_and_profile_tag(user_id, profile_tag, room_id, store): + rawrules = yield store.get_push_rules_for_user(user_id) + enabled_map = yield store.get_push_rules_enabled_for_user(user_id) our_member_event = yield store.get_current_state( room_id=room_id, event_type='m.room.member', - state_key=user_name, + state_key=user_id, ) defer.returnValue(PushRuleEvaluator( - user_name, profile_tag, rawrules, enabled_map, + user_id, profile_tag, rawrules, enabled_map, room_id, our_member_event, store )) @@ -46,9 +46,9 @@ class PushRuleEvaluator: DEFAULT_ACTIONS = [] INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") - def __init__(self, user_name, profile_tag, raw_rules, enabled_map, room_id, + def __init__(self, user_id, profile_tag, raw_rules, enabled_map, room_id, our_member_event, store): - self.user_name = user_name + self.user_id = user_id self.profile_tag = profile_tag self.room_id = room_id self.our_member_event = our_member_event @@ -61,7 +61,7 @@ class PushRuleEvaluator: rule['actions'] = json.loads(raw_rule['actions']) rules.append(rule) - user = UserID.from_string(self.user_name) + user = UserID.from_string(self.user_id) self.rules = baserules.list_with_base_rules(rules, user) self.enabled_map = enabled_map @@ -83,7 +83,7 @@ class PushRuleEvaluator: has configured both globally and per-room when we have the ability to do such things. """ - if ev['user_id'] == self.user_name: + if ev['user_id'] == self.user_id: # let's assume you probably know about messages you sent yourself defer.returnValue([]) @@ -124,13 +124,13 @@ class PushRuleEvaluator: if len(actions) == 0: logger.warn( "Ignoring rule id %s with no actions for user %s", - r['rule_id'], self.user_name + r['rule_id'], self.user_id ) continue if matches: logger.info( "%s matches for user %s, event %s", - r['rule_id'], self.user_name, ev['event_id'] + r['rule_id'], self.user_id, ev['event_id'] ) # filter out dont_notify as we treat an empty actions list @@ -141,7 +141,7 @@ class PushRuleEvaluator: logger.info( "No rules match for user %s, event %s", - self.user_name, ev['event_id'] + self.user_id, ev['event_id'] ) defer.returnValue(PushRuleEvaluator.DEFAULT_ACTIONS) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 4208e5c76..12c4af14b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -38,12 +38,12 @@ class PusherPool: @defer.inlineCallbacks def user_presence_changed(self, user, state): - user_name = user.to_string() + user_id = user.to_string() # until we have read receipts, pushers use this to reset a user's # badge counters to zero for p in self.pushers.values(): - if p.user_name == user_name: + if p.user_id == user_id: yield p.presence_changed(state) @defer.inlineCallbacks @@ -52,14 +52,14 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, access_token, profile_tag, kind, app_id, + def add_pusher(self, user_id, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, # recreated, added and started: this means we have only one # code path adding pushers. self._create_pusher({ - "user_name": user_name, + "user_name": user_id, "kind": kind, "profile_tag": profile_tag, "app_id": app_id, @@ -74,7 +74,7 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, access_token, profile_tag, kind, app_id, + user_id, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @@ -109,11 +109,11 @@ class PusherPool: self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, + def _add_pusher_to_store(self, user_id, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( - user_name=user_name, + user_id=user_id, access_token=access_token, profile_tag=profile_tag, kind=kind, @@ -125,14 +125,14 @@ class PusherPool: lang=lang, data=data, ) - self._refresh_pusher(app_id, pushkey, user_name) + self._refresh_pusher(app_id, pushkey, user_id) def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': return HttpPusher( self.hs, profile_tag=pusherdict['profile_tag'], - user_name=pusherdict['user_name'], + user_id=pusherdict['user_name'], app_id=pusherdict['app_id'], app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], @@ -150,14 +150,14 @@ class PusherPool: ) @defer.inlineCallbacks - def _refresh_pusher(self, app_id, pushkey, user_name): + def _refresh_pusher(self, app_id, pushkey, user_id): resultlist = yield self.store.get_pushers_by_app_id_and_pushkey( app_id, pushkey ) p = None for r in resultlist: - if r['user_name'] == user_name: + if r['user_name'] == user_id: p = r if p: @@ -186,12 +186,12 @@ class PusherPool: logger.info("Started pushers") @defer.inlineCallbacks - def remove_pusher(self, app_id, pushkey, user_name): - fullid = "%s:%s:%s" % (app_id, pushkey, user_name) + def remove_pusher(self, app_id, pushkey, user_id): + fullid = "%s:%s:%s" % (app_id, pushkey, user_id) if fullid in self.pushers: logger.info("Stopping pusher %s", fullid) self.pushers[fullid].stop() del self.pushers[fullid] - yield self.store.delete_pusher_by_app_id_pushkey_user_name( - app_id, pushkey, user_name + yield self.store.delete_pusher_by_app_id_pushkey_user_id( + app_id, pushkey, user_id ) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c0a21c0c1..df53824d2 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -73,7 +73,7 @@ class PushRuleRestServlet(ClientV1RestServlet): try: yield self.hs.get_datastore().add_push_rule( - user_name=requester.user.to_string(), + user_id=requester.user.to_string(), rule_id=_namespaced_rule_id_from_spec(spec), priority_class=priority_class, conditions=conditions, @@ -206,7 +206,7 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} - def set_rule_attr(self, user_name, spec, val): + def set_rule_attr(self, user_id, spec, val): if spec['attr'] == 'enabled': if isinstance(val, dict) and "enabled" in val: val = val["enabled"] @@ -217,15 +217,15 @@ class PushRuleRestServlet(ClientV1RestServlet): raise SynapseError(400, "Value for 'enabled' must be boolean") namespaced_rule_id = _namespaced_rule_id_from_spec(spec) self.hs.get_datastore().set_push_rule_enabled( - user_name, namespaced_rule_id, val + user_id, namespaced_rule_id, val ) else: raise UnrecognizedRequestError() - def get_rule_attr(self, user_name, namespaced_rule_id, attr): + def get_rule_attr(self, user_id, namespaced_rule_id, attr): if attr == 'enabled': return self.hs.get_datastore().get_push_rule_enabled_by_user_rule_id( - user_name, namespaced_rule_id + user_id, namespaced_rule_id ) else: raise UnrecognizedRequestError() diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index b162b210b..e218ed215 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -41,7 +41,7 @@ class PusherRestServlet(ClientV1RestServlet): and 'kind' in content and content['kind'] is None): yield pusher_pool.remove_pusher( - content['app_id'], content['pushkey'], user_name=user.to_string() + content['app_id'], content['pushkey'], user_id=user.to_string() ) defer.returnValue((200, {})) @@ -71,7 +71,7 @@ class PusherRestServlet(ClientV1RestServlet): try: yield pusher_pool.add_pusher( - user_name=user.to_string(), + user_id=user.to_string(), access_token=requester.access_token_id, profile_tag=content['profile_tag'], kind=content['kind'], diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 448009b4b..2adfefd99 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -25,11 +25,11 @@ logger = logging.getLogger(__name__) class PushRuleStore(SQLBaseStore): @cachedInlineCallbacks() - def get_push_rules_for_user(self, user_name): + def get_push_rules_for_user(self, user_id): rows = yield self._simple_select_list( table="push_rules", keyvalues={ - "user_name": user_name, + "user_name": user_id, }, retcols=( "user_name", "rule_id", "priority_class", "priority", @@ -45,11 +45,11 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(rows) @cachedInlineCallbacks() - def get_push_rules_enabled_for_user(self, user_name): + def get_push_rules_enabled_for_user(self, user_id): results = yield self._simple_select_list( table="push_rules_enable", keyvalues={ - 'user_name': user_name + 'user_name': user_id }, retcols=( "user_name", "rule_id", "enabled", @@ -122,7 +122,7 @@ class PushRuleStore(SQLBaseStore): ) defer.returnValue(ret) - def _add_push_rule_relative_txn(self, txn, user_name, **kwargs): + def _add_push_rule_relative_txn(self, txn, user_id, **kwargs): after = kwargs.pop("after", None) relative_to_rule = kwargs.pop("before", after) @@ -130,7 +130,7 @@ class PushRuleStore(SQLBaseStore): txn, table="push_rules", keyvalues={ - "user_name": user_name, + "user_name": user_id, "rule_id": relative_to_rule, }, retcols=["priority_class", "priority"], @@ -154,7 +154,7 @@ class PushRuleStore(SQLBaseStore): new_rule.pop("before", None) new_rule.pop("after", None) new_rule['priority_class'] = priority_class - new_rule['user_name'] = user_name + new_rule['user_name'] = user_id new_rule['id'] = self._push_rule_id_gen.get_next_txn(txn) # check if the priority before/after is free @@ -170,7 +170,7 @@ class PushRuleStore(SQLBaseStore): "SELECT COUNT(*) FROM push_rules" " WHERE user_name = ? AND priority_class = ? AND priority = ?" ) - txn.execute(sql, (user_name, priority_class, new_rule_priority)) + txn.execute(sql, (user_id, priority_class, new_rule_priority)) res = txn.fetchall() num_conflicting = res[0][0] @@ -187,14 +187,14 @@ class PushRuleStore(SQLBaseStore): else: sql += ">= ?" - txn.execute(sql, (user_name, priority_class, new_rule_priority)) + txn.execute(sql, (user_id, priority_class, new_rule_priority)) txn.call_after( - self.get_push_rules_for_user.invalidate, (user_name,) + self.get_push_rules_for_user.invalidate, (user_id,) ) txn.call_after( - self.get_push_rules_enabled_for_user.invalidate, (user_name,) + self.get_push_rules_enabled_for_user.invalidate, (user_id,) ) self._simple_insert_txn( @@ -203,14 +203,14 @@ class PushRuleStore(SQLBaseStore): values=new_rule, ) - def _add_push_rule_highest_priority_txn(self, txn, user_name, + def _add_push_rule_highest_priority_txn(self, txn, user_id, priority_class, **kwargs): # find the highest priority rule in that class sql = ( "SELECT COUNT(*), MAX(priority) FROM push_rules" " WHERE user_name = ? and priority_class = ?" ) - txn.execute(sql, (user_name, priority_class)) + txn.execute(sql, (user_id, priority_class)) res = txn.fetchall() (how_many, highest_prio) = res[0] @@ -221,15 +221,15 @@ class PushRuleStore(SQLBaseStore): # and insert the new rule new_rule = kwargs new_rule['id'] = self._push_rule_id_gen.get_next_txn(txn) - new_rule['user_name'] = user_name + new_rule['user_name'] = user_id new_rule['priority_class'] = priority_class new_rule['priority'] = new_prio txn.call_after( - self.get_push_rules_for_user.invalidate, (user_name,) + self.get_push_rules_for_user.invalidate, (user_id,) ) txn.call_after( - self.get_push_rules_enabled_for_user.invalidate, (user_name,) + self.get_push_rules_enabled_for_user.invalidate, (user_id,) ) self._simple_insert_txn( @@ -239,48 +239,48 @@ class PushRuleStore(SQLBaseStore): ) @defer.inlineCallbacks - def delete_push_rule(self, user_name, rule_id): + def delete_push_rule(self, user_id, rule_id): """ Delete a push rule. Args specify the row to be deleted and can be any of the columns in the push_rule table, but below are the standard ones Args: - user_name (str): The matrix ID of the push rule owner + user_id (str): The matrix ID of the push rule owner rule_id (str): The rule_id of the rule to be deleted """ yield self._simple_delete_one( "push_rules", - {'user_name': user_name, 'rule_id': rule_id}, + {'user_name': user_id, 'rule_id': rule_id}, desc="delete_push_rule", ) - self.get_push_rules_for_user.invalidate((user_name,)) - self.get_push_rules_enabled_for_user.invalidate((user_name,)) + self.get_push_rules_for_user.invalidate((user_id,)) + self.get_push_rules_enabled_for_user.invalidate((user_id,)) @defer.inlineCallbacks - def set_push_rule_enabled(self, user_name, rule_id, enabled): + def set_push_rule_enabled(self, user_id, rule_id, enabled): ret = yield self.runInteraction( "_set_push_rule_enabled_txn", self._set_push_rule_enabled_txn, - user_name, rule_id, enabled + user_id, rule_id, enabled ) defer.returnValue(ret) - def _set_push_rule_enabled_txn(self, txn, user_name, rule_id, enabled): + def _set_push_rule_enabled_txn(self, txn, user_id, rule_id, enabled): new_id = self._push_rules_enable_id_gen.get_next_txn(txn) self._simple_upsert_txn( txn, "push_rules_enable", - {'user_name': user_name, 'rule_id': rule_id}, + {'user_name': user_id, 'rule_id': rule_id}, {'enabled': 1 if enabled else 0}, {'id': new_id}, ) txn.call_after( - self.get_push_rules_for_user.invalidate, (user_name,) + self.get_push_rules_for_user.invalidate, (user_id,) ) txn.call_after( - self.get_push_rules_enabled_for_user.invalidate, (user_name,) + self.get_push_rules_enabled_for_user.invalidate, (user_id,) ) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 2b90d6c62..8ec706178 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -80,7 +80,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(rows) @defer.inlineCallbacks - def add_pusher(self, user_name, access_token, profile_tag, kind, app_id, + def add_pusher(self, user_id, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -90,7 +90,7 @@ class PusherStore(SQLBaseStore): dict( app_id=app_id, pushkey=pushkey, - user_name=user_name, + user_name=user_id, ), dict( access_token=access_token, @@ -112,38 +112,38 @@ class PusherStore(SQLBaseStore): raise StoreError(500, "Problem creating pusher.") @defer.inlineCallbacks - def delete_pusher_by_app_id_pushkey_user_name(self, app_id, pushkey, user_name): + def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id): yield self._simple_delete_one( "pushers", - {"app_id": app_id, "pushkey": pushkey, 'user_name': user_name}, - desc="delete_pusher_by_app_id_pushkey_user_name", + {"app_id": app_id, "pushkey": pushkey, 'user_name': user_id}, + desc="delete_pusher_by_app_id_pushkey_user_id", ) @defer.inlineCallbacks - def update_pusher_last_token(self, app_id, pushkey, user_name, last_token): + def update_pusher_last_token(self, app_id, pushkey, user_id, last_token): yield self._simple_update_one( "pushers", - {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_id}, {'last_token': last_token}, desc="update_pusher_last_token", ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, app_id, pushkey, user_name, + def update_pusher_last_token_and_success(self, app_id, pushkey, user_id, last_token, last_success): yield self._simple_update_one( "pushers", - {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_id}, {'last_token': last_token, 'last_success': last_success}, desc="update_pusher_last_token_and_success", ) @defer.inlineCallbacks - def update_pusher_failing_since(self, app_id, pushkey, user_name, + def update_pusher_failing_since(self, app_id, pushkey, user_id, failing_since): yield self._simple_update_one( "pushers", - {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_id}, {'failing_since': failing_since}, desc="update_pusher_failing_since", ) From 93afb40cd4413008c40f6bdde032ccf5f1cefd9f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 5 Jan 2016 18:12:37 +0000 Subject: [PATCH 069/294] Skip, rather than erroring, invalid guest requests Erroring causes problems when people make illegal requests, because they don't know what limit parameter they should pass. This is definitely buggy. It leaks message counts for rooms people don't have permission to see, via tokens. But apparently we already consciously decided to allow that as a team, so this preserves that behaviour. --- synapse/handlers/_base.py | 16 ++-------------- synapse/handlers/message.py | 4 ++-- synapse/handlers/room.py | 2 -- synapse/handlers/sync.py | 1 - synapse/notifier.py | 3 +-- 5 files changed, 5 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 5fd20285d..b474042e8 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -52,8 +52,7 @@ class BaseHandler(object): self.event_builder_factory = hs.get_event_builder_factory() @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, events, is_guest=False, - require_all_visible_for_guests=True): + def _filter_events_for_client(self, user_id, events, is_guest=False): # Assumes that user has at some point joined the room if not is_guest. def allowed(event, membership, visibility): @@ -114,17 +113,6 @@ class BaseHandler(object): if should_include: events_to_return.append(event) - if (require_all_visible_for_guests - and is_guest - and len(events_to_return) < len(events)): - # This indicates that some events in the requested range were not - # visible to guest users. To be safe, we reject the entire request, - # so that we don't have to worry about interpreting visibility - # boundaries. - raise AuthError(403, "User %s does not have permission" % ( - user_id - )) - defer.returnValue(events_to_return) def ratelimit(self, user_id): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index a1bed9b0d..5805190ce 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -685,7 +685,7 @@ class MessageHandler(BaseHandler): ).addErrback(unwrapFirstError) messages = yield self._filter_events_for_client( - user_id, messages, is_guest=is_guest, require_all_visible_for_guests=False + user_id, messages, is_guest=is_guest, ) start_token = now_token.copy_and_replace("room_key", token[0]) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13f66e0df..725b318cf 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -879,14 +879,12 @@ class RoomContextHandler(BaseHandler): user.to_string(), results["events_before"], is_guest=is_guest, - require_all_visible_for_guests=False ) results["events_after"] = yield self._filter_events_for_client( user.to_string(), results["events_after"], is_guest=is_guest, - require_all_visible_for_guests=False ) if results["events_after"]: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index feea407ea..9d52d592b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -648,7 +648,6 @@ class SyncHandler(BaseHandler): sync_config.user.to_string(), loaded_recents, is_guest=sync_config.is_guest, - require_all_visible_for_guests=False ) loaded_recents.extend(recents) recents = loaded_recents diff --git a/synapse/notifier.py b/synapse/notifier.py index fd5257832..0a5653b8d 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2014 - 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -386,7 +386,6 @@ class Notifier(object): user.to_string(), new_events, is_guest=is_guest, - require_all_visible_for_guests=False ) events.extend(new_events) From 49f33f64386954e08b8409842f28b449b011849e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Jan 2016 14:19:22 +0000 Subject: [PATCH 070/294] Add 'event' result to 'context' endpoint ... because the context isn't much use without the event. --- synapse/handlers/room.py | 35 ++++++++++++++++++++++------------ synapse/rest/client/v1/room.py | 6 ++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 725b318cf..529108936 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -864,28 +864,39 @@ class RoomContextHandler(BaseHandler): (excluding state). Returns: - dict + dict, or None if the event isn't found """ before_limit = math.floor(limit/2.) after_limit = limit - before_limit now_token = yield self.hs.get_event_sources().get_current_token() + def filter_evts(events): + return self._filter_events_for_client( + user.to_string(), + events, + is_guest=is_guest) + + event = yield self.store.get_event(event_id, get_prev_content=True, + allow_none=True) + if not event: + defer.returnValue(None) + return + + filtered = yield(filter_evts([event])) + if not filtered: + raise AuthError( + 403, + "You don't have permission to access that event." + ) + results = yield self.store.get_events_around( room_id, event_id, before_limit, after_limit ) - results["events_before"] = yield self._filter_events_for_client( - user.to_string(), - results["events_before"], - is_guest=is_guest, - ) - - results["events_after"] = yield self._filter_events_for_client( - user.to_string(), - results["events_after"], - is_guest=is_guest, - ) + results["events_before"] = yield filter_evts(results["events_before"]) + results["events_after"] = yield filter_evts(results["events_after"]) + results["event"] = event if results["events_after"]: last_event_id = results["events_after"][-1].event_id diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 6fe53f70e..62ce0f066 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -402,10 +402,16 @@ class RoomEventContext(ClientV1RestServlet): user, room_id, event_id, limit, is_guest ) + if not results: + raise SynapseError( + 404, "Event not found.", errcode=Codes.NOT_FOUND + ) + time_now = self.clock.time_msec() results["events_before"] = [ serialize_event(event, time_now) for event in results["events_before"] ] + results["event"] = serialize_event(results["event"], time_now) results["events_after"] = [ serialize_event(event, time_now) for event in results["events_after"] ] From 244b356a37ada45d2a4e6aec1f08986aaa7eaaa1 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 13 Jan 2016 17:03:58 +0000 Subject: [PATCH 071/294] Delete unused code --- synapse/storage/appservice.py | 59 ----------------------------------- 1 file changed, 59 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index eab58d9ce..1a2b4678a 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -15,7 +15,6 @@ import logging import urllib import yaml -from simplejson import JSONDecodeError import simplejson as json from twisted.internet import defer @@ -144,64 +143,6 @@ class ApplicationServiceStore(SQLBaseStore): return rooms_for_user_matching_user_id - def _parse_services_dict(self, results): - # SQL results in the form: - # [ - # { - # 'regex': "something", - # 'url': "something", - # 'namespace': enum, - # 'as_id': 0, - # 'token': "something", - # 'hs_token': "otherthing", - # 'id': 0 - # } - # ] - services = {} - for res in results: - as_token = res["token"] - if as_token is None: - continue - if as_token not in services: - # add the service - services[as_token] = { - "id": res["id"], - "url": res["url"], - "token": as_token, - "hs_token": res["hs_token"], - "sender": res["sender"], - "namespaces": { - ApplicationService.NS_USERS: [], - ApplicationService.NS_ALIASES: [], - ApplicationService.NS_ROOMS: [] - } - } - # add the namespace regex if one exists - ns_int = res["namespace"] - if ns_int is None: - continue - try: - services[as_token]["namespaces"][ - ApplicationService.NS_LIST[ns_int]].append( - json.loads(res["regex"]) - ) - except IndexError: - logger.error("Bad namespace enum '%s'. %s", ns_int, res) - except JSONDecodeError: - logger.error("Bad regex object '%s'", res["regex"]) - - service_list = [] - for service in services.values(): - service_list.append(ApplicationService( - token=service["token"], - url=service["url"], - namespaces=service["namespaces"], - hs_token=service["hs_token"], - sender=service["sender"], - id=service["id"] - )) - return service_list - def _load_appservice(self, as_info): required_string_fields = [ "url", "as_token", "hs_token", "sender_localpart" From f6fcff360250eb362986835c195a96825567a03d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 13 Jan 2016 17:09:24 +0000 Subject: [PATCH 072/294] Don't start server if ASes are invalidly configured --- synapse/storage/appservice.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index eab58d9ce..25a6f14f4 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -265,6 +265,7 @@ class ApplicationServiceStore(SQLBaseStore): except Exception as e: logger.error("Failed to load appservice from '%s'", config_file) logger.exception(e) + raise class ApplicationServiceTransactionStore(SQLBaseStore): From fcb05b4c8269ac7cb2e153e0a68c040a965d8279 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Jan 2016 17:37:09 +0000 Subject: [PATCH 073/294] Temporarily disable notification branch --- synapse/handlers/federation.py | 13 +++++++------ synapse/handlers/sync.py | 5 ++++- synapse/push/action_generator.py | 3 +++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5dee24e21..2f6359c76 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -36,7 +36,7 @@ from synapse.events.utils import prune_event from synapse.util.retryutils import NotRetryingDestination -from synapse.push.action_generator import ActionGenerator +# from synapse.push.action_generator import ActionGenerator from twisted.internet import defer @@ -244,11 +244,12 @@ class FederationHandler(BaseHandler): user = UserID.from_string(event.state_key) yield user_joined_room(self.distributor, user, event.room_id) - if not backfilled and not event.internal_metadata.is_outlier(): - action_generator = ActionGenerator(self.store) - yield action_generator.handle_push_actions_for_event( - event, self - ) + # Temporarily disable notifications due to performance concerns. + # if not backfilled and not event.internal_metadata.is_outlier(): + # action_generator = ActionGenerator(self.store) + # yield action_generator.handle_push_actions_for_event( + # event, self + # ) @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 33c1a4512..69707c47e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -840,7 +840,10 @@ class SyncHandler(BaseHandler): return False @defer.inlineCallbacks - def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): + def unreadnotifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): + # Temporarily disable notifications due to performance concerns. + return + last_unread_event_id = self.last_read_event_id_for_room_and_user( room_id, sync_config.user.to_string(), ephemeral_by_room ) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 4cf94f6c6..73467f3ad 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -36,6 +36,9 @@ class ActionGenerator: @defer.inlineCallbacks def handle_push_actions_for_event(self, event, handler): + # Temporarily disable notifications due to performance concerns. + return + if event.type == EventTypes.Redaction and event.redacts is not None: yield self.store.remove_push_actions_for_event_id( event.room_id, event.redacts From 2655d61d703cc6718f669dd43e814862750b7748 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Jan 2016 17:43:39 +0000 Subject: [PATCH 074/294] Don't change signature. Return empty list --- synapse/handlers/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 69707c47e..7dfa2eeed 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -840,9 +840,9 @@ class SyncHandler(BaseHandler): return False @defer.inlineCallbacks - def unreadnotifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): + def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): # Temporarily disable notifications due to performance concerns. - return + defer.returnValue([]) last_unread_event_id = self.last_read_event_id_for_room_and_user( room_id, sync_config.user.to_string(), ephemeral_by_room From 12623c99b6489e1ceefdc89341fa524b1215665d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 13 Jan 2016 18:55:57 +0000 Subject: [PATCH 075/294] Use the unread notification count to send accurate badge counts in push notifications. --- synapse/push/__init__.py | 104 ++++++++++++++++++++++++++++--------- synapse/push/httppusher.py | 14 ++--- synapse/push/pusherpool.py | 15 ------ 3 files changed, 87 insertions(+), 46 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index c5ddfb564..abfb324fb 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -16,7 +16,9 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig -from synapse.types import StreamToken +from synapse.types import StreamToken, UserID +from synapse.api.constants import Membership +from synapse.api.filtering import FilterCollection import synapse.util.async import push_rule_evaluator as push_rule_evaluator @@ -55,6 +57,7 @@ class Pusher(object): self.backoff_delay = Pusher.INITIAL_BACKOFF self.failing_since = failing_since self.alive = True + self.badge = None # The last value of last_active_time that we saw self.last_last_active_time = 0 @@ -92,8 +95,7 @@ class Pusher(object): # we fail to dispatch the push) config = PaginationConfig(from_token=None, limit='1') chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, timeout=0, affect_presence=False, - only_room_events=True + self.user_name, config, timeout=0, affect_presence=False ) self.last_token = chunk['end'] self.store.update_pusher_last_token( @@ -125,20 +127,30 @@ class Pusher(object): config = PaginationConfig(from_token=from_tok, limit='1') timeout = (300 + random.randint(-60, 60)) * 1000 chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, timeout=timeout, affect_presence=False, - only_room_events=True + self.user_name, config, timeout=timeout, affect_presence=False ) # limiting to 1 may get 1 event plus 1 presence event, so # pick out the actual event single_event = None + read_receipt = None for c in chunk['chunk']: if 'event_id' in c: # Hmmm... single_event = c - break + elif c['type'] == 'm.receipt': + read_receipt = c + + have_updated_badge = False + if read_receipt: + for receipt_part in read_receipt['content'].values(): + if 'm.read' in receipt_part: + if self.user_name in receipt_part['m.read'].keys(): + have_updated_badge = True + if not single_event: + if have_updated_badge: + yield self.update_badge() self.last_token = chunk['end'] - logger.debug("Event stream timeout for pushkey %s", self.pushkey) yield self.store.update_pusher_last_token( self.app_id, self.pushkey, @@ -161,7 +173,8 @@ class Pusher(object): tweaks = rule_evaluator.tweaks_for_actions(actions) if 'notify' in actions: - rejected = yield self.dispatch_push(single_event, tweaks) + self.badge = yield self._get_badge_count() + rejected = yield self.dispatch_push(single_event, tweaks, self.badge) self.has_unread = True if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True @@ -182,6 +195,8 @@ class Pusher(object): self.app_id, pk, self.user_name ) else: + if have_updated_badge: + yield self.update_badge() processed = True if not self.alive: @@ -254,7 +269,7 @@ class Pusher(object): def stop(self): self.alive = False - def dispatch_push(self, p, tweaks): + def dispatch_push(self, p, tweaks, badge): """ Overridden by implementing classes to actually deliver the notification Args: @@ -266,23 +281,64 @@ class Pusher(object): """ pass - def reset_badge_count(self): + @defer.inlineCallbacks + def update_badge(self): + new_badge = yield self._get_badge_count() + if self.badge != new_badge: + self.badge = new_badge + yield self.send_badge(self.badge) + + def send_badge(self, badge): + """ + Overridden by implementing classes to send an updated badge count + """ pass - def presence_changed(self, state): - """ - We clear badge counts whenever a user's last_active time is bumped - This is by no means perfect but I think it's the best we can do - without read receipts. - """ - if 'last_active' in state.state: - last_active = state.state['last_active'] - if last_active > self.last_last_active_time: - self.last_last_active_time = last_active - if self.has_unread: - logger.info("Resetting badge count for %s", self.user_name) - self.reset_badge_count() - self.has_unread = False + @defer.inlineCallbacks + def _get_badge_count(self): + membership_list = (Membership.INVITE, Membership.JOIN) + + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=self.user_name, + membership_list=membership_list + ) + + user_is_guest = yield self.store.is_guest(UserID.from_string(self.user_name)) + + # XXX: importing inside method to break circular dependency. + # should sort out the mess by moving all this logic out of + # push/__init__.py and probably moving the logic we use from the sync + # handler to somewhere more amenable to re-use. + from synapse.handlers.sync import SyncConfig + sync_config = SyncConfig( + user=UserID.from_string(self.user_name), + filter=FilterCollection({}), + is_guest=user_is_guest, + ) + now_token = yield self.hs.get_event_sources().get_current_token() + sync_handler = self.hs.get_handlers().sync_handler + _, ephemeral_by_room = yield sync_handler.ephemeral_by_room( + sync_config, now_token + ) + + badge = 0 + + for r in room_list: + if r.membership == Membership.INVITE: + badge += 1 + else: + last_unread_event_id = sync_handler.last_read_event_id_for_room_and_user( + r.room_id, self.user_name, ephemeral_by_room + ) + + if last_unread_event_id: + notifs = yield ( + self.store.get_unread_event_push_actions_by_room_for_user( + r.room_id, self.user_name, last_unread_event_id + ) + ) + badge += len(notifs) + defer.returnValue(badge) class PusherConfigException(Exception): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 7866db6a2..acb687d11 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -51,7 +51,7 @@ class HttpPusher(Pusher): del self.data_minus_url['url'] @defer.inlineCallbacks - def _build_notification_dict(self, event, tweaks): + def _build_notification_dict(self, event, tweaks, badge): # we probably do not want to push for every presence update # (we may want to be able to set up notifications when specific # people sign in, but we'd want to only deliver the pertinent ones) @@ -71,7 +71,7 @@ class HttpPusher(Pusher): 'counts': { # -- we don't mark messages as read yet so # we have no way of knowing # Just set the badge to 1 until we have read receipts - 'unread': 1, + 'unread': badge, # 'missed_calls': 2 }, 'devices': [ @@ -101,8 +101,8 @@ class HttpPusher(Pusher): defer.returnValue(d) @defer.inlineCallbacks - def dispatch_push(self, event, tweaks): - notification_dict = yield self._build_notification_dict(event, tweaks) + def dispatch_push(self, event, tweaks, badge): + notification_dict = yield self._build_notification_dict(event, tweaks, badge) if not notification_dict: defer.returnValue([]) try: @@ -116,15 +116,15 @@ class HttpPusher(Pusher): defer.returnValue(rejected) @defer.inlineCallbacks - def reset_badge_count(self): + def send_badge(self, badge): + logger.info("Sending updated badge count %d to %r", badge, self.user_name) d = { 'notification': { 'id': '', 'type': None, 'sender': '', 'counts': { - 'unread': 0, - 'missed_calls': 0 + 'unread': badge }, 'devices': [ { diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 4208e5c76..5d1179abf 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -31,21 +31,6 @@ class PusherPool: self.pushers = {} self.last_pusher_started = -1 - distributor = self.hs.get_distributor() - distributor.observe( - "user_presence_changed", self.user_presence_changed - ) - - @defer.inlineCallbacks - def user_presence_changed(self, user, state): - user_name = user.to_string() - - # until we have read receipts, pushers use this to reset a user's - # badge counters to zero - for p in self.pushers.values(): - if p.user_name == user_name: - yield p.presence_changed(state) - @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() From bce602eb4ea5d477abf818849ab3e689381a6954 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Jan 2016 09:56:26 +0000 Subject: [PATCH 076/294] Use logger not logging --- synapse/handlers/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 7dfa2eeed..1942268c3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -671,7 +671,7 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, since_token, ) - logging.debug("Recents %r", batch) + logger.debug("Recents %r", batch) current_state = yield self.get_state_at(room_id, now_token) @@ -734,7 +734,7 @@ class SyncHandler(BaseHandler): leave_event.room_id, sync_config, leave_token, since_token, ) - logging.debug("Recents %r", batch) + logger.debug("Recents %r", batch) state_events_at_leave = yield self.store.get_state_for_event( leave_event.event_id From 339c8f013319f812629bb9a294fb235ada1e888d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Jan 2016 10:22:02 +0000 Subject: [PATCH 077/294] Clamp pagination limits to at most 1000 --- synapse/streams/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 8c082bf4e..4f089bfb9 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -22,6 +22,9 @@ import logging logger = logging.getLogger(__name__) +MAX_LIMIT = 1000 + + class SourcePaginationConfig(object): """A configuration object which stores pagination parameters for a @@ -32,7 +35,7 @@ class SourcePaginationConfig(object): self.from_key = from_key self.to_key = to_key self.direction = 'f' if direction == 'f' else 'b' - self.limit = int(limit) if limit is not None else None + self.limit = min(int(limit), MAX_LIMIT) if limit is not None else None def __repr__(self): return ( @@ -49,7 +52,7 @@ class PaginationConfig(object): self.from_token = from_token self.to_token = to_token self.direction = 'f' if direction == 'f' else 'b' - self.limit = int(limit) if limit is not None else None + self.limit = min(int(limit), MAX_LIMIT) if limit is not None else None @classmethod def from_request(cls, request, raise_invalid_params=True, From a7927c13fd2181df75f1a3f3cf47889b0a240449 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 Jan 2016 10:53:44 +0000 Subject: [PATCH 078/294] Fix enabling & disabling push rules --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index df53824d2..0cbd9fe08 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -51,7 +51,7 @@ class PushRuleRestServlet(ClientV1RestServlet): content = _parse_json(request) if 'attr' in spec: - self.set_rule_attr(requester.user, spec, content) + self.set_rule_attr(requester.user.to_string(), spec, content) defer.returnValue((200, {})) try: From 2680043bc6a64053b93b9bab144aeb5f45007976 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 14 Jan 2016 14:34:01 +0000 Subject: [PATCH 079/294] Require ID and as_token be unique for ASs Defaults ID to as_token if not specified. This will change when IDs are fully supported. --- synapse/storage/appservice.py | 26 ++++++- tests/appservice/test_appservice.py | 1 + tests/storage/test_appservice.py | 101 +++++++++++++++++++++++----- 3 files changed, 111 insertions(+), 17 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index f4bc457ec..b5aa55c0a 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -20,6 +20,7 @@ from twisted.internet import defer from synapse.api.constants import Membership from synapse.appservice import ApplicationService, AppServiceTransaction +from synapse.config._base import ConfigError from synapse.storage.roommember import RoomsForUser from synapse.types import UserID from ._base import SQLBaseStore @@ -145,6 +146,7 @@ class ApplicationServiceStore(SQLBaseStore): def _load_appservice(self, as_info): required_string_fields = [ + # TODO: Add id here when it's stable to release "url", "as_token", "hs_token", "sender_localpart" ] for field in required_string_fields: @@ -186,7 +188,7 @@ class ApplicationServiceStore(SQLBaseStore): namespaces=as_info["namespaces"], hs_token=as_info["hs_token"], sender=user_id, - id=as_info["as_token"] # the token is the only unique thing here + id=as_info["id"] if "id" in as_info else as_info["as_token"], ) def _populate_appservice_cache(self, config_files): @@ -197,10 +199,32 @@ class ApplicationServiceStore(SQLBaseStore): ) return + # Dicts of value -> filename + seen_as_tokens = {} + seen_ids = {} + for config_file in config_files: try: with open(config_file, 'r') as f: appservice = self._load_appservice(yaml.load(f)) + if appservice.id in seen_ids: + raise ConfigError( + "Cannot reuse ID across application services: " + "%s (files: %s, %s)" % ( + appservice.id, config_file, seen_ids[appservice.id], + ) + ) + seen_ids[appservice.id] = config_file + if appservice.token in seen_as_tokens: + raise ConfigError( + "Cannot reuse as_token across application services: " + "%s (files: %s, %s)" % ( + appservice.token, + config_file, + seen_as_tokens[appservice.token], + ) + ) + seen_as_tokens[appservice.token] = config_file logger.info("Loaded application service: %s", appservice) self.services_cache.append(appservice) except Exception as e: diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 191c420c4..ef48bbc29 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -29,6 +29,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def setUp(self): self.service = ApplicationService( + id="unique_identifier", url="some_url", token="some_token", namespaces={ diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index a5a464640..5abecdf6e 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -12,12 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import tempfile +from synapse.config._base import ConfigError from tests import unittest from twisted.internet import defer from tests.utils import setup_test_homeserver from synapse.appservice import ApplicationService, ApplicationServiceState -from synapse.server import HomeServer from synapse.storage.appservice import ( ApplicationServiceStore, ApplicationServiceTransactionStore ) @@ -26,7 +27,6 @@ import json import os import yaml from mock import Mock -from tests.utils import SQLiteMemoryDbPool, MockClock class ApplicationServiceStoreTestCase(unittest.TestCase): @@ -41,9 +41,16 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): self.as_token = "token1" self.as_url = "some_url" - self._add_appservice(self.as_token, self.as_url, "some_hs_token", "bob") - self._add_appservice("token2", "some_url", "some_hs_token", "bob") - self._add_appservice("token3", "some_url", "some_hs_token", "bob") + self.as_id = "as1" + self._add_appservice( + self.as_token, + self.as_id, + self.as_url, + "some_hs_token", + "bob" + ) + self._add_appservice("token2", "as2", "some_url", "some_hs_token", "bob") + self._add_appservice("token3", "as3", "some_url", "some_hs_token", "bob") # must be done after inserts self.store = ApplicationServiceStore(hs) @@ -55,9 +62,9 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): except: pass - def _add_appservice(self, as_token, url, hs_token, sender): + def _add_appservice(self, as_token, id, url, hs_token, sender): as_yaml = dict(url=url, as_token=as_token, hs_token=hs_token, - sender_localpart=sender, namespaces={}) + id=id, sender_localpart=sender, namespaces={}) # use the token as the filename with open(as_token, 'w') as outfile: outfile.write(yaml.dump(as_yaml)) @@ -74,6 +81,7 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): self.as_token ) self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.id, self.as_id) self.assertEquals(stored_service.url, self.as_url) self.assertEquals( stored_service.namespaces[ApplicationService.NS_ALIASES], @@ -110,34 +118,34 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): { "token": "token1", "url": "https://matrix-as.org", - "id": "token1" + "id": "id_1" }, { "token": "alpha_tok", "url": "https://alpha.com", - "id": "alpha_tok" + "id": "id_alpha" }, { "token": "beta_tok", "url": "https://beta.com", - "id": "beta_tok" + "id": "id_beta" }, { - "token": "delta_tok", - "url": "https://delta.com", - "id": "delta_tok" + "token": "gamma_tok", + "url": "https://gamma.com", + "id": "id_gamma" }, ] for s in self.as_list: - yield self._add_service(s["url"], s["token"]) + yield self._add_service(s["url"], s["token"], s["id"]) self.as_yaml_files = [] self.store = TestTransactionStore(hs) - def _add_service(self, url, as_token): + def _add_service(self, url, as_token, id): as_yaml = dict(url=url, as_token=as_token, hs_token="something", - sender_localpart="a_sender", namespaces={}) + id=id, sender_localpart="a_sender", namespaces={}) # use the token as the filename with open(as_token, 'w') as outfile: outfile.write(yaml.dump(as_yaml)) @@ -405,3 +413,64 @@ class TestTransactionStore(ApplicationServiceTransactionStore, def __init__(self, hs): super(TestTransactionStore, self).__init__(hs) + + +class ApplicationServiceStoreConfigTestCase(unittest.TestCase): + + def _write_config(self, suffix, **kwargs): + vals = { + "id": "id" + suffix, + "url": "url" + suffix, + "as_token": "as_token" + suffix, + "hs_token": "hs_token" + suffix, + "sender_localpart": "sender_localpart" + suffix, + "namespaces": {}, + } + vals.update(kwargs) + + _, path = tempfile.mkstemp(prefix="as_config") + with open(path, "w") as f: + f.write(yaml.dump(vals)) + return path + + @defer.inlineCallbacks + def test_unique_works(self): + f1 = self._write_config(suffix="1") + f2 = self._write_config(suffix="2") + + config = Mock(app_service_config_files=[f1, f2]) + hs = yield setup_test_homeserver(config=config) + + ApplicationServiceStore(hs) + + @defer.inlineCallbacks + def test_duplicate_ids(self): + f1 = self._write_config(id="id", suffix="1") + f2 = self._write_config(id="id", suffix="2") + + config = Mock(app_service_config_files=[f1, f2]) + hs = yield setup_test_homeserver(config=config) + + with self.assertRaises(ConfigError) as cm: + ApplicationServiceStore(hs) + + e = cm.exception + self.assertIn(f1, e.message) + self.assertIn(f2, e.message) + self.assertIn("id", e.message) + + @defer.inlineCallbacks + def test_duplicate_as_tokens(self): + f1 = self._write_config(as_token="as_token", suffix="1") + f2 = self._write_config(as_token="as_token", suffix="2") + + config = Mock(app_service_config_files=[f1, f2]) + hs = yield setup_test_homeserver(config=config) + + with self.assertRaises(ConfigError) as cm: + ApplicationServiceStore(hs) + + e = cm.exception + self.assertIn(f1, e.message) + self.assertIn(f2, e.message) + self.assertIn("as_token", e.message) From 3f8db3d597dc631af02c31995426a3690746c8b5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 Jan 2016 17:21:04 +0000 Subject: [PATCH 080/294] Add specific error code for invalid user names. --- synapse/api/errors.py | 1 + synapse/handlers/register.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index be0c58a4c..e6d32acce 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -42,6 +42,7 @@ class Codes(object): EXCLUSIVE = "M_EXCLUSIVE" THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED" THREEPID_IN_USE = "THREEPID_IN_USE" + INVALID_USER_NAME = "M_INVALID_USER_NAME" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index ba26d13d4..83f4daaa8 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -53,7 +53,8 @@ class RegistrationHandler(BaseHandler): raise SynapseError( 400, "User ID must only contain characters which do not" - " require URL encoding." + " require URL encoding.", + Codes.INVALID_USER_NAME ) user = UserID(localpart, self.hs.hostname) From 5819b7a78ccbb07914d9c03ab426df084ba86f2c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 15 Jan 2016 10:06:34 +0000 Subject: [PATCH 081/294] M_INVALID_USERNAME to be consistent with the parameter name --- synapse/api/errors.py | 2 +- synapse/handlers/register.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e6d32acce..ce0fc5366 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -42,7 +42,7 @@ class Codes(object): EXCLUSIVE = "M_EXCLUSIVE" THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED" THREEPID_IN_USE = "THREEPID_IN_USE" - INVALID_USER_NAME = "M_INVALID_USER_NAME" + INVALID_USERNAME = "M_INVALID_USERNAME" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 83f4daaa8..8e601b052 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -54,7 +54,7 @@ class RegistrationHandler(BaseHandler): 400, "User ID must only contain characters which do not" " require URL encoding.", - Codes.INVALID_USER_NAME + Codes.INVALID_USERNAME ) user = UserID(localpart, self.hs.hostname) From b5ce4f042721aaa3ce26c268dcd29787386408cb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 15 Jan 2016 13:54:03 +0000 Subject: [PATCH 082/294] Remove unused parameters --- synapse/handlers/_base.py | 8 +++----- synapse/handlers/room.py | 16 +++++----------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 66e35de6e..bb2c6733d 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -171,12 +171,10 @@ class BaseHandler(object): ) @defer.inlineCallbacks - def handle_new_client_event(self, event, context, extra_destinations=[], - extra_users=[], suppress_auth=False): + def handle_new_client_event(self, event, context, extra_users=[]): # We now need to go and hit out to wherever we need to hit out to. - if not suppress_auth: - self.auth.check(event, auth_events=context.current_state) + self.auth.check(event, auth_events=context.current_state) yield self.maybe_kick_guest_users(event, context.current_state.values()) @@ -258,7 +256,7 @@ class BaseHandler(object): event, self ) - destinations = set(extra_destinations) + destinations = set() for k, s in context.current_state.items(): try: if k[0] == EventTypes.Member: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3a26f99a8..a410e4394 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -397,7 +397,7 @@ class RoomMemberHandler(BaseHandler): remotedomains.add(member.domain) @defer.inlineCallbacks - def change_membership(self, event, context, do_auth=True, is_guest=False): + def change_membership(self, event, context, is_guest=False): """ Change the membership status of a user in a room. Args: @@ -432,7 +432,7 @@ class RoomMemberHandler(BaseHandler): if not is_guest_access_allowed: raise AuthError(403, "Guest access not allowed") - yield self._do_join(event, context, do_auth=do_auth) + yield self._do_join(event, context) else: if event.membership == Membership.LEAVE: is_host_in_room = yield self.is_host_in_room(room_id, context) @@ -459,9 +459,7 @@ class RoomMemberHandler(BaseHandler): yield self._do_local_membership_update( event, - membership=event.content["membership"], context=context, - do_auth=do_auth, ) if prev_state and prev_state.membership == Membership.JOIN: @@ -497,12 +495,12 @@ class RoomMemberHandler(BaseHandler): }) event, context = yield self._create_new_client_event(builder) - yield self._do_join(event, context, room_hosts=hosts, do_auth=True) + yield self._do_join(event, context, room_hosts=hosts) defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def _do_join(self, event, context, room_hosts=None, do_auth=True): + def _do_join(self, event, context, room_hosts=None): room_id = event.room_id # XXX: We don't do an auth check if we are doing an invite @@ -536,9 +534,7 @@ class RoomMemberHandler(BaseHandler): yield self._do_local_membership_update( event, - membership=event.content["membership"], context=context, - do_auth=do_auth, ) prev_state = context.current_state.get((event.type, event.state_key)) @@ -603,8 +599,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue(room_ids) @defer.inlineCallbacks - def _do_local_membership_update(self, event, membership, context, - do_auth): + def _do_local_membership_update(self, event, context): yield run_on_reactor() target_user = UserID.from_string(event.state_key) @@ -613,7 +608,6 @@ class RoomMemberHandler(BaseHandler): event, context, extra_users=[target_user], - suppress_auth=(not do_auth), ) @defer.inlineCallbacks From ac5a4477adc772e4416c868e8b16ae41a2c0c4ef Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 15 Jan 2016 16:27:26 +0000 Subject: [PATCH 083/294] Require unbanning before other membership changes --- synapse/api/errors.py | 1 + synapse/handlers/federation.py | 4 +-- synapse/handlers/message.py | 57 +++++++++++++++++++++++++++------- synapse/handlers/room.py | 55 ++++++++++++++++++++++++++++++-- synapse/rest/client/v1/room.py | 51 +++++++----------------------- tests/handlers/test_room.py | 6 ++-- 6 files changed, 116 insertions(+), 58 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ce0fc5366..b106fbed6 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -29,6 +29,7 @@ class Codes(object): USER_IN_USE = "M_USER_IN_USE" ROOM_IN_USE = "M_ROOM_IN_USE" BAD_PAGINATION = "M_BAD_PAGINATION" + BAD_STATE = "M_BAD_STATE" UNKNOWN = "M_UNKNOWN" NOT_FOUND = "M_NOT_FOUND" MISSING_TOKEN = "M_MISSING_TOKEN" diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2f6359c76..26402ea9c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1693,7 +1693,7 @@ class FederationHandler(BaseHandler): self.auth.check(event, context.current_state) yield self._validate_keyserver(event, auth_events=context.current_state) member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.change_membership(event, context) + yield member_handler.send_membership_event(event, context) else: destinations = set([x.split(":", 1)[-1] for x in (sender, room_id)]) yield self.replication_layer.forward_third_party_invite( @@ -1722,7 +1722,7 @@ class FederationHandler(BaseHandler): # TODO: Make sure the signatures actually are correct. event.signatures.update(returned_invite.signatures) member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.change_membership(event, context) + yield member_handler.send_membership_event(event, context) @defer.inlineCallbacks def add_display_name_to_third_party_invite(self, event_dict, event, context): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5805190ce..4c7bf2bef 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -174,30 +174,25 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict, ratelimit=True, - token_id=None, txn_id=None, is_guest=False): - """ Given a dict from a client, create and handle a new event. + def create_event(self, event_dict, token_id=None, txn_id=None): + """ + Given a dict from a client, create a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, etc. Adds display names to Join membership events. - Persists and notifies local clients and federation. - Args: event_dict (dict): An entire event + + Returns: + Tuple of created event (FrozenEvent), Context """ builder = self.event_builder_factory.new(event_dict) self.validator.validate_new(builder) - if ratelimit: - self.ratelimit(builder.user_id) - # TODO(paul): Why does 'event' not have a 'user' object? - user = UserID.from_string(builder.user_id) - assert self.hs.is_mine(user), "User must be our own: %s" % (user,) - if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: @@ -216,6 +211,25 @@ class MessageHandler(BaseHandler): event, context = yield self._create_new_client_event( builder=builder, ) + defer.returnValue((event, context)) + + @defer.inlineCallbacks + def send_event(self, event, context, ratelimit=True, is_guest=False): + """ + Persists and notifies local clients and federation of an event. + + Args: + event (FrozenEvent) the event to send. + context (Context) the context of the event. + ratelimit (bool): Whether to rate limit this send. + is_guest (bool): Whether the sender is a guest. + """ + user = UserID.from_string(event.sender) + + assert self.hs.is_mine(user), "User must be our own: %s" % (user,) + + if ratelimit: + self.ratelimit(event.sender) if event.is_state(): prev_state = context.current_state.get((event.type, event.state_key)) @@ -229,7 +243,7 @@ class MessageHandler(BaseHandler): if event.type == EventTypes.Member: member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.change_membership(event, context, is_guest=is_guest) + yield member_handler.send_membership_event(event, context, is_guest=is_guest) else: yield self.handle_new_client_event( event=event, @@ -241,6 +255,25 @@ class MessageHandler(BaseHandler): with PreserveLoggingContext(): presence.bump_presence_active_time(user) + @defer.inlineCallbacks + def create_and_send_event(self, event_dict, ratelimit=True, + token_id=None, txn_id=None, is_guest=False): + """ + Creates an event, then sends it. + + See self.create_event and self.send_event. + """ + event, context = yield self.create_event( + event_dict, + token_id=token_id, + txn_id=txn_id + ) + yield self.send_event( + event, + context, + ratelimit=ratelimit, + is_guest=is_guest + ) defer.returnValue(event) @defer.inlineCallbacks diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a410e4394..a1baf9d20 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -22,7 +22,7 @@ from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import ( EventTypes, Membership, JoinRules, RoomCreationPreset, ) -from synapse.api.errors import AuthError, StoreError, SynapseError +from synapse.api.errors import AuthError, StoreError, SynapseError, Codes from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor @@ -397,7 +397,58 @@ class RoomMemberHandler(BaseHandler): remotedomains.add(member.domain) @defer.inlineCallbacks - def change_membership(self, event, context, is_guest=False): + def update_membership(self, requester, target, room_id, action, txn_id=None): + effective_membership_state = action + if action in ["kick", "unban"]: + effective_membership_state = "leave" + elif action == "forget": + effective_membership_state = "leave" + + msg_handler = self.hs.get_handlers().message_handler + + content = {"membership": unicode(effective_membership_state)} + if requester.is_guest: + content["kind"] = "guest" + + event, context = yield msg_handler.create_event( + { + "type": EventTypes.Member, + "content": content, + "room_id": room_id, + "sender": requester.user.to_string(), + "state_key": target.to_string(), + }, + token_id=requester.access_token_id, + txn_id=txn_id, + ) + + old_state = context.current_state.get((EventTypes.Member, event.state_key)) + old_membership = old_state.content.get("membership") if old_state else None + if action == "unban" and old_membership != "ban": + raise SynapseError( + 403, + "Cannot unban user who was not banned (membership=%s)" % old_membership, + errcode=Codes.BAD_STATE + ) + if old_membership == "ban" and action != "unban": + raise SynapseError( + 403, + "Cannot %s user who was is banned" % (action,), + errcode=Codes.BAD_STATE + ) + + yield msg_handler.send_event( + event, + context, + ratelimit=True, + is_guest=requester.is_guest + ) + + if action == "forget": + yield self.forget(requester.user, room_id) + + @defer.inlineCallbacks + def send_membership_event(self, event, context, is_guest=False): """ Change the membership status of a user in a room. Args: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 8b1b2b852..85b9f253e 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -442,7 +442,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): def register(self, http_server): # /rooms/$roomid/[invite|join|leave] PATTERNS = ("/rooms/(?P[^/]*)/" - "(?Pjoin|invite|leave|ban|kick|forget)") + "(?Pjoin|invite|leave|ban|unban|kick|forget)") register_txn_path(self, PATTERNS, http_server) @defer.inlineCallbacks @@ -451,9 +451,6 @@ class RoomMembershipRestServlet(ClientV1RestServlet): request, allow_guest=True, ) - user = requester.user - - effective_membership_action = membership_action if requester.is_guest and membership_action not in { Membership.JOIN, @@ -463,13 +460,10 @@ class RoomMembershipRestServlet(ClientV1RestServlet): content = _parse_json(request) - # target user is you unless it is an invite - state_key = user.to_string() - if membership_action == "invite" and self._has_3pid_invite_keys(content): yield self.handlers.room_member_handler.do_3pid_invite( room_id, - user, + requester.user, content["medium"], content["address"], content["id_server"], @@ -478,42 +472,21 @@ class RoomMembershipRestServlet(ClientV1RestServlet): ) defer.returnValue((200, {})) return - elif membership_action in ["invite", "ban", "kick"]: - if "user_id" in content: - state_key = content["user_id"] - else: + + target = requester.user + if membership_action in ["invite", "ban", "unban", "kick"]: + if "user_id" not in content: raise SynapseError(400, "Missing user_id key.") + target = UserID.from_string(content["user_id"]) - # make sure it looks like a user ID; it'll throw if it's invalid. - UserID.from_string(state_key) - - if membership_action == "kick": - effective_membership_action = "leave" - elif membership_action == "forget": - effective_membership_action = "leave" - - msg_handler = self.handlers.message_handler - - content = {"membership": unicode(effective_membership_action)} - if requester.is_guest: - content["kind"] = "guest" - - yield msg_handler.create_and_send_event( - { - "type": EventTypes.Member, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - "state_key": state_key, - }, - token_id=requester.access_token_id, + yield self.handlers.room_member_handler.update_membership( + requester=requester, + target=target, + room_id=room_id, + action=membership_action, txn_id=txn_id, - is_guest=requester.is_guest, ) - if membership_action == "forget": - yield self.handlers.room_member_handler.forget(user, room_id) - defer.returnValue((200, {})) def _has_3pid_invite_keys(self, content): diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 97491848a..e7a12a2ba 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -156,7 +156,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): builder ) - yield room_handler.change_membership(event, context) + yield room_handler.send_membership_event(event, context) self.state_handler.compute_event_context.assert_called_once_with( builder @@ -232,7 +232,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ) # Actual invocation - yield room_handler.change_membership(event, context) + yield room_handler.send_membership_event(event, context) self.federation.handle_new_event.assert_called_once_with( event, destinations=set() @@ -312,7 +312,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.distributor.observe("user_left_room", leave_signal_observer) # Actual invocation - yield room_handler.change_membership(event, context) + yield room_handler.send_membership_event(event, context) self.federation.handle_new_event.assert_called_once_with( event, destinations=set(['red']) From cc66a9a5e3fc954b0da48ba891e9f77be31aa832 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 10:45:09 +0000 Subject: [PATCH 084/294] Allow filtering events for multiple users at once --- synapse/handlers/_base.py | 93 ++++++++++++++++++++--------------- synapse/storage/roommember.py | 13 +++++ 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index bb2c6733d..2d1167296 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -53,16 +53,54 @@ class BaseHandler(object): self.event_builder_factory = hs.get_event_builder_factory() @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, events, is_guest=False): - # Assumes that user has at some point joined the room if not is_guest. + def _filter_events_for_clients(self, users, events): + """ Returns dict of user_id -> list of events that user is allowed to + see. + """ + event_id_to_state = yield self.store.get_state_for_events( + frozenset(e.event_id for e in events), + types=( + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, None), + ) + ) + + forgotten = yield defer.gatherResults([ + self.store.who_forgot_in_room( + room_id, + ) + for room_id in frozenset(e.room_id for e in events) + ], consumeErrors=True) + + # Set of membership event_ids that have been forgotten + event_id_forgotten = frozenset( + row["event_id"] for rows in forgotten for row in rows + ) + + def allowed(event, user_id, is_guest): + state = event_id_to_state[event.event_id] + + visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None) + if visibility_event: + visibility = visibility_event.content.get("history_visibility", "shared") + else: + visibility = "shared" - def allowed(event, membership, visibility): if visibility == "world_readable": return True if is_guest: return False + membership_event = state.get((EventTypes.Member, user_id), None) + if membership_event: + if membership_event.event_id in event_id_forgotten: + membership = None + else: + membership = membership_event.membership + else: + membership = None + if membership == Membership.JOIN: return True @@ -78,43 +116,20 @@ class BaseHandler(object): return True - event_id_to_state = yield self.store.get_state_for_events( - frozenset(e.event_id for e in events), - types=( - (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.Member, user_id), - ) - ) + defer.returnValue({ + user_id: [ + event + for event in events + if allowed(event, user_id, is_guest) + ] + for user_id, is_guest in users + }) - events_to_return = [] - for event in events: - state = event_id_to_state[event.event_id] - - membership_event = state.get((EventTypes.Member, user_id), None) - if membership_event: - was_forgotten_at_event = yield self.store.was_forgotten_at( - membership_event.state_key, - membership_event.room_id, - membership_event.event_id - ) - if was_forgotten_at_event: - membership = None - else: - membership = membership_event.membership - else: - membership = None - - visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None) - if visibility_event: - visibility = visibility_event.content.get("history_visibility", "shared") - else: - visibility = "shared" - - should_include = allowed(event, membership, visibility) - if should_include: - events_to_return.append(event) - - defer.returnValue(events_to_return) + @defer.inlineCallbacks + def _filter_events_for_client(self, user_id, events, is_guest=False): + # Assumes that user has at some point joined the room if not is_guest. + res = yield self._filter_events_for_clients([(user_id, is_guest)], events) + defer.returnValue(res.get(user_id, [])) def ratelimit(self, user_id): time_now = self.clock.time() diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 7d3ce4579..68ac88905 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -287,6 +287,7 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (user_id, room_id)) yield self.runInteraction("forget_membership", f) self.was_forgotten_at.invalidate_all() + self.who_forgot_in_room.invalidate_all() self.did_forget.invalidate((user_id, room_id)) @cachedInlineCallbacks(num_args=2) @@ -336,3 +337,15 @@ class RoomMemberStore(SQLBaseStore): return rows[0][0] forgot = yield self.runInteraction("did_forget_membership_at", f) defer.returnValue(forgot == 1) + + @cached() + def who_forgot_in_room(self, room_id): + return self._simple_select_list( + table="room_memberships", + retcols=("user_id", "event_id"), + keyvalues={ + "room_id": room_id, + "forgotten": 1, + }, + desc="who_forgot" + ) From 2068678b8cf40a1ec7f73bd9c400bb297e031454 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 10:46:09 +0000 Subject: [PATCH 085/294] Make Event objects behave more like dicts --- synapse/events/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 869a62309..bbfa5a726 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -117,6 +117,15 @@ class EventBase(object): def __set__(self, instance, value): raise AttributeError("Unrecognized attribute %s" % (instance,)) + def __getitem__(self, field): + return self._event_dict[field] + + def __contains__(self, field): + return field in self._event_dict + + def items(self): + return self._event_dict.items() + class FrozenEvent(EventBase): def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None): From f59b56450797746230046137b2e2008cb66cb604 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 14:09:47 +0000 Subject: [PATCH 086/294] Make notifications go quicker --- synapse/push/bulk_push_rule_evaluator.py | 112 ++++++----- synapse/push/push_rule_evaluator.py | 226 +++++++++++++++-------- synapse/storage/push_rule.py | 23 ++- synapse/storage/registration.py | 26 ++- 4 files changed, 258 insertions(+), 129 deletions(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index ce244fa95..b9f78fd59 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -14,16 +14,16 @@ # limitations under the License. import logging -import simplejson as json +import ujson as json from twisted.internet import defer +import baserules +from push_rule_evaluator import PushRuleEvaluatorForEvent + +from synapse.api.constants import EventTypes from synapse.types import UserID -import baserules -from push_rule_evaluator import PushRuleEvaluator - -from synapse.events.utils import serialize_event logger = logging.getLogger(__name__) @@ -35,28 +35,25 @@ def decode_rule_json(rule): @defer.inlineCallbacks -def evaluator_for_room_id(room_id, store): - users = yield store.get_users_in_room(room_id) - rules_by_user = yield store.bulk_get_push_rules(users) +def _get_rules(room_id, user_ids, store): + rules_by_user = yield store.bulk_get_push_rules(user_ids) rules_by_user = { uid: baserules.list_with_base_rules( - [decode_rule_json(rule_list) for rule_list in rules_by_user[uid]] - if uid in rules_by_user else [], + [decode_rule_json(rule_list) for rule_list in rules_by_user.get(uid, [])], UserID.from_string(uid), ) - for uid in users + for uid in user_ids } - member_events = yield store.get_current_state( - room_id=room_id, - event_type='m.room.member', - ) - display_names = {} - for ev in member_events: - if ev.content.get("displayname"): - display_names[ev.state_key] = ev.content.get("displayname") + defer.returnValue(rules_by_user) + + +@defer.inlineCallbacks +def evaluator_for_room_id(room_id, store): + users = yield store.get_users_in_room(room_id) + rules_by_user = yield _get_rules(room_id, users, store) defer.returnValue(BulkPushRuleEvaluator( - room_id, rules_by_user, display_names, users, store + room_id, rules_by_user, users, store )) @@ -69,10 +66,9 @@ class BulkPushRuleEvaluator: the same logic to run the actual rules, but could be optimised further (see https://matrix.org/jira/browse/SYN-562) """ - def __init__(self, room_id, rules_by_user, display_names, users_in_room, store): + def __init__(self, room_id, rules_by_user, users_in_room, store): self.room_id = room_id self.rules_by_user = rules_by_user - self.display_names = display_names self.users_in_room = users_in_room self.store = store @@ -80,15 +76,30 @@ class BulkPushRuleEvaluator: def action_for_event_by_user(self, event, handler): actions_by_user = {} - for uid, rules in self.rules_by_user.items(): - display_name = None - if uid in self.display_names: - display_name = self.display_names[uid] + users_dict = yield self.store.are_guests(self.rules_by_user.keys()) - is_guest = yield self.store.is_guest(UserID.from_string(uid)) - filtered = yield handler._filter_events_for_client( - uid, [event], is_guest=is_guest - ) + filtered_by_user = yield handler._filter_events_for_clients( + users_dict.items(), [event] + ) + + evaluator = PushRuleEvaluatorForEvent.create(event, len(self.users_in_room)) + + condition_cache = {} + + member_state = yield self.store.get_state_for_event( + event.event_id, + ) + + display_names = {} + for ev in member_state.values(): + nm = ev.content.get("displayname", None) + if nm and ev.type == EventTypes.Member: + display_names[ev.state_key] = nm + + for uid, rules in self.rules_by_user.items(): + display_name = display_names.get(uid, None) + + filtered = filtered_by_user[uid] if len(filtered) == 0: continue @@ -96,29 +107,32 @@ class BulkPushRuleEvaluator: if 'enabled' in rule and not rule['enabled']: continue - # XXX: profile tags - if BulkPushRuleEvaluator.event_matches_rule( - event, rule, - display_name, len(self.users_in_room), None - ): + matches = _condition_checker( + evaluator, rule['conditions'], display_name, condition_cache + ) + if matches: actions = [x for x in rule['actions'] if x != 'dont_notify'] - if len(actions) > 0: + if actions: actions_by_user[uid] = actions break defer.returnValue(actions_by_user) - @staticmethod - def event_matches_rule(event, rule, - display_name, room_member_count, profile_tag): - matches = True - # passing the clock all the way into here is extremely awkward and push - # rules do not care about any of the relative timestamps, so we just - # pass 0 for the current time. - client_event = serialize_event(event, 0) +def _condition_checker(evaluator, conditions, display_name, cache): + for cond in conditions: + _id = cond.get("_id", None) + if _id: + res = cache.get(_id, None) + if res is False: + break + elif res is True: + continue - for cond in rule['conditions']: - matches &= PushRuleEvaluator._event_fulfills_condition( - client_event, cond, display_name, room_member_count, profile_tag - ) - return matches + res = evaluator.matches(cond, display_name, None) + if _id: + cache[_id] = res + + if res is False: + return False + + return True diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index b0283743a..bbc8308c2 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -15,17 +15,22 @@ from twisted.internet import defer -from synapse.types import UserID - import baserules import logging import simplejson as json import re +from synapse.types import UserID + logger = logging.getLogger(__name__) +GLOB_REGEX = re.compile(r'\\\[(\\\!|)(.*)\\\]') +IS_GLOB = re.compile(r'[\?\*\[\]]') +INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") + + @defer.inlineCallbacks def evaluator_for_user_id_and_profile_tag(user_id, profile_tag, room_id, store): rawrules = yield store.get_push_rules_for_user(user_id) @@ -42,9 +47,34 @@ def evaluator_for_user_id_and_profile_tag(user_id, profile_tag, room_id, store): )) +def _room_member_count(ev, condition, room_member_count): + if 'is' not in condition: + return False + m = INEQUALITY_EXPR.match(condition['is']) + if not m: + return False + ineq = m.group(1) + rhs = m.group(2) + if not rhs.isdigit(): + return False + rhs = int(rhs) + + if ineq == '' or ineq == '==': + return room_member_count == rhs + elif ineq == '<': + return room_member_count < rhs + elif ineq == '>': + return room_member_count > rhs + elif ineq == '>=': + return room_member_count >= rhs + elif ineq == '<=': + return room_member_count <= rhs + else: + return False + + class PushRuleEvaluator: DEFAULT_ACTIONS = [] - INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") def __init__(self, user_id, profile_tag, raw_rules, enabled_map, room_id, our_member_event, store): @@ -98,6 +128,8 @@ class PushRuleEvaluator: room_members = yield self.store.get_users_in_room(room_id) room_member_count = len(room_members) + evaluator = PushRuleEvaluatorForEvent.create(ev, room_member_count) + for r in self.rules: if r['rule_id'] in self.enabled_map: r['enabled'] = self.enabled_map[r['rule_id']] @@ -105,21 +137,10 @@ class PushRuleEvaluator: r['enabled'] = True if not r['enabled']: continue - matches = True conditions = r['conditions'] actions = r['actions'] - for c in conditions: - matches &= self._event_fulfills_condition( - ev, c, display_name=my_display_name, - room_member_count=room_member_count, - profile_tag=self.profile_tag - ) - logger.debug( - "Rule %s %s", - r['rule_id'], "matches" if matches else "doesn't match" - ) # ignore rules with no actions (we have an explict 'dont_notify') if len(actions) == 0: logger.warn( @@ -127,6 +148,18 @@ class PushRuleEvaluator: r['rule_id'], self.user_id ) continue + + matches = True + for c in conditions: + matches = evaluator.matches(c, my_display_name, self.profile_tag) + if not matches: + break + + logger.debug( + "Rule %s %s", + r['rule_id'], "matches" if matches else "doesn't match" + ) + if matches: logger.info( "%s matches for user %s, event %s", @@ -145,81 +178,84 @@ class PushRuleEvaluator: ) defer.returnValue(PushRuleEvaluator.DEFAULT_ACTIONS) - @staticmethod - def _glob_to_regexp(glob): - r = re.escape(glob) - r = re.sub(r'\\\*', r'.*?', r) - r = re.sub(r'\\\?', r'.', r) - # handle [abc], [a-z] and [!a-z] style ranges. - r = re.sub(r'\\\[(\\\!|)(.*)\\\]', - lambda x: ('[%s%s]' % (x.group(1) and '^' or '', - re.sub(r'\\\-', '-', x.group(2)))), r) - return r +class PushRuleEvaluatorForEvent(object): + WORD_BOUNDARY = re.compile(r'\b') + + def __init__(self, event, body_parts, room_member_count): + self._event = event + self._body_parts = body_parts + self._room_member_count = room_member_count + + self._value_cache = _flatten_dict(event) @staticmethod - def _event_fulfills_condition(ev, condition, - display_name, room_member_count, profile_tag): + def create(event, room_member_count): + body = event.get("content", {}).get("body", None) + if body: + body_parts = PushRuleEvaluatorForEvent.WORD_BOUNDARY.split(body) + body_parts[:] = [ + part.lower() for part in body_parts + ] + else: + body_parts = [] + + return PushRuleEvaluatorForEvent(event, body_parts, room_member_count) + + def matches(self, condition, display_name, profile_tag): if condition['kind'] == 'event_match': - if 'pattern' not in condition: - logger.warn("event_match condition with no pattern") - return False - # XXX: optimisation: cache our pattern regexps - if condition['key'] == 'content.body': - r = r'\b%s\b' % PushRuleEvaluator._glob_to_regexp(condition['pattern']) - else: - r = r'^%s$' % PushRuleEvaluator._glob_to_regexp(condition['pattern']) - val = _value_for_dotted_key(condition['key'], ev) - if val is None: - return False - return re.search(r, val, flags=re.IGNORECASE) is not None - + return self._event_match(condition) elif condition['kind'] == 'device': if 'profile_tag' not in condition: return True return condition['profile_tag'] == profile_tag - elif condition['kind'] == 'contains_display_name': - # This is special because display names can be different - # between rooms and so you can't really hard code it in a rule. - # Optimisation: we should cache these names and update them from - # the event stream. - if 'content' not in ev or 'body' not in ev['content']: - return False - if not display_name: - return False - return re.search( - r"\b%s\b" % re.escape(display_name), ev['content']['body'], - flags=re.IGNORECASE - ) is not None - + return self._contains_display_name(display_name) elif condition['kind'] == 'room_member_count': - if 'is' not in condition: - return False - m = PushRuleEvaluator.INEQUALITY_EXPR.match(condition['is']) - if not m: - return False - ineq = m.group(1) - rhs = m.group(2) - if not rhs.isdigit(): - return False - rhs = int(rhs) - - if ineq == '' or ineq == '==': - return room_member_count == rhs - elif ineq == '<': - return room_member_count < rhs - elif ineq == '>': - return room_member_count > rhs - elif ineq == '>=': - return room_member_count >= rhs - elif ineq == '<=': - return room_member_count <= rhs - else: - return False + return _room_member_count( + self._event, condition, self._room_member_count + ) else: return True + def _event_match(self, condition): + pattern = condition.get('pattern', None) + + if not pattern: + logger.warn("event_match condition with no pattern") + return False + + # XXX: optimisation: cache our pattern regexps + if condition['key'] == 'content.body': + matcher = _glob_to_matcher(pattern) + + for part in self._body_parts: + if matcher(part): + return True + return False + else: + haystack = self._get_value(condition['key']) + if haystack is None: + return False + + matcher = _glob_to_matcher(pattern) + + return matcher(haystack.lower()) + + def _contains_display_name(self, display_name): + if not display_name: + return False + + lower_display_name = display_name.lower() + for part in self._body_parts: + if part == lower_display_name: + return True + + return False + + def _get_value(self, dotted_key): + return self._value_cache.get(dotted_key, None) + def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") @@ -229,4 +265,42 @@ def _value_for_dotted_key(dotted_key, event): return None val = val[parts[0]] parts = parts[1:] + return val + + +def _glob_to_matcher(glob): + glob = glob.lower() + + if not IS_GLOB.search(glob): + return lambda value: value == glob + + r = re.escape(glob) + + r = r.replace(r'\*', '.*?') + r = r.replace(r'\?', '.') + + # handle [abc], [a-z] and [!a-z] style ranges. + r = GLOB_REGEX.sub( + lambda x: ( + '[%s%s]' % ( + x.group(1) and '^' or '', + x.group(2).replace(r'\\\-', '-') + ) + ), + r, + ) + + r = r + "$" + r = re.compile(r) + return lambda value: r.match(value) + + +def _flatten_dict(d, prefix=[], result={}): + for key, value in d.items(): + if isinstance(value, basestring): + result[".".join(prefix + [key])] = value.lower() + elif hasattr(value, "items"): + _flatten_dict(value, prefix=(prefix+[key]), result=result) + + return result diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 2adfefd99..1adf28b89 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cachedInlineCallbacks +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList from twisted.internet import defer import logging @@ -60,6 +60,27 @@ class PushRuleStore(SQLBaseStore): r['rule_id']: False if r['enabled'] == 0 else True for r in results }) + @cached() + def _get_push_rules_enabled_for_user(self, user_id): + def f(txn): + sql = ( + "SELECT pr.*" + " FROM push_rules AS pr" + " LEFT JOIN push_rules_enable AS pre" + " ON pr.user_name = pre.user_name AND pr.rule_id = pre.rule_id" + " WHERE pr.user_name = ?" + " AND (pre.enabled IS NULL OR pre.enabled = 1)" + " ORDER BY pr.priority_class DESC, pr.priority DESC" + ) + txn.execute(sql, (user_id,)) + return self.cursor_to_dict(txn) + + return self.runInteraction( + "_get_push_rules_enabled_for_user", f + ) + + # @cachedList(cache=_get_push_rules_enabled_for_user.cache, list_name="user_ids", + # num_args=1, inlineCallbacks=True) @defer.inlineCallbacks def bulk_get_push_rules(self, user_ids): if not user_ids: diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 999b710fb..70cde0d04 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError, Codes from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cached, cachedInlineCallbacks +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList class RegistrationStore(SQLBaseStore): @@ -256,10 +256,10 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(res if res else False) @cachedInlineCallbacks() - def is_guest(self, user): + def is_guest(self, user_id): res = yield self._simple_select_one_onecol( table="users", - keyvalues={"name": user.to_string()}, + keyvalues={"name": user_id}, retcol="is_guest", allow_none=True, desc="is_guest", @@ -267,6 +267,26 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(res if res else False) + @cachedList(cache=is_guest.cache, list_name="user_ids", num_args=1, + inlineCallbacks=True) + def are_guests(self, user_ids): + sql = "SELECT name, is_guest FROM users WHERE name IN (%s)" % ( + ",".join("?" for _ in user_ids), + ) + + rows = yield self._execute( + "are_guests", self.cursor_to_dict, sql, *user_ids + ) + + result = {user_id: False for user_id in user_ids} + + result.update({ + row["name"]: bool(row["is_guest"]) + for row in rows + }) + + defer.returnValue(result) + def _query_for_auth(self, txn, token): sql = ( "SELECT users.name, users.is_guest, access_tokens.id as token_id" From 63485b30298f906efbdbe39311dd7503101e7602 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 14:11:43 +0000 Subject: [PATCH 087/294] Re-enable urnead notifications --- synapse/handlers/federation.py | 13 ++++++------- synapse/handlers/sync.py | 3 --- synapse/push/action_generator.py | 3 --- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 26402ea9c..4b94940e9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -36,7 +36,7 @@ from synapse.events.utils import prune_event from synapse.util.retryutils import NotRetryingDestination -# from synapse.push.action_generator import ActionGenerator +from synapse.push.action_generator import ActionGenerator from twisted.internet import defer @@ -244,12 +244,11 @@ class FederationHandler(BaseHandler): user = UserID.from_string(event.state_key) yield user_joined_room(self.distributor, user, event.room_id) - # Temporarily disable notifications due to performance concerns. - # if not backfilled and not event.internal_metadata.is_outlier(): - # action_generator = ActionGenerator(self.store) - # yield action_generator.handle_push_actions_for_event( - # event, self - # ) + if not backfilled and not event.internal_metadata.is_outlier(): + action_generator = ActionGenerator(self.store) + yield action_generator.handle_push_actions_for_event( + event, self + ) @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1942268c3..52202d8e6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -841,9 +841,6 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): - # Temporarily disable notifications due to performance concerns. - defer.returnValue([]) - last_unread_event_id = self.last_read_event_id_for_room_and_user( room_id, sync_config.user.to_string(), ephemeral_by_room ) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 73467f3ad..4cf94f6c6 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -36,9 +36,6 @@ class ActionGenerator: @defer.inlineCallbacks def handle_push_actions_for_event(self, event, handler): - # Temporarily disable notifications due to performance concerns. - return - if event.type == EventTypes.Redaction and event.redacts is not None: yield self.store.remove_push_actions_for_event_id( event.room_id, event.redacts From 2c176e02ae910ce52197539b31f78ae1b1ef4c3c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 14:24:31 +0000 Subject: [PATCH 088/294] Make unit tests work --- synapse/storage/push_rule.py | 2 +- tests/handlers/test_federation.py | 141 ---------- tests/handlers/test_room.py | 418 ------------------------------ 3 files changed, 1 insertion(+), 560 deletions(-) delete mode 100644 tests/handlers/test_federation.py delete mode 100644 tests/handlers/test_room.py diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 1adf28b89..f210e6c14 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks from twisted.internet import defer import logging diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py deleted file mode 100644 index 11a3d94bb..000000000 --- a/tests/handlers/test_federation.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2014-2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from twisted.internet import defer -from tests import unittest - -from synapse.api.constants import EventTypes -from synapse.events import FrozenEvent -from synapse.handlers.federation import FederationHandler - -from mock import NonCallableMock, ANY, Mock - -from ..utils import setup_test_homeserver - - -class FederationTestCase(unittest.TestCase): - - @defer.inlineCallbacks - def setUp(self): - - self.state_handler = NonCallableMock(spec_set=[ - "compute_event_context", - ]) - - self.auth = NonCallableMock(spec_set=[ - "check", - "check_host_in_room", - ]) - - self.hostname = "test" - hs = yield setup_test_homeserver( - self.hostname, - datastore=NonCallableMock(spec_set=[ - "persist_event", - "store_room", - "get_room", - "get_destination_retry_timings", - "set_destination_retry_timings", - "have_events", - "get_users_in_room", - "bulk_get_push_rules", - "get_current_state", - "set_push_actions_for_event_and_users", - "is_guest", - "get_state_for_events", - ]), - resource_for_federation=NonCallableMock(), - http_client=NonCallableMock(spec_set=[]), - notifier=NonCallableMock(spec_set=["on_new_room_event"]), - handlers=NonCallableMock(spec_set=[ - "room_member_handler", - "federation_handler", - ]), - auth=self.auth, - state_handler=self.state_handler, - keyring=Mock(), - ) - - self.datastore = hs.get_datastore() - self.handlers = hs.get_handlers() - self.notifier = hs.get_notifier() - self.hs = hs - - self.handlers.federation_handler = FederationHandler(self.hs) - - self.datastore.get_state_for_events.return_value = {"$a:b": {}} - - @defer.inlineCallbacks - def test_msg(self): - pdu = FrozenEvent({ - "type": EventTypes.Message, - "room_id": "foo", - "content": {"msgtype": u"fooo"}, - "origin_server_ts": 0, - "event_id": "$a:b", - "user_id":"@a:b", - "origin": "b", - "auth_events": [], - "hashes": {"sha256":"AcLrgtUIqqwaGoHhrEvYG1YLDIsVPYJdSRGhkp3jJp8"}, - }) - - self.datastore.persist_event.return_value = defer.succeed((1,1)) - self.datastore.get_room.return_value = defer.succeed(True) - self.datastore.get_users_in_room.return_value = ["@a:b"] - self.datastore.bulk_get_push_rules.return_value = {} - self.datastore.get_current_state.return_value = {} - self.auth.check_host_in_room.return_value = defer.succeed(True) - - retry_timings_res = { - "destination": "", - "retry_last_ts": 0, - "retry_interval": 0, - } - self.datastore.get_destination_retry_timings.return_value = ( - defer.succeed(retry_timings_res) - ) - - def have_events(event_ids): - return defer.succeed({}) - self.datastore.have_events.side_effect = have_events - - def annotate(ev, old_state=None, outlier=False): - context = Mock() - context.current_state = {} - context.auth_events = {} - return defer.succeed(context) - self.state_handler.compute_event_context.side_effect = annotate - - yield self.handlers.federation_handler.on_receive_pdu( - "fo", pdu, False - ) - - self.datastore.persist_event.assert_called_once_with( - ANY, - is_new_state=True, - backfilled=False, - current_state=None, - context=ANY, - ) - - self.state_handler.compute_event_context.assert_called_once_with( - ANY, old_state=None, outlier=False - ) - - self.auth.check.assert_called_once_with(ANY, auth_events={}) - - self.notifier.on_new_room_event.assert_called_once_with( - ANY, 1, 1, extra_users=[] - ) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py deleted file mode 100644 index e7a12a2ba..000000000 --- a/tests/handlers/test_room.py +++ /dev/null @@ -1,418 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014-2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from twisted.internet import defer -from .. import unittest - -from synapse.api.constants import EventTypes, Membership -from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler -from synapse.handlers.profile import ProfileHandler -from synapse.types import UserID -from ..utils import setup_test_homeserver - -from mock import Mock, NonCallableMock - - -class RoomMemberHandlerTestCase(unittest.TestCase): - - @defer.inlineCallbacks - def setUp(self): - self.hostname = "red" - hs = yield setup_test_homeserver( - self.hostname, - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - datastore=NonCallableMock(spec_set=[ - "persist_event", - "get_room_member", - "get_room", - "store_room", - "get_latest_events_in_room", - "add_event_hashes", - "get_users_in_room", - "bulk_get_push_rules", - "get_current_state", - "set_push_actions_for_event_and_users", - "get_state_for_events", - "is_guest", - ]), - resource_for_federation=NonCallableMock(), - http_client=NonCallableMock(spec_set=[]), - notifier=NonCallableMock(spec_set=["on_new_room_event"]), - handlers=NonCallableMock(spec_set=[ - "room_member_handler", - "profile_handler", - "federation_handler", - ]), - auth=NonCallableMock(spec_set=[ - "check", - "add_auth_events", - "check_host_in_room", - ]), - state_handler=NonCallableMock(spec_set=[ - "compute_event_context", - "get_current_state", - ]), - ) - - self.federation = NonCallableMock(spec_set=[ - "handle_new_event", - "send_invite", - "get_state_for_room", - ]) - - self.datastore = hs.get_datastore() - self.handlers = hs.get_handlers() - self.notifier = hs.get_notifier() - self.state_handler = hs.get_state_handler() - self.distributor = hs.get_distributor() - self.auth = hs.get_auth() - self.hs = hs - - self.handlers.federation_handler = self.federation - - self.distributor.declare("collect_presencelike_data") - - self.handlers.room_member_handler = RoomMemberHandler(self.hs) - self.handlers.profile_handler = ProfileHandler(self.hs) - self.room_member_handler = self.handlers.room_member_handler - - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - self.datastore.persist_event.return_value = (1,1) - self.datastore.add_event_hashes.return_value = [] - self.datastore.get_users_in_room.return_value = ["@bob:red"] - self.datastore.bulk_get_push_rules.return_value = {} - - @defer.inlineCallbacks - def test_invite(self): - room_id = "!foo:red" - user_id = "@bob:red" - target_user_id = "@red:blue" - content = {"membership": Membership.INVITE} - - builder = self.hs.get_event_builder_factory().new({ - "type": EventTypes.Member, - "sender": user_id, - "state_key": target_user_id, - "room_id": room_id, - "content": content, - }) - - self.datastore.get_latest_events_in_room.return_value = ( - defer.succeed([]) - ) - self.datastore.get_current_state.return_value = {} - self.datastore.get_state_for_events = lambda event_ids,types: {x: {} for x in event_ids} - - def annotate(_): - ctx = Mock() - ctx.current_state = { - (EventTypes.Member, "@alice:green"): self._create_member( - user_id="@alice:green", - room_id=room_id, - ), - (EventTypes.Member, "@bob:red"): self._create_member( - user_id="@bob:red", - room_id=room_id, - ), - } - ctx.prev_state_events = [] - - return defer.succeed(ctx) - - self.state_handler.compute_event_context.side_effect = annotate - - def add_auth(_, ctx): - ctx.auth_events = ctx.current_state[ - (EventTypes.Member, "@bob:red") - ] - - return defer.succeed(True) - self.auth.add_auth_events.side_effect = add_auth - - def send_invite(domain, event): - return defer.succeed(event) - - self.federation.send_invite.side_effect = send_invite - - room_handler = self.room_member_handler - event, context = yield room_handler._create_new_client_event( - builder - ) - - yield room_handler.send_membership_event(event, context) - - self.state_handler.compute_event_context.assert_called_once_with( - builder - ) - - self.auth.add_auth_events.assert_called_once_with( - builder, context - ) - - self.federation.send_invite.assert_called_once_with( - "blue", event, - ) - - self.datastore.persist_event.assert_called_once_with( - event, context=context, - ) - self.notifier.on_new_room_event.assert_called_once_with( - event, 1, 1, extra_users=[UserID.from_string(target_user_id)] - ) - self.assertFalse(self.datastore.get_room.called) - self.assertFalse(self.datastore.store_room.called) - self.assertFalse(self.federation.get_state_for_room.called) - - @defer.inlineCallbacks - def test_simple_join(self): - room_id = "!foo:red" - user_id = "@bob:red" - user = UserID.from_string(user_id) - - join_signal_observer = Mock() - self.distributor.observe("user_joined_room", join_signal_observer) - - builder = self.hs.get_event_builder_factory().new({ - "type": EventTypes.Member, - "sender": user_id, - "state_key": user_id, - "room_id": room_id, - "content": {"membership": Membership.JOIN}, - }) - - self.datastore.get_latest_events_in_room.return_value = ( - defer.succeed([]) - ) - self.datastore.get_current_state.return_value = {} - self.datastore.get_state_for_events = lambda event_ids,types: {x: {} for x in event_ids} - - def annotate(_): - ctx = Mock() - ctx.current_state = { - (EventTypes.Member, "@bob:red"): self._create_member( - user_id="@bob:red", - room_id=room_id, - membership=Membership.INVITE - ), - } - ctx.prev_state_events = [] - - return defer.succeed(ctx) - - self.state_handler.compute_event_context.side_effect = annotate - - def add_auth(_, ctx): - ctx.auth_events = ctx.current_state[ - (EventTypes.Member, "@bob:red") - ] - - return defer.succeed(True) - self.auth.add_auth_events.side_effect = add_auth - - room_handler = self.room_member_handler - event, context = yield room_handler._create_new_client_event( - builder - ) - - # Actual invocation - yield room_handler.send_membership_event(event, context) - - self.federation.handle_new_event.assert_called_once_with( - event, destinations=set() - ) - - self.datastore.persist_event.assert_called_once_with( - event, context=context - ) - self.notifier.on_new_room_event.assert_called_once_with( - event, 1, 1, extra_users=[user] - ) - - join_signal_observer.assert_called_with( - user=user, room_id=room_id - ) - - def _create_member(self, user_id, room_id, membership=Membership.JOIN): - builder = self.hs.get_event_builder_factory().new({ - "type": EventTypes.Member, - "sender": user_id, - "state_key": user_id, - "room_id": room_id, - "content": {"membership": membership}, - }) - - return builder.build() - - @defer.inlineCallbacks - def test_simple_leave(self): - room_id = "!foo:red" - user_id = "@bob:red" - user = UserID.from_string(user_id) - - builder = self.hs.get_event_builder_factory().new({ - "type": EventTypes.Member, - "sender": user_id, - "state_key": user_id, - "room_id": room_id, - "content": {"membership": Membership.LEAVE}, - }) - - self.datastore.get_latest_events_in_room.return_value = ( - defer.succeed([]) - ) - self.datastore.get_current_state.return_value = {} - self.datastore.get_state_for_events = lambda event_ids,types: {x: {} for x in event_ids} - - def annotate(_): - ctx = Mock() - ctx.current_state = { - (EventTypes.Member, "@bob:red"): self._create_member( - user_id="@bob:red", - room_id=room_id, - membership=Membership.JOIN - ), - } - ctx.prev_state_events = [] - - return defer.succeed(ctx) - - self.state_handler.compute_event_context.side_effect = annotate - - def add_auth(_, ctx): - ctx.auth_events = ctx.current_state[ - (EventTypes.Member, "@bob:red") - ] - - return defer.succeed(True) - self.auth.add_auth_events.side_effect = add_auth - - room_handler = self.room_member_handler - event, context = yield room_handler._create_new_client_event( - builder - ) - - leave_signal_observer = Mock() - self.distributor.observe("user_left_room", leave_signal_observer) - - # Actual invocation - yield room_handler.send_membership_event(event, context) - - self.federation.handle_new_event.assert_called_once_with( - event, destinations=set(['red']) - ) - - self.datastore.persist_event.assert_called_once_with( - event, context=context - ) - self.notifier.on_new_room_event.assert_called_once_with( - event, 1, 1, extra_users=[user] - ) - - leave_signal_observer.assert_called_with( - user=user, room_id=room_id - ) - - -class RoomCreationTest(unittest.TestCase): - - @defer.inlineCallbacks - def setUp(self): - self.hostname = "red" - - hs = yield setup_test_homeserver( - self.hostname, - datastore=NonCallableMock(spec_set=[ - "store_room", - "snapshot_room", - "persist_event", - "get_joined_hosts_for_room", - ]), - http_client=NonCallableMock(spec_set=[]), - notifier=NonCallableMock(spec_set=["on_new_room_event"]), - handlers=NonCallableMock(spec_set=[ - "room_creation_handler", - "message_handler", - ]), - auth=NonCallableMock(spec_set=["check", "add_auth_events"]), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - ) - - self.federation = NonCallableMock(spec_set=[ - "handle_new_event", - ]) - - self.handlers = hs.get_handlers() - - self.handlers.room_creation_handler = RoomCreationHandler(hs) - self.room_creation_handler = self.handlers.room_creation_handler - - self.message_handler = self.handlers.message_handler - - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - @defer.inlineCallbacks - def test_room_creation(self): - user_id = "@foo:red" - room_id = "!bobs_room:red" - config = {"visibility": "private"} - - yield self.room_creation_handler.create_room( - user_id=user_id, - room_id=room_id, - config=config, - ) - - self.assertTrue(self.message_handler.create_and_send_event.called) - - event_dicts = [ - e[0][0] - for e in self.message_handler.create_and_send_event.call_args_list - ] - - self.assertTrue(len(event_dicts) > 3) - - self.assertDictContainsSubset( - { - "type": EventTypes.Create, - "sender": user_id, - "room_id": room_id, - }, - event_dicts[0] - ) - - self.assertEqual(user_id, event_dicts[0]["content"]["creator"]) - - self.assertDictContainsSubset( - { - "type": EventTypes.Member, - "sender": user_id, - "room_id": room_id, - "state_key": user_id, - }, - event_dicts[1] - ) - - self.assertEqual( - Membership.JOIN, - event_dicts[1]["content"]["membership"] - ) From 345ff2196a8b381a3dfea537ab659aac0837045a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 14:39:34 +0000 Subject: [PATCH 089/294] Don't edit ruleset --- synapse/push/push_rule_evaluator.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index bbc8308c2..60d9f1f23 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -131,11 +131,10 @@ class PushRuleEvaluator: evaluator = PushRuleEvaluatorForEvent.create(ev, room_member_count) for r in self.rules: - if r['rule_id'] in self.enabled_map: - r['enabled'] = self.enabled_map[r['rule_id']] - elif 'enabled' not in r: - r['enabled'] = True - if not r['enabled']: + if self.enabled_map.get(r['rule_id'], None) is False: + continue + + if not r.get("enabled", True): continue conditions = r['conditions'] From 0e39dcd1352bcab53908fca574cae43a89e407f0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 14:39:43 +0000 Subject: [PATCH 090/294] Remove internal ids --- synapse/rest/client/v1/push_rule.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 0cbd9fe08..b176efd8a 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -140,6 +140,10 @@ class PushRuleRestServlet(ClientV1RestServlet): template_name = _priority_class_to_template_name(r['priority_class']) + # Remove internal stuff. + for c in r["conditions"]: + c.pop("_id", None) + if r['priority_class'] > PRIORITY_CLASS_MAP['override']: # per-device rule profile_tag = _profile_tag_from_conditions(r["conditions"]) From d1f56f732e1c213e203f287945d84966c3eec6f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 10:09:14 +0000 Subject: [PATCH 091/294] Use static for const dicts --- synapse/push/baserules.py | 364 +++++++++++------------ synapse/push/bulk_push_rule_evaluator.py | 15 +- synapse/push/push_rule_evaluator.py | 20 +- synapse/rest/client/v1/push_rule.py | 8 +- 4 files changed, 209 insertions(+), 198 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 8bac7fd6a..d8a0eda9f 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -15,27 +15,25 @@ from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP -def list_with_base_rules(rawrules, user_id): +def list_with_base_rules(rawrules): ruleslist = [] # shove the server default rules for each kind onto the end of each current_prio_class = PRIORITY_CLASS_INVERSE_MAP.keys()[-1] ruleslist.extend(make_base_prepend_rules( - user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) for r in rawrules: if r['priority_class'] < current_prio_class: while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_append_rules( - user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) current_prio_class -= 1 if current_prio_class > 0: ruleslist.extend(make_base_prepend_rules( - user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) @@ -43,28 +41,26 @@ def list_with_base_rules(rawrules, user_id): while current_prio_class > 0: ruleslist.extend(make_base_append_rules( - user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) current_prio_class -= 1 if current_prio_class > 0: ruleslist.extend(make_base_prepend_rules( - user_id, PRIORITY_CLASS_INVERSE_MAP[current_prio_class] )) return ruleslist -def make_base_append_rules(user, kind): +def make_base_append_rules(kind): rules = [] if kind == 'override': - rules = make_base_append_override_rules() + rules = BASE_APPEND_OVRRIDE_RULES elif kind == 'underride': - rules = make_base_append_underride_rules(user) + rules = BASE_APPEND_UNDERRIDE_RULES elif kind == 'content': - rules = make_base_append_content_rules(user) + rules = BASE_APPEND_CONTENT_RULES for r in rules: r['priority_class'] = PRIORITY_CLASS_MAP[kind] @@ -73,11 +69,11 @@ def make_base_append_rules(user, kind): return rules -def make_base_prepend_rules(user, kind): +def make_base_prepend_rules(kind): rules = [] if kind == 'override': - rules = make_base_prepend_override_rules() + rules = BASE_PREPEND_OVERRIDE_RULES for r in rules: r['priority_class'] = PRIORITY_CLASS_MAP[kind] @@ -86,180 +82,182 @@ def make_base_prepend_rules(user, kind): return rules -def make_base_append_content_rules(user): - return [ - { - 'rule_id': 'global/content/.m.rule.contains_user_name', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'content.body', - 'pattern': user.localpart, # Matrix ID match - } - ], - 'actions': [ - 'notify', - { - 'set_tweak': 'sound', - 'value': 'default', - }, { - 'set_tweak': 'highlight' - } - ] - }, - ] +BASE_APPEND_CONTENT_RULES = [ + { + 'rule_id': 'global/content/.m.rule.contains_user_name', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.body', + 'pattern_type': 'user_localpart' + } + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'default', + }, { + 'set_tweak': 'highlight' + } + ] + }, +] -def make_base_prepend_override_rules(): - return [ - { - 'rule_id': 'global/override/.m.rule.master', - 'enabled': False, - 'conditions': [], - 'actions': [ - "dont_notify" - ] - } - ] +BASE_PREPEND_OVERRIDE_RULES = [ + { + 'rule_id': 'global/override/.m.rule.master', + 'enabled': False, + 'conditions': [], + 'actions': [ + "dont_notify" + ] + } +] -def make_base_append_override_rules(): - return [ - { - 'rule_id': 'global/override/.m.rule.suppress_notices', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'content.msgtype', - 'pattern': 'm.notice', - } - ], - 'actions': [ - 'dont_notify', - ] - } - ] +BASE_APPEND_OVRRIDE_RULES = [ + { + 'rule_id': 'global/override/.m.rule.suppress_notices', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.msgtype', + 'pattern': 'm.notice', + '_id': '_suppress_notices', + } + ], + 'actions': [ + 'dont_notify', + ] + } +] -def make_base_append_underride_rules(user): - return [ - { - 'rule_id': 'global/underride/.m.rule.call', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'type', - 'pattern': 'm.call.invite', - } - ], - 'actions': [ - 'notify', - { - 'set_tweak': 'sound', - 'value': 'ring' - }, { - 'set_tweak': 'highlight', - 'value': False - } - ] - }, - { - 'rule_id': 'global/underride/.m.rule.contains_display_name', - 'conditions': [ - { - 'kind': 'contains_display_name' - } - ], - 'actions': [ - 'notify', - { - 'set_tweak': 'sound', - 'value': 'default' - }, { - 'set_tweak': 'highlight' - } - ] - }, - { - 'rule_id': 'global/underride/.m.rule.room_one_to_one', - 'conditions': [ - { - 'kind': 'room_member_count', - 'is': '2' - } - ], - 'actions': [ - 'notify', - { - 'set_tweak': 'sound', - 'value': 'default' - }, { - 'set_tweak': 'highlight', - 'value': False - } - ] - }, - { - 'rule_id': 'global/underride/.m.rule.invite_for_me', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'type', - 'pattern': 'm.room.member', - }, - { - 'kind': 'event_match', - 'key': 'content.membership', - 'pattern': 'invite', - }, - { - 'kind': 'event_match', - 'key': 'state_key', - 'pattern': user.to_string(), - }, - ], - 'actions': [ - 'notify', - { - 'set_tweak': 'sound', - 'value': 'default' - }, { - 'set_tweak': 'highlight', - 'value': False - } - ] - }, - { - 'rule_id': 'global/underride/.m.rule.member_event', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'type', - 'pattern': 'm.room.member', - } - ], - 'actions': [ - 'notify', { - 'set_tweak': 'highlight', - 'value': False - } - ] - }, - { - 'rule_id': 'global/underride/.m.rule.message', - 'enabled': False, - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'type', - 'pattern': 'm.room.message', - } - ], - 'actions': [ - 'notify', { - 'set_tweak': 'highlight', - 'value': False - } - ] - } - ] +BASE_APPEND_UNDERRIDE_RULES = [ + { + 'rule_id': 'global/underride/.m.rule.call', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.call.invite', + '_id': '_call', + } + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'ring' + }, { + 'set_tweak': 'highlight', + 'value': False + } + ] + }, + { + 'rule_id': 'global/underride/.m.rule.contains_display_name', + 'conditions': [ + { + 'kind': 'contains_display_name' + } + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'default' + }, { + 'set_tweak': 'highlight' + } + ] + }, + { + 'rule_id': 'global/underride/.m.rule.room_one_to_one', + 'conditions': [ + { + 'kind': 'room_member_count', + 'is': '2' + } + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'default' + }, { + 'set_tweak': 'highlight', + 'value': False + } + ] + }, + { + 'rule_id': 'global/underride/.m.rule.invite_for_me', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.room.member', + '_id': '_invite_type', + }, + { + 'kind': 'event_match', + 'key': 'content.membership', + 'pattern': 'invite', + '_id': '_invite_member', + }, + { + 'kind': 'event_match', + 'key': 'state_key', + 'pattern_type': 'user_id' + }, + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'default' + }, { + 'set_tweak': 'highlight', + 'value': False + } + ] + }, + { + 'rule_id': 'global/underride/.m.rule.member_event', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.room.member', + '_id': '_member', + } + ], + 'actions': [ + 'notify', { + 'set_tweak': 'highlight', + 'value': False + } + ] + }, + { + 'rule_id': 'global/underride/.m.rule.message', + 'enabled': False, + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.room.message', + '_id': '_message', + } + ], + 'actions': [ + 'notify', { + 'set_tweak': 'highlight', + 'value': False + } + ] + } +] diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index b9f78fd59..f1910f7da 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -22,7 +22,6 @@ import baserules from push_rule_evaluator import PushRuleEvaluatorForEvent from synapse.api.constants import EventTypes -from synapse.types import UserID logger = logging.getLogger(__name__) @@ -38,10 +37,10 @@ def decode_rule_json(rule): def _get_rules(room_id, user_ids, store): rules_by_user = yield store.bulk_get_push_rules(user_ids) rules_by_user = { - uid: baserules.list_with_base_rules( - [decode_rule_json(rule_list) for rule_list in rules_by_user.get(uid, [])], - UserID.from_string(uid), - ) + uid: baserules.list_with_base_rules([ + decode_rule_json(rule_list) + for rule_list in rules_by_user.get(uid, []) + ]) for uid in user_ids } defer.returnValue(rules_by_user) @@ -108,7 +107,7 @@ class BulkPushRuleEvaluator: continue matches = _condition_checker( - evaluator, rule['conditions'], display_name, condition_cache + evaluator, rule['conditions'], uid, display_name, condition_cache ) if matches: actions = [x for x in rule['actions'] if x != 'dont_notify'] @@ -118,7 +117,7 @@ class BulkPushRuleEvaluator: defer.returnValue(actions_by_user) -def _condition_checker(evaluator, conditions, display_name, cache): +def _condition_checker(evaluator, conditions, uid, display_name, cache): for cond in conditions: _id = cond.get("_id", None) if _id: @@ -128,7 +127,7 @@ def _condition_checker(evaluator, conditions, display_name, cache): elif res is True: continue - res = evaluator.matches(cond, display_name, None) + res = evaluator.matches(cond, uid, display_name, None) if _id: cache[_id] = res diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 60d9f1f23..9332fe5c5 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -91,8 +91,7 @@ class PushRuleEvaluator: rule['actions'] = json.loads(raw_rule['actions']) rules.append(rule) - user = UserID.from_string(self.user_id) - self.rules = baserules.list_with_base_rules(rules, user) + self.rules = baserules.list_with_base_rules(rules) self.enabled_map = enabled_map @@ -150,7 +149,9 @@ class PushRuleEvaluator: matches = True for c in conditions: - matches = evaluator.matches(c, my_display_name, self.profile_tag) + matches = evaluator.matches( + c, self.user_id, my_display_name, self.profile_tag + ) if not matches: break @@ -201,9 +202,9 @@ class PushRuleEvaluatorForEvent(object): return PushRuleEvaluatorForEvent(event, body_parts, room_member_count) - def matches(self, condition, display_name, profile_tag): + def matches(self, condition, user_id, display_name, profile_tag): if condition['kind'] == 'event_match': - return self._event_match(condition) + return self._event_match(condition, user_id) elif condition['kind'] == 'device': if 'profile_tag' not in condition: return True @@ -217,9 +218,16 @@ class PushRuleEvaluatorForEvent(object): else: return True - def _event_match(self, condition): + def _event_match(self, condition, user_id): pattern = condition.get('pattern', None) + if not pattern: + pattern_type = condition.get('pattern_type', None) + if pattern_type == "user_id": + pattern = user_id + elif pattern_type == "user_localpart": + pattern = UserID.from_string(user_id).localpart + if not pattern: logger.warn("event_match condition with no pattern") return False diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b176efd8a..aa861e703 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -126,7 +126,7 @@ class PushRuleRestServlet(ClientV1RestServlet): rule["actions"] = json.loads(rawrule["actions"]) ruleslist.append(rule) - ruleslist = baserules.list_with_base_rules(ruleslist, user) + ruleslist = baserules.list_with_base_rules(ruleslist) rules = {'global': {}, 'device': {}} @@ -144,6 +144,12 @@ class PushRuleRestServlet(ClientV1RestServlet): for c in r["conditions"]: c.pop("_id", None) + pattern_type = c.pop("pattern_type", None) + if pattern_type == "user_id": + c["pattern"] = user.to_string() + elif pattern_type == "user_localpart": + c["pattern"] = user.localpart + if r['priority_class'] > PRIORITY_CLASS_MAP['override']: # per-device rule profile_tag = _profile_tag_from_conditions(r["conditions"]) From 866fe27e783697f74f45573d0eb202e3803fe756 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 15:29:41 +0000 Subject: [PATCH 092/294] Do for loop once at start --- synapse/push/baserules.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index d8a0eda9f..8be47f097 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -62,10 +62,6 @@ def make_base_append_rules(kind): elif kind == 'content': rules = BASE_APPEND_CONTENT_RULES - for r in rules: - r['priority_class'] = PRIORITY_CLASS_MAP[kind] - r['default'] = True # Deprecated, left for backwards compat - return rules @@ -75,10 +71,6 @@ def make_base_prepend_rules(kind): if kind == 'override': rules = BASE_PREPEND_OVERRIDE_RULES - for r in rules: - r['priority_class'] = PRIORITY_CLASS_MAP[kind] - r['default'] = True # Deprecated, left for backwards compat - return rules @@ -261,3 +253,20 @@ BASE_APPEND_UNDERRIDE_RULES = [ ] } ] + + +for r in BASE_APPEND_CONTENT_RULES: + r['priority_class'] = PRIORITY_CLASS_MAP['content'] + r['default'] = True + +for r in BASE_PREPEND_OVERRIDE_RULES: + r['priority_class'] = PRIORITY_CLASS_MAP['override'] + r['default'] = True + +for r in BASE_APPEND_OVRRIDE_RULES: + r['priority_class'] = PRIORITY_CLASS_MAP['override'] + r['default'] = True + +for r in BASE_APPEND_UNDERRIDE_RULES: + r['priority_class'] = PRIORITY_CLASS_MAP['underride'] + r['default'] = True From 7dd14e5d1c8950f50279efccce83b9ff30f0bcfb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 15:42:23 +0000 Subject: [PATCH 093/294] Add comments and remove dead code --- synapse/push/push_rule_evaluator.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 9332fe5c5..eab7fc7d5 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -184,9 +184,14 @@ class PushRuleEvaluatorForEvent(object): def __init__(self, event, body_parts, room_member_count): self._event = event + + # This is a list of words of the content.body (if event has one). Each + # word has been converted to lower case. self._body_parts = body_parts + self._room_member_count = room_member_count + # Maps strings of e.g. 'content.body' -> event["content"]["body"] self._value_cache = _flatten_dict(event) @staticmethod @@ -264,19 +269,13 @@ class PushRuleEvaluatorForEvent(object): return self._value_cache.get(dotted_key, None) -def _value_for_dotted_key(dotted_key, event): - parts = dotted_key.split(".") - val = event - while len(parts) > 0: - if parts[0] not in val: - return None - val = val[parts[0]] - parts = parts[1:] - - return val - - def _glob_to_matcher(glob): + """Takes a glob and returns a `func(string) -> bool`, which returns if the + string matches the glob. Assumes given string is lower case. + + The matcher returned is either a simple string comparison for globs without + wildcards, or a regex matcher for globs with wildcards. + """ glob = glob.lower() if not IS_GLOB.search(glob): From d16dcf642e56e2ad3e4e6fb5834844251f5383f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 15:44:04 +0000 Subject: [PATCH 094/294] Drop log levels --- synapse/push/push_rule_evaluator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index eab7fc7d5..0816b632b 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -161,7 +161,7 @@ class PushRuleEvaluator: ) if matches: - logger.info( + logger.debug( "%s matches for user %s, event %s", r['rule_id'], self.user_id, ev['event_id'] ) @@ -172,7 +172,7 @@ class PushRuleEvaluator: defer.returnValue(actions) - logger.info( + logger.debug( "No rules match for user %s, event %s", self.user_id, ev['event_id'] ) From 74474a6d637359de6913ce6d02d93fdf82450df1 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 18 Jan 2016 16:32:33 +0000 Subject: [PATCH 095/294] Pull out app service user lookup I find this a lot simpler than nested try-catches and stuff --- synapse/api/auth.py | 59 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e36313e2f..cc0296adf 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -510,42 +510,14 @@ class Auth(object): """ # Can optionally look elsewhere in the request (e.g. headers) try: - access_token = request.args["access_token"][0] - - # Check for application service tokens with a user_id override - try: - app_service = yield self.store.get_app_service_by_token( - access_token - ) - if not app_service: - raise KeyError - - user_id = app_service.sender - if "user_id" in request.args: - user_id = request.args["user_id"][0] - if not app_service.is_interested_in_user(user_id): - raise AuthError( - 403, - "Application service cannot masquerade as this user." - ) - if not (yield self.store.get_user_by_id(user_id)): - raise AuthError( - 403, - "Application service has not registered this user" - ) - - if not user_id: - raise KeyError - + user_id = yield self._get_appservice_user_id(request.args) + if user_id: request.authenticated_entity = user_id - defer.returnValue( Requester(UserID.from_string(user_id), "", False) ) - return - except KeyError: - pass # normal users won't have the user_id query parameter set. + access_token = request.args["access_token"][0] user_info = yield self._get_user_by_access_token(access_token) user = user_info["user"] token_id = user_info["token_id"] @@ -578,6 +550,31 @@ class Auth(object): errcode=Codes.MISSING_TOKEN ) + @defer.inlineCallbacks + def _get_appservice_user_id(self, request_args): + app_service = yield self.store.get_app_service_by_token( + request_args["access_token"][0] + ) + if app_service is None: + defer.returnValue(None) + + if "user_id" not in request_args: + defer.returnValue(app_service.sender) + + user_id = request_args["user_id"][0] + + if not app_service.is_interested_in_user(user_id): + raise AuthError( + 403, + "Application service cannot masquerade as this user." + ) + if not (yield self.store.get_user_by_id(user_id)): + raise AuthError( + 403, + "Application service has not registered this user" + ) + defer.returnValue(user_id) + @defer.inlineCallbacks def _get_user_by_access_token(self, token): """ Get a registered user's ID. From 808a8aedab4dbd2166b5935b86edf65501cc24a3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 18 Jan 2016 16:33:05 +0000 Subject: [PATCH 096/294] Don't error on AS non-ghost user use This will probably go away either when we fix our existing ASes, or when we kill the concept of non-ghost users. --- synapse/api/auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index cc0296adf..b5536e856 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -562,6 +562,8 @@ class Auth(object): defer.returnValue(app_service.sender) user_id = request_args["user_id"][0] + if app_service.sender == user_id: + defer.returnValue(app_service.sender) if not app_service.is_interested_in_user(user_id): raise AuthError( From 29c353c5536520cd149e1aacf6bd42c7c3f8f4e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 16:48:17 +0000 Subject: [PATCH 097/294] Don't split at word boundaries, actually use regex --- synapse/push/bulk_push_rule_evaluator.py | 2 +- synapse/push/push_rule_evaluator.py | 105 ++++++++++------------- 2 files changed, 46 insertions(+), 61 deletions(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index f1910f7da..b0b3a38db 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -81,7 +81,7 @@ class BulkPushRuleEvaluator: users_dict.items(), [event] ) - evaluator = PushRuleEvaluatorForEvent.create(event, len(self.users_in_room)) + evaluator = PushRuleEvaluatorForEvent(event, len(self.users_in_room)) condition_cache = {} diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 0816b632b..78d4b564d 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -127,7 +127,7 @@ class PushRuleEvaluator: room_members = yield self.store.get_users_in_room(room_id) room_member_count = len(room_members) - evaluator = PushRuleEvaluatorForEvent.create(ev, room_member_count) + evaluator = PushRuleEvaluatorForEvent(ev, room_member_count) for r in self.rules: if self.enabled_map.get(r['rule_id'], None) is False: @@ -180,33 +180,13 @@ class PushRuleEvaluator: class PushRuleEvaluatorForEvent(object): - WORD_BOUNDARY = re.compile(r'\b') - - def __init__(self, event, body_parts, room_member_count): + def __init__(self, event, room_member_count): self._event = event - - # This is a list of words of the content.body (if event has one). Each - # word has been converted to lower case. - self._body_parts = body_parts - self._room_member_count = room_member_count # Maps strings of e.g. 'content.body' -> event["content"]["body"] self._value_cache = _flatten_dict(event) - @staticmethod - def create(event, room_member_count): - body = event.get("content", {}).get("body", None) - if body: - body_parts = PushRuleEvaluatorForEvent.WORD_BOUNDARY.split(body) - body_parts[:] = [ - part.lower() for part in body_parts - ] - else: - body_parts = [] - - return PushRuleEvaluatorForEvent(event, body_parts, room_member_count) - def matches(self, condition, user_id, display_name, profile_tag): if condition['kind'] == 'event_match': return self._event_match(condition, user_id) @@ -239,67 +219,72 @@ class PushRuleEvaluatorForEvent(object): # XXX: optimisation: cache our pattern regexps if condition['key'] == 'content.body': - matcher = _glob_to_matcher(pattern) + body = self._event["content"].get("body", None) + if not body: + return False - for part in self._body_parts: - if matcher(part): - return True - return False + return _glob_matches(pattern, body, word_boundary=True) else: haystack = self._get_value(condition['key']) if haystack is None: return False - matcher = _glob_to_matcher(pattern) - - return matcher(haystack.lower()) + return _glob_matches(pattern, haystack) def _contains_display_name(self, display_name): if not display_name: return False - lower_display_name = display_name.lower() - for part in self._body_parts: - if part == lower_display_name: - return True + body = self._event["content"].get("body", None) + if not body: + return False - return False + return _glob_matches(display_name, body, word_boundary=True) def _get_value(self, dotted_key): return self._value_cache.get(dotted_key, None) -def _glob_to_matcher(glob): - """Takes a glob and returns a `func(string) -> bool`, which returns if the - string matches the glob. Assumes given string is lower case. +def _glob_matches(glob, value, word_boundary=False): + """Tests if value matches glob. - The matcher returned is either a simple string comparison for globs without - wildcards, or a regex matcher for globs with wildcards. + Args: + glob (string) + value (string): String to test against glob. + word_boundary (bool): Whether to match against word boundaries or entire + string. Defaults to False. + + Returns: + bool """ - glob = glob.lower() + if IS_GLOB.search(glob): + r = re.escape(glob) - if not IS_GLOB.search(glob): - return lambda value: value == glob + r = r.replace(r'\*', '.*?') + r = r.replace(r'\?', '.') - r = re.escape(glob) + # handle [abc], [a-z] and [!a-z] style ranges. + r = GLOB_REGEX.sub( + lambda x: ( + '[%s%s]' % ( + x.group(1) and '^' or '', + x.group(2).replace(r'\\\-', '-') + ) + ), + r, + ) + r = r + "$" + r = re.compile(r, flags=re.IGNORECASE) - r = r.replace(r'\*', '.*?') - r = r.replace(r'\?', '.') + return r.match(value) + elif word_boundary: + r = re.escape(glob) + r = "\b%s\b" % (r,) + r = re.compile(r, flags=re.IGNORECASE) - # handle [abc], [a-z] and [!a-z] style ranges. - r = GLOB_REGEX.sub( - lambda x: ( - '[%s%s]' % ( - x.group(1) and '^' or '', - x.group(2).replace(r'\\\-', '-') - ) - ), - r, - ) - - r = r + "$" - r = re.compile(r) - return lambda value: r.match(value) + return r.search(value) + else: + return value.lower() == glob.lower() def _flatten_dict(d, prefix=[], result={}): From 5cd2126a6a6e23ec1f694cfba7be7bbf29bd1506 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 16:48:37 +0000 Subject: [PATCH 098/294] Remove dead code --- synapse/storage/push_rule.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index f210e6c14..2adfefd99 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cached, cachedInlineCallbacks +from synapse.util.caches.descriptors import cachedInlineCallbacks from twisted.internet import defer import logging @@ -60,27 +60,6 @@ class PushRuleStore(SQLBaseStore): r['rule_id']: False if r['enabled'] == 0 else True for r in results }) - @cached() - def _get_push_rules_enabled_for_user(self, user_id): - def f(txn): - sql = ( - "SELECT pr.*" - " FROM push_rules AS pr" - " LEFT JOIN push_rules_enable AS pre" - " ON pr.user_name = pre.user_name AND pr.rule_id = pre.rule_id" - " WHERE pr.user_name = ?" - " AND (pre.enabled IS NULL OR pre.enabled = 1)" - " ORDER BY pr.priority_class DESC, pr.priority DESC" - ) - txn.execute(sql, (user_id,)) - return self.cursor_to_dict(txn) - - return self.runInteraction( - "_get_push_rules_enabled_for_user", f - ) - - # @cachedList(cache=_get_push_rules_enabled_for_user.cache, list_name="user_ids", - # num_args=1, inlineCallbacks=True) @defer.inlineCallbacks def bulk_get_push_rules(self, user_ids): if not user_ids: From 47f82e4408763d834a0097dceb8b2cca4b0ba4d5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 17:04:36 +0000 Subject: [PATCH 099/294] Fix branch didn't check word_boundary --- synapse/push/push_rule_evaluator.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 78d4b564d..86d8ac5c4 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -273,10 +273,16 @@ def _glob_matches(glob, value, word_boundary=False): ), r, ) - r = r + "$" - r = re.compile(r, flags=re.IGNORECASE) + if word_boundary: + r = "\b%s\b" % (r,) + r = re.compile(r, flags=re.IGNORECASE) - return r.match(value) + return r.search(value) + else: + r = r + "$" + r = re.compile(r, flags=re.IGNORECASE) + + return r.match(value) elif word_boundary: r = re.escape(glob) r = "\b%s\b" % (r,) From a284ad4092871d55dfb213398d5a1994bb666a78 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 17:20:44 +0000 Subject: [PATCH 100/294] You need to escape backslashes --- synapse/push/push_rule_evaluator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 86d8ac5c4..252492213 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -274,7 +274,7 @@ def _glob_matches(glob, value, word_boundary=False): r, ) if word_boundary: - r = "\b%s\b" % (r,) + r = r"\b%s\b" % (r,) r = re.compile(r, flags=re.IGNORECASE) return r.search(value) @@ -285,7 +285,7 @@ def _glob_matches(glob, value, word_boundary=False): return r.match(value) elif word_boundary: r = re.escape(glob) - r = "\b%s\b" % (r,) + r = r"\b%s\b" % (r,) r = re.compile(r, flags=re.IGNORECASE) return r.search(value) From 003853e702a9ce1c58da7d326402a1f2cfa69e1f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 17:34:02 +0000 Subject: [PATCH 101/294] Preserve truthiness --- synapse/push/push_rule_evaluator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 252492213..379652c51 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -130,7 +130,8 @@ class PushRuleEvaluator: evaluator = PushRuleEvaluatorForEvent(ev, room_member_count) for r in self.rules: - if self.enabled_map.get(r['rule_id'], None) is False: + enabled = self.enabled_map.get(r['rule_id'], None) + if enabled is not None and not enabled: continue if not r.get("enabled", True): From f750a442f738ef9bdc2994810cc3220586f8c765 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 10:14:53 +0000 Subject: [PATCH 102/294] Update _id --- synapse/push/baserules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 8be47f097..e1217b5c5 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -171,7 +171,8 @@ BASE_APPEND_UNDERRIDE_RULES = [ 'conditions': [ { 'kind': 'room_member_count', - 'is': '2' + 'is': '2', + '_id': 'member_count', } ], 'actions': [ @@ -192,7 +193,7 @@ BASE_APPEND_UNDERRIDE_RULES = [ 'kind': 'event_match', 'key': 'type', 'pattern': 'm.room.member', - '_id': '_invite_type', + '_id': '_member', }, { 'kind': 'event_match', From 0d241e1114c20e7225d49678f34e9e4e5b3872c9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 10:15:12 +0000 Subject: [PATCH 103/294] Take a deepcopy of push rules before mutating them --- synapse/rest/client/v1/push_rule.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index aa861e703..2272d66dc 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -27,6 +27,7 @@ from synapse.push.rulekinds import ( PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP ) +import copy import simplejson as json @@ -126,7 +127,8 @@ class PushRuleRestServlet(ClientV1RestServlet): rule["actions"] = json.loads(rawrule["actions"]) ruleslist.append(rule) - ruleslist = baserules.list_with_base_rules(ruleslist) + # We're going to be mutating this a lot, so do a deep copy + ruleslist = copy.deepcopy(baserules.list_with_base_rules(ruleslist)) rules = {'global': {}, 'device': {}} From 3adcc4c86aade29f502b7245acc2353326a62256 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 11:35:50 +0000 Subject: [PATCH 104/294] Return highlight_count in /sync --- synapse/handlers/sync.py | 42 +++++++++++++++++++++++---- synapse/rest/client/v2_alpha/sync.py | 1 + synapse/storage/event_push_actions.py | 5 ++-- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 52202d8e6..66e57bd4d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -55,6 +55,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ "ephemeral", "account_data", "unread_notification_count", + "unread_highlight_count", ])): __slots__ = [] @@ -292,9 +293,14 @@ class SyncHandler(BaseHandler): notifs = yield self.unread_notifs_for_room_id( room_id, sync_config, ephemeral_by_room ) + notif_count = None + highlight_count = None if notifs is not None: notif_count = len(notifs) + highlight_count = len([ + 1 for notif in notifs if _action_has_highlight(notif["actions"]) + ]) current_state = yield self.get_state_at(room_id, now_token) @@ -307,6 +313,7 @@ class SyncHandler(BaseHandler): room_id, tags_by_room, account_data_by_room ), unread_notification_count=notif_count, + unread_highlight_count=highlight_count, )) def account_data_for_user(self, account_data): @@ -529,9 +536,14 @@ class SyncHandler(BaseHandler): notifs = yield self.unread_notifs_for_room_id( room_id, sync_config, all_ephemeral_by_room ) + notif_count = None + highlight_count = None if notifs is not None: notif_count = len(notifs) + highlight_count = len([ + 1 for notif in notifs if _action_has_highlight(notif["actions"]) + ]) just_joined = yield self.check_joined_room(sync_config, state) if just_joined: @@ -553,7 +565,8 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), - unread_notification_count=notif_count + unread_notification_count=notif_count, + unread_highlight_count=highlight_count, ) logger.debug("Result for room %s: %r", room_id, room_sync) @@ -692,9 +705,14 @@ class SyncHandler(BaseHandler): notifs = yield self.unread_notifs_for_room_id( room_id, sync_config, ephemeral_by_room ) + notif_count = None + highlight_count = None if notifs is not None: notif_count = len(notifs) + highlight_count = len([ + 1 for notif in notifs if _action_has_highlight(notif["actions"]) + ]) room_sync = JoinedSyncResult( room_id=room_id, @@ -705,6 +723,7 @@ class SyncHandler(BaseHandler): room_id, tags_by_room, account_data_by_room ), unread_notification_count=notif_count, + unread_highlight_count=highlight_count, ) logger.debug("Room sync: %r", room_sync) @@ -850,8 +869,19 @@ class SyncHandler(BaseHandler): notifs = yield self.store.get_unread_event_push_actions_by_room_for_user( room_id, sync_config.user.to_string(), last_unread_event_id ) - else: - # There is no new information in this period, so your notification - # count is whatever it was last time. - defer.returnValue(None) - defer.returnValue(notifs) + defer.returnValue(notifs) + + # There is no new information in this period, so your notification + # count is whatever it was last time. + defer.returnValue(None) + + +def _action_has_highlight(actions): + for action in actions: + try: + if action.get("set_tweak", None) == "highlight": + return action.get("value", True) + except AttributeError: + pass + + return False diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 826f9db18..e300ced21 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -313,6 +313,7 @@ class SyncRestServlet(RestServlet): ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) result["ephemeral"] = {"events": ephemeral_events} result["unread_notification_count"] = room.unread_notification_count + result["unread_highlight_count"] = room.unread_highlight_count return result diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index d99171ee8..6b7cebc9c 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -17,7 +17,7 @@ from ._base import SQLBaseStore from twisted.internet import defer import logging -import simplejson as json +import ujson as json logger = logging.getLogger(__name__) @@ -84,7 +84,8 @@ class EventPushActionsStore(SQLBaseStore): ) ) return [ - {"event_id": row[0], "actions": row[1]} for row in txn.fetchall() + {"event_id": row[0], "actions": json.loads(row[1])} + for row in txn.fetchall() ] ret = yield self.runInteraction( From 7ecd211163dd4eb624f2ad79aa11b2e6cb307274 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 14:22:02 +0000 Subject: [PATCH 105/294] Except truthy values --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index b0b3a38db..c3a3b1873 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -131,7 +131,7 @@ def _condition_checker(evaluator, conditions, uid, display_name, cache): if _id: cache[_id] = res - if res is False: + if not res: return False return True From 9654ee084867256ab77062761bc4e2c63254e275 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 14:24:59 +0000 Subject: [PATCH 106/294] Return don't break --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index c3a3b1873..efd686fa6 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -123,7 +123,7 @@ def _condition_checker(evaluator, conditions, uid, display_name, cache): if _id: res = cache.get(_id, None) if res is False: - break + return False elif res is True: continue From b8518ffe654d2ee40b0f51983fbb425bbf03e544 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 14:26:58 +0000 Subject: [PATCH 107/294] Use all_ephemeral_by_room in incremental_sync_with_gap_for_room --- synapse/handlers/sync.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 66e57bd4d..d2864977b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -588,7 +588,8 @@ class SyncHandler(BaseHandler): for room_id in joined_room_ids: room_sync = yield self.incremental_sync_with_gap_for_room( room_id, sync_config, since_token, now_token, - ephemeral_by_room, tags_by_room, account_data_by_room + ephemeral_by_room, tags_by_room, account_data_by_room, + all_ephemeral_by_room=all_ephemeral_by_room, ) if room_sync: joined.append(room_sync) @@ -668,7 +669,8 @@ class SyncHandler(BaseHandler): def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, ephemeral_by_room, tags_by_room, - account_data_by_room): + account_data_by_room, + all_ephemeral_by_room): """ Get the incremental delta needed to bring the client up to date for the room. Gives the client the most recent events and the changes to state. @@ -703,7 +705,7 @@ class SyncHandler(BaseHandler): state = yield self.get_state_at(room_id, now_token) notifs = yield self.unread_notifs_for_room_id( - room_id, sync_config, ephemeral_by_room + room_id, sync_config, all_ephemeral_by_room ) notif_count = None From d056a0a3d83933133f25f082b938d4115c803145 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 14:43:24 +0000 Subject: [PATCH 108/294] Handle glob -> regex errors --- synapse/push/push_rule_evaluator.py | 58 +++++++++++++++-------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 379652c51..4654994d2 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -258,40 +258,44 @@ def _glob_matches(glob, value, word_boundary=False): Returns: bool """ - if IS_GLOB.search(glob): - r = re.escape(glob) + try: + if IS_GLOB.search(glob): + r = re.escape(glob) - r = r.replace(r'\*', '.*?') - r = r.replace(r'\?', '.') + r = r.replace(r'\*', '.*?') + r = r.replace(r'\?', '.') - # handle [abc], [a-z] and [!a-z] style ranges. - r = GLOB_REGEX.sub( - lambda x: ( - '[%s%s]' % ( - x.group(1) and '^' or '', - x.group(2).replace(r'\\\-', '-') - ) - ), - r, - ) - if word_boundary: + # handle [abc], [a-z] and [!a-z] style ranges. + r = GLOB_REGEX.sub( + lambda x: ( + '[%s%s]' % ( + x.group(1) and '^' or '', + x.group(2).replace(r'\\\-', '-') + ) + ), + r, + ) + if word_boundary: + r = r"\b%s\b" % (r,) + r = re.compile(r, flags=re.IGNORECASE) + + return r.search(value) + else: + r = r + "$" + r = re.compile(r, flags=re.IGNORECASE) + + return r.match(value) + elif word_boundary: + r = re.escape(glob) r = r"\b%s\b" % (r,) r = re.compile(r, flags=re.IGNORECASE) return r.search(value) else: - r = r + "$" - r = re.compile(r, flags=re.IGNORECASE) - - return r.match(value) - elif word_boundary: - r = re.escape(glob) - r = r"\b%s\b" % (r,) - r = re.compile(r, flags=re.IGNORECASE) - - return r.search(value) - else: - return value.lower() == glob.lower() + return value.lower() == glob.lower() + except re.error: + logger.warn("Failed to parse glob to regex: %r", glob) + return False def _flatten_dict(d, prefix=[], result={}): From 5a7d1ecffcab7a94caf70471a2eec56eb868573c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 16:01:05 +0000 Subject: [PATCH 109/294] Add regex cache. Only caculate push actions for users that have sent read receipts, and are on that server --- synapse/handlers/_base.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/push/action_generator.py | 7 ++++--- synapse/push/bulk_push_rule_evaluator.py | 15 ++++++++++----- synapse/push/push_rule_evaluator.py | 20 +++++++++++++++++--- synapse/server.py | 4 ++++ synapse/storage/receipts.py | 14 +++++++++++++- 7 files changed, 50 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 2d1167296..5c7617de4 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -266,7 +266,7 @@ class BaseHandler(object): event, context=context ) - action_generator = ActionGenerator(self.store) + action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( event, self ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4b94940e9..6c19d6ae8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -245,7 +245,7 @@ class FederationHandler(BaseHandler): yield user_joined_room(self.distributor, user, event.room_id) if not backfilled and not event.internal_metadata.is_outlier(): - action_generator = ActionGenerator(self.store) + action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( event, self ) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 4cf94f6c6..1d2e558f9 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -25,8 +25,9 @@ logger = logging.getLogger(__name__) class ActionGenerator: - def __init__(self, store): - self.store = store + def __init__(self, hs): + self.hs = hs + self.store = hs.get_datastore() # really we want to get all user ids and all profile tags too, # since we want the actions for each profile tag for every user and # also actions for a client with no profile tag for each user. @@ -42,7 +43,7 @@ class ActionGenerator: ) bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( - event.room_id, self.store + event.room_id, self.hs, self.store ) actions_by_user = yield bulk_evaluator.action_for_event_by_user(event, handler) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index efd686fa6..1000ae630 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -36,6 +36,7 @@ def decode_rule_json(rule): @defer.inlineCallbacks def _get_rules(room_id, user_ids, store): rules_by_user = yield store.bulk_get_push_rules(user_ids) + rules_by_user = { uid: baserules.list_with_base_rules([ decode_rule_json(rule_list) @@ -47,12 +48,16 @@ def _get_rules(room_id, user_ids, store): @defer.inlineCallbacks -def evaluator_for_room_id(room_id, store): - users = yield store.get_users_in_room(room_id) - rules_by_user = yield _get_rules(room_id, users, store) +def evaluator_for_room_id(room_id, hs, store): + results = yield store.get_receipts_for_room(room_id, "m.read") + user_ids = [ + row["user_id"] for row in results + if hs.is_mine_id(row["user_id"]) + ] + rules_by_user = yield _get_rules(room_id, user_ids, store) defer.returnValue(BulkPushRuleEvaluator( - room_id, rules_by_user, users, store + room_id, rules_by_user, user_ids, store )) @@ -129,7 +134,7 @@ def _condition_checker(evaluator, conditions, uid, display_name, cache): res = evaluator.matches(cond, uid, display_name, None) if _id: - cache[_id] = res + cache[_id] = bool(res) if not res: return False diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 4654994d2..753b6469e 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -22,6 +22,7 @@ import simplejson as json import re from synapse.types import UserID +from synapse.util.caches.lrucache import LruCache logger = logging.getLogger(__name__) @@ -277,18 +278,18 @@ def _glob_matches(glob, value, word_boundary=False): ) if word_boundary: r = r"\b%s\b" % (r,) - r = re.compile(r, flags=re.IGNORECASE) + r = _compile_regex(r) return r.search(value) else: r = r + "$" - r = re.compile(r, flags=re.IGNORECASE) + r = _compile_regex(r) return r.match(value) elif word_boundary: r = re.escape(glob) r = r"\b%s\b" % (r,) - r = re.compile(r, flags=re.IGNORECASE) + r = _compile_regex(r) return r.search(value) else: @@ -306,3 +307,16 @@ def _flatten_dict(d, prefix=[], result={}): _flatten_dict(value, prefix=(prefix+[key]), result=result) return result + + +regex_cache = LruCache(100000) + + +def _compile_regex(regex_str): + r = regex_cache.get(regex_str, None) + if r: + return r + + r = re.compile(regex_str, flags=re.IGNORECASE) + regex_cache[regex_str] = r + return r diff --git a/synapse/server.py b/synapse/server.py index ffd4f936d..63f905983 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -96,6 +96,7 @@ class BaseHomeServer(object): hostname : The hostname for the server. """ self.hostname = hostname + self.hostname_with_colon = ":" + hostname self._building = {} # Other kwargs are explicit dependencies @@ -139,6 +140,9 @@ class BaseHomeServer(object): def is_mine(self, domain_specific_string): return domain_specific_string.domain == self.hostname + def is_mine_id(self, string): + return string.endswith(self.hostname_with_colon) + # Build magic accessors for every dependency for depname in BaseHomeServer.DEPENDENCIES: BaseHomeServer._make_dependency_method(depname) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 21cf88b3d..c80e57662 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList +from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList, cached from synapse.util.caches import cache_counter, caches_by_name from twisted.internet import defer @@ -33,6 +33,18 @@ class ReceiptsStore(SQLBaseStore): self._receipts_stream_cache = _RoomStreamChangeCache() + @cached(num_args=2) + def get_receipts_for_room(self, room_id, receipt_type): + return self._simple_select_list( + table="receipts_linearized", + keyvalues={ + "room_id": room_id, + "receipt_type": receipt_type, + }, + retcols=("user_id", "event_id"), + desc="get_receipts_for_room", + ) + @defer.inlineCallbacks def get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None): """Get receipts for multiple rooms for sending to clients. From fb5d8e58ff280c9fc24123adca3254e46ac63097 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 16:07:07 +0000 Subject: [PATCH 110/294] Change regex cache size to 5000 --- synapse/push/push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 753b6469e..dca018af9 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -309,7 +309,7 @@ def _flatten_dict(d, prefix=[], result={}): return result -regex_cache = LruCache(100000) +regex_cache = LruCache(5000) def _compile_regex(regex_str): From 2818a000aa5ca7968f196898908f31a732387791 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 16:11:39 +0000 Subject: [PATCH 111/294] Use split rather than endswith --- synapse/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index 63f905983..4a5796b98 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -96,7 +96,6 @@ class BaseHomeServer(object): hostname : The hostname for the server. """ self.hostname = hostname - self.hostname_with_colon = ":" + hostname self._building = {} # Other kwargs are explicit dependencies @@ -141,7 +140,7 @@ class BaseHomeServer(object): return domain_specific_string.domain == self.hostname def is_mine_id(self, string): - return string.endswith(self.hostname_with_colon) + return string.split(":", 1)[1] == self.hostname # Build magic accessors for every dependency for depname in BaseHomeServer.DEPENDENCIES: From 892ee473d9bda85ba1bb505272f1177664275674 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 17:12:46 +0000 Subject: [PATCH 112/294] Don't use form of get_state_for_events with None state_key --- synapse/handlers/_base.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 5c7617de4..46abb8ec5 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -53,16 +53,23 @@ class BaseHandler(object): self.event_builder_factory = hs.get_event_builder_factory() @defer.inlineCallbacks - def _filter_events_for_clients(self, users, events): + def _filter_events_for_clients(self, user_tuples, events): """ Returns dict of user_id -> list of events that user is allowed to see. """ + # If there is only one user, just get the state for that one user, + # otherwise just get all the state. + if len(user_tuples) == 1: + types = ( + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, user_tuples[0][0]), + ) + else: + types = None + event_id_to_state = yield self.store.get_state_for_events( frozenset(e.event_id for e in events), - types=( - (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.Member, None), - ) + types=types ) forgotten = yield defer.gatherResults([ @@ -122,7 +129,7 @@ class BaseHandler(object): for event in events if allowed(event, user_id, is_guest) ] - for user_id, is_guest in users + for user_id, is_guest in user_tuples }) @defer.inlineCallbacks From ac2842ff1e1c53da80cc0743e31647c5a0152124 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 17:19:53 +0000 Subject: [PATCH 113/294] Only compute unread notifications for rooms we send down stream --- synapse/handlers/sync.py | 50 ++++++++++++---------------- synapse/rest/client/v2_alpha/sync.py | 3 +- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index d2864977b..aca200c1e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -54,8 +54,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ "state", # dict[(str, str), FrozenEvent] "ephemeral", "account_data", - "unread_notification_count", - "unread_highlight_count", + "unread_notifications", ])): __slots__ = [] @@ -294,11 +293,10 @@ class SyncHandler(BaseHandler): room_id, sync_config, ephemeral_by_room ) - notif_count = None - highlight_count = None + unread_notifications = {} if notifs is not None: - notif_count = len(notifs) - highlight_count = len([ + unread_notifications["notification_count"] = len(notifs) + unread_notifications["highlight_count"] = len([ 1 for notif in notifs if _action_has_highlight(notif["actions"]) ]) @@ -312,8 +310,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), - unread_notification_count=notif_count, - unread_highlight_count=highlight_count, + unread_notifications=unread_notifications, )) def account_data_for_user(self, account_data): @@ -533,18 +530,6 @@ class SyncHandler(BaseHandler): else: prev_batch = now_token - notifs = yield self.unread_notifs_for_room_id( - room_id, sync_config, all_ephemeral_by_room - ) - - notif_count = None - highlight_count = None - if notifs is not None: - notif_count = len(notifs) - highlight_count = len([ - 1 for notif in notifs if _action_has_highlight(notif["actions"]) - ]) - just_joined = yield self.check_joined_room(sync_config, state) if just_joined: logger.debug("User has just joined %s: needs full state", @@ -565,12 +550,23 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), - unread_notification_count=notif_count, - unread_highlight_count=highlight_count, + unread_notifications={}, ) logger.debug("Result for room %s: %r", room_id, room_sync) if room_sync: + notifs = yield self.unread_notifs_for_room_id( + room_id, sync_config, all_ephemeral_by_room + ) + + if notifs is not None: + notif_dict = room_sync.unread_notifications + notif_dict["notification_count"] = len(notifs) + notif_dict["highlight_count"] = len([ + 1 for notif in notifs + if _action_has_highlight(notif["actions"]) + ]) + joined.append(room_sync) else: @@ -708,11 +704,10 @@ class SyncHandler(BaseHandler): room_id, sync_config, all_ephemeral_by_room ) - notif_count = None - highlight_count = None + unread_notifications = {} if notifs is not None: - notif_count = len(notifs) - highlight_count = len([ + unread_notifications["notification_count"] = len(notifs) + unread_notifications["highlight_count"] = len([ 1 for notif in notifs if _action_has_highlight(notif["actions"]) ]) @@ -724,8 +719,7 @@ class SyncHandler(BaseHandler): account_data=self.account_data_for_room( room_id, tags_by_room, account_data_by_room ), - unread_notification_count=notif_count, - unread_highlight_count=highlight_count, + unread_notifications=unread_notifications, ) logger.debug("Room sync: %r", room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index e300ced21..df4b305b4 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -312,8 +312,7 @@ class SyncRestServlet(RestServlet): if joined: ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) result["ephemeral"] = {"events": ephemeral_events} - result["unread_notification_count"] = room.unread_notification_count - result["unread_highlight_count"] = room.unread_highlight_count + result["unread_notifications"] = room.unread_notifications return result From 191070123da7f472bca99c0a89d27fbdca51f972 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Jan 2016 11:34:09 +0000 Subject: [PATCH 114/294] Cache dns lookups, and use the cache if we fail to lookup servers later --- synapse/http/endpoint.py | 101 ++++++++++++++++++++++++---------- tests/test_dns.py | 115 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 30 deletions(-) create mode 100644 tests/test_dns.py diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 4341ded96..a9e024a41 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -17,7 +17,7 @@ from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint from twisted.internet import defer from twisted.internet.error import ConnectError from twisted.names import client, dns -from twisted.names.error import DNSNameError +from twisted.names.error import DNSNameError, DomainError import collections import logging @@ -27,6 +27,14 @@ import random logger = logging.getLogger(__name__) +SERVER_CACHE = {} + + +_Server = collections.namedtuple( + "_Server", "priority weight host port" +) + + def matrix_federation_endpoint(reactor, destination, ssl_context_factory=None, timeout=None): """Construct an endpoint for the given matrix destination. @@ -73,10 +81,6 @@ class SRVClientEndpoint(object): Implements twisted.internet.interfaces.IStreamClientEndpoint. """ - _Server = collections.namedtuple( - "_Server", "priority weight host port" - ) - def __init__(self, reactor, service, domain, protocol="tcp", default_port=None, endpoint=TCP4ClientEndpoint, endpoint_kw_args={}): @@ -101,32 +105,8 @@ class SRVClientEndpoint(object): @defer.inlineCallbacks def fetch_servers(self): - try: - answers, auth, add = yield client.lookupService(self.service_name) - except DNSNameError: - answers = [] - - if (len(answers) == 1 - and answers[0].type == dns.SRV - and answers[0].payload - and answers[0].payload.target == dns.Name('.')): - raise ConnectError("Service %s unavailable", self.service_name) - - self.servers = [] self.used_servers = [] - - for answer in answers: - if answer.type != dns.SRV or not answer.payload: - continue - payload = answer.payload - self.servers.append(self._Server( - host=str(payload.target), - port=int(payload.port), - priority=int(payload.priority), - weight=int(payload.weight) - )) - - self.servers.sort() + self.servers = yield resolve_service(self.service_name) def pick_server(self): if not self.servers: @@ -170,3 +150,64 @@ class SRVClientEndpoint(object): ) connection = yield endpoint.connect(protocolFactory) defer.returnValue(connection) + + +@defer.inlineCallbacks +def resolve_service(service_name, dns_client=client, cache=SERVER_CACHE): + servers = [] + + try: + try: + answers, _, _ = yield dns_client.lookupService(service_name) + except DNSNameError: + defer.returnValue([]) + + if (len(answers) == 1 + and answers[0].type == dns.SRV + and answers[0].payload + and answers[0].payload.target == dns.Name('.')): + raise ConnectError("Service %s unavailable", service_name) + + for answer in answers: + if answer.type != dns.SRV or not answer.payload: + continue + + payload = answer.payload + + host = str(payload.target) + + try: + answers, _, _ = yield dns_client.lookupAddress(host) + except DNSNameError: + continue + + ips = [ + answer.payload.dottedQuad() + for answer in answers + if answer.type == dns.A and answer.payload + ] + + for ip in ips: + servers.append(_Server( + host=ip, + port=int(payload.port), + priority=int(payload.priority), + weight=int(payload.weight) + )) + + servers.sort() + cache[service_name] = list(servers) + except DomainError as e: + # We failed to resolve the name (other than a NameError) + # Try something in the cache, else rereaise + cache_entry = cache.get(service_name, None) + if cache_entry: + logger.warn( + "Failed to resolve %r, falling back to cache. %r", + service_name, e + ) + servers = list(cache_entry) + else: + raise e + + defer.returnValue(servers) diff --git a/tests/test_dns.py b/tests/test_dns.py new file mode 100644 index 000000000..637b1606f --- /dev/null +++ b/tests/test_dns.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Copyright 2014-2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import unittest +from twisted.internet import defer +from twisted.names import dns, error + +from mock import Mock + +from synapse.http.endpoint import resolve_service + + +class DnsTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def test_resolve(self): + dns_client_mock = Mock() + + service_name = "test_service.examle.com" + host_name = "example.com" + ip_address = "127.0.0.1" + + answer_srv = dns.RRHeader( + type=dns.SRV, + payload=dns.Record_SRV( + target=host_name, + ) + ) + + answer_a = dns.RRHeader( + type=dns.A, + payload=dns.Record_A( + address=ip_address, + ) + ) + + dns_client_mock.lookupService.return_value = ([answer_srv], None, None) + dns_client_mock.lookupAddress.return_value = ([answer_a], None, None) + + cache = {} + + servers = yield resolve_service( + service_name, dns_client=dns_client_mock, cache=cache + ) + + dns_client_mock.lookupService.assert_called_once_with(service_name) + dns_client_mock.lookupAddress.assert_called_once_with(host_name) + + self.assertEquals(len(servers), 1) + self.assertEquals(servers, cache[service_name]) + self.assertEquals(servers[0].host, ip_address) + + @defer.inlineCallbacks + def test_from_cache(self): + dns_client_mock = Mock() + dns_client_mock.lookupService.return_value = defer.fail(error.DNSServerError()) + + service_name = "test_service.examle.com" + + cache = { + service_name: [object()] + } + + servers = yield resolve_service( + service_name, dns_client=dns_client_mock, cache=cache + ) + + dns_client_mock.lookupService.assert_called_once_with(service_name) + + self.assertEquals(len(servers), 1) + self.assertEquals(servers, cache[service_name]) + + @defer.inlineCallbacks + def test_empty_cache(self): + dns_client_mock = Mock() + + dns_client_mock.lookupService.return_value = defer.fail(error.DNSServerError()) + + service_name = "test_service.examle.com" + + cache = {} + + with self.assertRaises(error.DNSServerError): + yield resolve_service( + service_name, dns_client=dns_client_mock, cache=cache + ) + + @defer.inlineCallbacks + def test_name_error(self): + dns_client_mock = Mock() + + dns_client_mock.lookupService.return_value = defer.fail(error.DNSNameError()) + + service_name = "test_service.examle.com" + + cache = {} + + servers = yield resolve_service( + service_name, dns_client=dns_client_mock, cache=cache + ) + + self.assertEquals(len(servers), 0) + self.assertEquals(len(cache), 0) From 259d1ecd1dc721b8dcda01194cc0d5a0c12a591e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Jan 2016 13:24:59 +0000 Subject: [PATCH 115/294] Don't generate push actions for our own events --- synapse/push/bulk_push_rule_evaluator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 1000ae630..f27ae62db 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -107,6 +107,9 @@ class BulkPushRuleEvaluator: if len(filtered) == 0: continue + if filtered.sender == uid: + continue + for rule in rules: if 'enabled' in rule and not rule['enabled']: continue From 746f6e0eb3adf087363e4ee43315c584edd12175 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Jan 2016 13:44:04 +0000 Subject: [PATCH 116/294] 'filtered' is a list of zero or 1 --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index f27ae62db..b91c165e2 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -107,7 +107,7 @@ class BulkPushRuleEvaluator: if len(filtered) == 0: continue - if filtered.sender == uid: + if filtered[0].sender == uid: continue for rule in rules: From d7265977376eb391007cde55c4b2d9b8f54d452b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Jan 2016 13:49:00 +0000 Subject: [PATCH 117/294] Simplify badge updating code by just updating it every time we get woken up and it's not an event --- synapse/push/__init__.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 46c453b6c..438341b51 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -126,6 +126,9 @@ class Pusher(object): from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') timeout = (300 + random.randint(-60, 60)) * 1000 + # note that we need to get read receipts down the stream as we need to + # wake up when one arrives. we don't need to explicitly look for + # them though. chunk = yield self.evStreamHandler.get_stream( self.user_id, config, timeout=timeout, affect_presence=False ) @@ -133,23 +136,12 @@ class Pusher(object): # limiting to 1 may get 1 event plus 1 presence event, so # pick out the actual event single_event = None - read_receipt = None for c in chunk['chunk']: if 'event_id' in c: # Hmmm... single_event = c - elif c['type'] == 'm.receipt': - read_receipt = c - - have_updated_badge = False - if read_receipt: - for receipt_part in read_receipt['content'].values(): - if 'm.read' in receipt_part: - if self.user_id in receipt_part['m.read'].keys(): - have_updated_badge = True if not single_event: - if have_updated_badge: - yield self.update_badge() + yield self.update_badge() self.last_token = chunk['end'] yield self.store.update_pusher_last_token( self.app_id, @@ -194,9 +186,6 @@ class Pusher(object): yield self.hs.get_pusherpool().remove_pusher( self.app_id, pk, self.user_id ) - else: - if have_updated_badge: - yield self.update_badge() processed = True if not self.alive: From 7cc047455e3f0893569853edbafa19bfe2b5d4ed Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Jan 2016 13:50:28 +0000 Subject: [PATCH 118/294] Inline membership specifier --- synapse/push/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 438341b51..ffae02a28 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -285,11 +285,9 @@ class Pusher(object): @defer.inlineCallbacks def _get_badge_count(self): - membership_list = (Membership.INVITE, Membership.JOIN) - room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=self.user_id, - membership_list=membership_list + membership_list=(Membership.INVITE, Membership.JOIN) ) user_is_guest = yield self.store.is_guest(self.user_id) From 3fa344c0376045d1be396701978ba46c552065b6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Jan 2016 15:30:31 +0000 Subject: [PATCH 119/294] Add storage function to get all receipts for a user. Also add some cache invalidation to the receipts storage because there wasn't any, and remove a method that was unused. --- synapse/push/__init__.py | 28 ++++--------------- synapse/storage/receipts.py | 55 +++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index ffae02a28..9a4af2b3c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -16,9 +16,8 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig -from synapse.types import StreamToken, UserID +from synapse.types import StreamToken from synapse.api.constants import Membership -from synapse.api.filtering import FilterCollection import synapse.util.async import push_rule_evaluator as push_rule_evaluator @@ -290,22 +289,9 @@ class Pusher(object): membership_list=(Membership.INVITE, Membership.JOIN) ) - user_is_guest = yield self.store.is_guest(self.user_id) - - # XXX: importing inside method to break circular dependency. - # should sort out the mess by moving all this logic out of - # push/__init__.py and probably moving the logic we use from the sync - # handler to somewhere more amenable to re-use. - from synapse.handlers.sync import SyncConfig - sync_config = SyncConfig( - user=UserID.from_string(self.user_id), - filter=FilterCollection({}), - is_guest=user_is_guest, - ) - now_token = yield self.hs.get_event_sources().get_current_token() - sync_handler = self.hs.get_handlers().sync_handler - _, ephemeral_by_room = yield sync_handler.ephemeral_by_room( - sync_config, now_token + my_receipts_by_room = yield self.store.get_receipts_for_user( + self.user_id, + "m.read", ) badge = 0 @@ -314,11 +300,9 @@ class Pusher(object): if r.membership == Membership.INVITE: badge += 1 else: - last_unread_event_id = sync_handler.last_read_event_id_for_room_and_user( - r.room_id, self.user_id, ephemeral_by_room - ) + if r.room_id in my_receipts_by_room: + last_unread_event_id = my_receipts_by_room[r.room_id] - if last_unread_event_id: notifs = yield ( self.store.get_unread_event_push_actions_by_room_for_user( r.room_id, self.user_id, last_unread_event_id diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index c80e57662..018140f47 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -45,6 +45,21 @@ class ReceiptsStore(SQLBaseStore): desc="get_receipts_for_room", ) + @cachedInlineCallbacks(num_args=2) + def get_receipts_for_user(self, user_id, receipt_type): + def f(txn): + sql = ( + "SELECT room_id,event_id " + "FROM receipts_linearized " + "WHERE user_id = ? AND receipt_type = ? " + ) + txn.execute(sql, (user_id, receipt_type)) + return txn.fetchall() + + defer.returnValue(dict( + (yield self.runInteraction("get_receipts_for_user", f)) + )) + @defer.inlineCallbacks def get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None): """Get receipts for multiple rooms for sending to clients. @@ -194,29 +209,16 @@ class ReceiptsStore(SQLBaseStore): def get_max_receipt_stream_id(self): return self._receipts_id_gen.get_max_token(self) - @cachedInlineCallbacks() - def get_graph_receipts_for_room(self, room_id): - """Get receipts for sending to remote servers. - """ - rows = yield self._simple_select_list( - table="receipts_graph", - keyvalues={"room_id": room_id}, - retcols=["receipt_type", "user_id", "event_id"], - desc="get_linearized_receipts_for_room", - ) - - result = {} - for row in rows: - result.setdefault( - row["user_id"], {} - ).setdefault( - row["receipt_type"], [] - ).append(row["event_id"]) - - defer.returnValue(result) - def insert_linearized_receipt_txn(self, txn, room_id, receipt_type, user_id, event_id, data, stream_id): + txn.call_after( + self.get_receipts_for_room.invalidate, (room_id, receipt_type) + ) + txn.call_after( + self.get_receipts_for_user.invalidate, (user_id, receipt_type) + ) + # FIXME: This shouldn't invalidate the whole cache + txn.call_after(self.get_linearized_receipts_for_room.invalidate_all) # We don't want to clobber receipts for more recent events, so we # have to compare orderings of existing receipts @@ -324,6 +326,7 @@ class ReceiptsStore(SQLBaseStore): ) max_persisted_id = yield self._stream_id_gen.get_max_token(self) + defer.returnValue((stream_id, max_persisted_id)) def insert_graph_receipt(self, room_id, receipt_type, user_id, event_ids, @@ -336,6 +339,16 @@ class ReceiptsStore(SQLBaseStore): def insert_graph_receipt_txn(self, txn, room_id, receipt_type, user_id, event_ids, data): + txn.call_after( + self.get_receipts_for_room.invalidate, (room_id, receipt_type) + ) + txn.call_after( + self.get_receipts_for_user.invalidate, (user_id, receipt_type) + ) + # FIXME: This shouldn't invalidate the whole cache + txn.call_after(self.get_linearized_receipts_for_room.invalidate_all) + + self._simple_delete_txn( txn, table="receipts_graph", From d4315bbf6bf2f7ab979761df10ad1d32d07a89fa Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Jan 2016 15:33:27 +0000 Subject: [PATCH 120/294] Add index by user id on receipts_linearized --- .../schema/delta/28/receipts_user_id_index.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 synapse/storage/schema/delta/28/receipts_user_id_index.sql diff --git a/synapse/storage/schema/delta/28/receipts_user_id_index.sql b/synapse/storage/schema/delta/28/receipts_user_id_index.sql new file mode 100644 index 000000000..452a1b3c6 --- /dev/null +++ b/synapse/storage/schema/delta/28/receipts_user_id_index.sql @@ -0,0 +1,18 @@ +/* Copyright 2015, 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE INDEX receipts_linearized_user ON receipts_linearized( + user_id +); From da417aa56d3582ab91805b1a623c18691995a3e6 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 20 Jan 2016 15:34:07 +0000 Subject: [PATCH 121/294] Allow non-guests to peek on rooms using /events --- synapse/handlers/_base.py | 14 ++--- synapse/handlers/events.py | 2 +- synapse/handlers/message.py | 91 ++++++++++++++++---------------- synapse/handlers/room.py | 2 +- synapse/handlers/sync.py | 4 +- synapse/notifier.py | 71 +++++++++++++++++-------- synapse/rest/client/v1/events.py | 1 + synapse/rest/client/v1/room.py | 6 +-- 8 files changed, 107 insertions(+), 84 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 46abb8ec5..744a9ee50 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -84,7 +84,7 @@ class BaseHandler(object): row["event_id"] for rows in forgotten for row in rows ) - def allowed(event, user_id, is_guest): + def allowed(event, user_id, is_peeking): state = event_id_to_state[event.event_id] visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None) @@ -96,7 +96,7 @@ class BaseHandler(object): if visibility == "world_readable": return True - if is_guest: + if is_peeking: return False membership_event = state.get((EventTypes.Member, user_id), None) @@ -112,7 +112,7 @@ class BaseHandler(object): return True if event.type == EventTypes.RoomHistoryVisibility: - return not is_guest + return not is_peeking if visibility == "shared": return True @@ -127,15 +127,15 @@ class BaseHandler(object): user_id: [ event for event in events - if allowed(event, user_id, is_guest) + if allowed(event, user_id, is_peeking) ] - for user_id, is_guest in user_tuples + for user_id, is_peeking in user_tuples }) @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, events, is_guest=False): + def _filter_events_for_client(self, user_id, events, is_peeking=False): # Assumes that user has at some point joined the room if not is_guest. - res = yield self._filter_events_for_clients([(user_id, is_guest)], events) + res = yield self._filter_events_for_clients([(user_id, is_peeking)], events) defer.returnValue(res.get(user_id, [])) def ratelimit(self, user_id): diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index c73eec2b9..aca4b6754 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -135,7 +135,7 @@ class EventStreamHandler(BaseHandler): events, tokens = yield self.notifier.get_events_for( auth_user, pagin_config, timeout, only_room_events=only_room_events, - is_guest=is_guest, guest_room_id=room_id + is_guest=is_guest, explicit_room_id=room_id ) time_now = self.clock.time_msec() diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4c7bf2bef..ff800f8af 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -78,21 +78,20 @@ class MessageHandler(BaseHandler): defer.returnValue(None) @defer.inlineCallbacks - def get_messages(self, user_id=None, room_id=None, pagin_config=None, - as_client_event=True, is_guest=False): + def get_messages(self, requester, room_id=None, pagin_config=None, + as_client_event=True): """Get messages in a room. Args: - user_id (str): The user requesting messages. + requester (Requester): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. as_client_event (bool): True to get events in client-server format. - is_guest (bool): Whether the requesting user is a guest (as opposed - to a fully registered user). Returns: dict: Pagination API results """ + user_id = requester.user.to_string() data_source = self.hs.get_event_sources().sources["room"] if pagin_config.from_token: @@ -115,36 +114,33 @@ class MessageHandler(BaseHandler): source_config = pagin_config.get_source_config("room") - if not is_guest: - member_event = yield self.auth.check_user_was_in_room(room_id, user_id) - if member_event.membership == Membership.LEAVE: - # If they have left the room then clamp the token to be before - # they left the room. - # If they're a guest, we'll just 403 them if they're asking for - # events they can't see. - leave_token = yield self.store.get_topological_token_for_event( - member_event.event_id - ) - leave_token = RoomStreamToken.parse(leave_token) - if leave_token.topological < room_token.topological: - source_config.from_key = str(leave_token) + membership, member_event_id = yield self._check_in_room_or_world_readable( + room_id, user_id + ) + if membership == Membership.LEAVE: + # If they have left the room then clamp the token to be before + # they left the room. + leave_token = yield self.store.get_topological_token_for_event( + member_event_id + ) + leave_token = RoomStreamToken.parse(leave_token) + if leave_token.topological < room_token.topological: + source_config.from_key = str(leave_token) - if source_config.direction == "f": - if source_config.to_key is None: + if source_config.direction == "f": + if source_config.to_key is None: + source_config.to_key = str(leave_token) + else: + to_token = RoomStreamToken.parse(source_config.to_key) + if leave_token.topological < to_token.topological: source_config.to_key = str(leave_token) - else: - to_token = RoomStreamToken.parse(source_config.to_key) - if leave_token.topological < to_token.topological: - source_config.to_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological ) - user = UserID.from_string(user_id) - events, next_key = yield data_source.get_pagination_rows( - user, source_config, room_id + requester.user, source_config, room_id ) next_token = pagin_config.from_token.copy_and_replace( @@ -158,7 +154,11 @@ class MessageHandler(BaseHandler): "end": next_token.to_string(), }) - events = yield self._filter_events_for_client(user_id, events, is_guest=is_guest) + events = yield self._filter_events_for_client( + user_id, + events, + is_peeking=(member_event_id is None), + ) time_now = self.clock.time_msec() @@ -289,7 +289,7 @@ class MessageHandler(BaseHandler): SynapseError if something went wrong. """ membership, membership_event_id = yield self._check_in_room_or_world_readable( - room_id, user_id, is_guest + room_id, user_id ) if membership == Membership.JOIN: @@ -306,7 +306,7 @@ class MessageHandler(BaseHandler): defer.returnValue(data) @defer.inlineCallbacks - def _check_in_room_or_world_readable(self, room_id, user_id, is_guest): + def _check_in_room_or_world_readable(self, room_id, user_id): try: # check_user_was_in_room will return the most recent membership # event for the user if: @@ -316,7 +316,7 @@ class MessageHandler(BaseHandler): member_event = yield self.auth.check_user_was_in_room(room_id, user_id) defer.returnValue((member_event.membership, member_event.event_id)) return - except AuthError, auth_error: + except AuthError: visibility = yield self.state_handler.get_current_state( room_id, EventTypes.RoomHistoryVisibility, "" ) @@ -326,8 +326,6 @@ class MessageHandler(BaseHandler): ): defer.returnValue((Membership.JOIN, None)) return - if not is_guest: - raise auth_error raise AuthError( 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN ) @@ -345,7 +343,7 @@ class MessageHandler(BaseHandler): A list of dicts representing state events. [{}, {}, {}] """ membership, membership_event_id = yield self._check_in_room_or_world_readable( - room_id, user_id, is_guest + room_id, user_id ) if membership == Membership.JOIN: @@ -556,13 +554,13 @@ class MessageHandler(BaseHandler): defer.returnValue(ret) @defer.inlineCallbacks - def room_initial_sync(self, user_id, room_id, pagin_config=None, is_guest=False): + def room_initial_sync(self, requester, room_id, pagin_config=None): """Capture the a snapshot of a room. If user is currently a member of the room this will be what is currently in the room. If the user left the room this will be what was in the room when they left. Args: - user_id(str): The user to get a snapshot for. + requester(Requester): The user to get a snapshot for. room_id(str): The room to get a snapshot of. pagin_config(synapse.streams.config.PaginationConfig): The pagination config used to determine how many messages to @@ -573,19 +571,20 @@ class MessageHandler(BaseHandler): A JSON serialisable dict with the snapshot of the room. """ + user_id = requester.user.to_string() + membership, member_event_id = yield self._check_in_room_or_world_readable( - room_id, - user_id, - is_guest + room_id, user_id, ) + is_peeking = member_event_id is None if membership == Membership.JOIN: result = yield self._room_initial_sync_joined( - user_id, room_id, pagin_config, membership, is_guest + user_id, room_id, pagin_config, membership, is_peeking ) elif membership == Membership.LEAVE: result = yield self._room_initial_sync_parted( - user_id, room_id, pagin_config, membership, member_event_id, is_guest + user_id, room_id, pagin_config, membership, member_event_id, is_peeking ) account_data_events = [] @@ -609,7 +608,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def _room_initial_sync_parted(self, user_id, room_id, pagin_config, - membership, member_event_id, is_guest): + membership, member_event_id, is_peeking): room_state = yield self.store.get_state_for_events( [member_event_id], None ) @@ -631,7 +630,7 @@ class MessageHandler(BaseHandler): ) messages = yield self._filter_events_for_client( - user_id, messages, is_guest=is_guest + user_id, messages, is_peeking=is_peeking ) start_token = StreamToken(token[0], 0, 0, 0, 0) @@ -654,7 +653,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def _room_initial_sync_joined(self, user_id, room_id, pagin_config, - membership, is_guest): + membership, is_peeking): current_state = yield self.state.get_current_state( room_id=room_id, ) @@ -718,7 +717,7 @@ class MessageHandler(BaseHandler): ).addErrback(unwrapFirstError) messages = yield self._filter_events_for_client( - user_id, messages, is_guest=is_guest, + user_id, messages, is_peeking=is_peeking, ) start_token = now_token.copy_and_replace("room_key", token[0]) @@ -737,7 +736,7 @@ class MessageHandler(BaseHandler): "presence": presence, "receipts": receipts, } - if not is_guest: + if not is_peeking: ret["membership"] = membership defer.returnValue(ret) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a1baf9d20..58e2d25f9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -936,7 +936,7 @@ class RoomContextHandler(BaseHandler): return self._filter_events_for_client( user.to_string(), events, - is_guest=is_guest) + is_peeking=is_guest) event = yield self.store.get_event(event_id, get_prev_content=True, allow_none=True) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index aca200c1e..53e1eb050 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -150,7 +150,7 @@ class SyncHandler(BaseHandler): return self.current_sync_for_user(sync_config, since_token) result = yield self.notifier.wait_for_events( - sync_config.user, timeout, current_sync_callback, + sync_config.user.to_string(), timeout, current_sync_callback, from_token=since_token ) defer.returnValue(result) @@ -640,7 +640,7 @@ class SyncHandler(BaseHandler): loaded_recents = yield self._filter_events_for_client( sync_config.user.to_string(), loaded_recents, - is_guest=sync_config.is_guest, + is_peeking=sync_config.is_guest, ) loaded_recents.extend(recents) recents = loaded_recents diff --git a/synapse/notifier.py b/synapse/notifier.py index 0a5653b8d..328548755 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -63,9 +63,9 @@ class _NotifierUserStream(object): so that it can remove itself from the indexes in the Notifier class. """ - def __init__(self, user, rooms, current_token, time_now_ms, + def __init__(self, user_id, rooms, current_token, time_now_ms, appservice=None): - self.user = str(user) + self.user_id = user_id self.appservice = appservice self.rooms = set(rooms) self.current_token = current_token @@ -98,7 +98,7 @@ class _NotifierUserStream(object): lst = notifier.room_to_user_streams.get(room, set()) lst.discard(self) - notifier.user_to_user_stream.pop(self.user) + notifier.user_to_user_stream.pop(self.user_id) if self.appservice: notifier.appservice_to_user_streams.get( @@ -271,21 +271,20 @@ class Notifier(object): logger.exception("Failed to notify listener") @defer.inlineCallbacks - def wait_for_events(self, user, timeout, callback, room_ids=None, + def wait_for_events(self, user_id, timeout, callback, room_ids=None, from_token=StreamToken("s0", "0", "0", "0", "0")): """Wait until the callback returns a non empty response or the timeout fires. """ - user = str(user) - user_stream = self.user_to_user_stream.get(user) + user_stream = self.user_to_user_stream.get(user_id) if user_stream is None: - appservice = yield self.store.get_app_service_by_user_id(user) + appservice = yield self.store.get_app_service_by_user_id(user_id) current_token = yield self.event_sources.get_current_token() if room_ids is None: - rooms = yield self.store.get_rooms_for_user(user) + rooms = yield self.store.get_rooms_for_user(user_id) room_ids = [room.room_id for room in rooms] user_stream = _NotifierUserStream( - user=user, + user_id=user_id, rooms=room_ids, appservice=appservice, current_token=current_token, @@ -333,12 +332,17 @@ class Notifier(object): @defer.inlineCallbacks def get_events_for(self, user, pagination_config, timeout, only_room_events=False, - is_guest=False, guest_room_id=None): + is_guest=False, explicit_room_id=None): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any new events to happen before returning. If `only_room_events` is `True` only room events will be returned. + + If explicit_room_id is not set, the user's joined rooms will be polled + for events. + If explicit_room_id is set, that room will be polled for events only if + it is world readable or the user has joined the room. """ from_token = pagination_config.from_token if not from_token: @@ -346,15 +350,8 @@ class Notifier(object): limit = pagination_config.limit - room_ids = [] - if is_guest: - if guest_room_id: - if not (yield self._is_world_readable(guest_room_id)): - raise AuthError(403, "Guest access not allowed") - room_ids = [guest_room_id] - else: - rooms = yield self.store.get_rooms_for_user(user.to_string()) - room_ids = [room.room_id for room in rooms] + room_ids, is_joined = yield self._get_room_ids(user, explicit_room_id) + is_peeking = not is_joined @defer.inlineCallbacks def check_for_updates(before_token, after_token): @@ -376,7 +373,7 @@ class Notifier(object): user=user, from_key=getattr(from_token, keyname), limit=limit, - is_guest=is_guest, + is_guest=is_peeking, room_ids=room_ids, ) @@ -385,7 +382,7 @@ class Notifier(object): new_events = yield room_member_handler._filter_events_for_client( user.to_string(), new_events, - is_guest=is_guest, + is_peeking=is_peeking, ) events.extend(new_events) @@ -396,8 +393,24 @@ class Notifier(object): else: defer.returnValue(None) + user_id_for_stream = user.to_string() + if is_peeking: + # Internally, the notifier keeps an event stream per user_id. + # This is used by both /sync and /events. + # We want /events to be used for peeking independently of /sync, + # without polluting its contents. So we invent an illegal user ID + # (which thus cannot clash with any real users) for keying peeking + # over /events. + # + # I am sorry for what I have done. + user_id_for_stream = "_PEEKING_" + user_id_for_stream + result = yield self.wait_for_events( - user, timeout, check_for_updates, room_ids=room_ids, from_token=from_token + user_id_for_stream, + timeout, + check_for_updates, + room_ids=room_ids, + from_token=from_token, ) if result is None: @@ -405,6 +418,18 @@ class Notifier(object): defer.returnValue(result) + @defer.inlineCallbacks + def _get_room_ids(self, user, explicit_room_id): + joined_rooms = yield self.store.get_rooms_for_user(user.to_string()) + joined_room_ids = map(lambda r: r.room_id, joined_rooms) + if explicit_room_id: + if explicit_room_id in joined_room_ids: + defer.returnValue(([explicit_room_id], True)) + if (yield self._is_world_readable(explicit_room_id)): + defer.returnValue(([explicit_room_id], False)) + raise AuthError(403, "Non-joined access not allowed") + defer.returnValue((joined_room_ids, True)) + @defer.inlineCallbacks def _is_world_readable(self, room_id): state = yield self.hs.get_state_handler().get_current_state( @@ -432,7 +457,7 @@ class Notifier(object): @log_function def _register_with_keys(self, user_stream): - self.user_to_user_stream[user_stream.user] = user_stream + self.user_to_user_stream[user_stream.user_id] = user_stream for room in user_stream.rooms: s = self.room_to_user_streams.setdefault(room, set()) diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index e89118b37..d1afa0f0d 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -43,6 +43,7 @@ class EventStreamRestServlet(ClientV1RestServlet): if is_guest: if "room_id" not in request.args: raise SynapseError(400, "Guest users must specify room_id param") + if "room_id" in request.args: room_id = request.args["room_id"][0] try: handler = self.handlers.event_stream_handler diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 85b9f253e..c7ea15c62 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -348,8 +348,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): handler = self.handlers.message_handler msgs = yield handler.get_messages( room_id=room_id, - user_id=requester.user.to_string(), - is_guest=requester.is_guest, + requester=requester, pagin_config=pagination_config, as_client_event=as_client_event ) @@ -384,9 +383,8 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, - user_id=requester.user.to_string(), + requester=requester, pagin_config=pagination_config, - is_guest=requester.is_guest, ) defer.returnValue((200, content)) From b088291f14cc1f021d68d3e1657952efef7fdfd0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Jan 2016 15:40:25 +0000 Subject: [PATCH 122/294] Don't explode when given a unicode username in /register/ --- synapse/handlers/register.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 8e601b052..1e99c1303 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -49,11 +49,10 @@ class RegistrationHandler(BaseHandler): def check_username(self, localpart, guest_access_token=None): yield run_on_reactor() - if urllib.quote(localpart) != localpart: + if urllib.quote(localpart.encode('utf-8')) != localpart: raise SynapseError( 400, - "User ID must only contain characters which do not" - " require URL encoding.", + "User ID can only contain characters a-z, 0-9, or '-./'", Codes.INVALID_USERNAME ) From 73ca8e58344524610a2d512977f5a8cc0cc1b4a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Jan 2016 15:42:57 +0000 Subject: [PATCH 123/294] Whine if we give a from param to /sync --- synapse/rest/client/v2_alpha/sync.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index df4b305b4..4114a7e43 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -85,6 +85,13 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): + if "from" in request.args: + # /events used to use 'from', but /sync uses 'since'. + # Lets be helpful and whine if we see a 'from'. + raise SynapseError( + 400, "'from' is not a valid query parameter. Did you mean 'since'?" + ) + requester = yield self.auth.get_user_by_req( request, allow_guest=True ) From 367cfab4e633c892e0d662e3abcd8e1a9c7f2daf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Jan 2016 16:05:09 +0000 Subject: [PATCH 124/294] peppate --- synapse/storage/receipts.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 018140f47..c4232bdc6 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -57,7 +57,7 @@ class ReceiptsStore(SQLBaseStore): return txn.fetchall() defer.returnValue(dict( - (yield self.runInteraction("get_receipts_for_user", f)) + (yield self.runInteraction("get_receipts_for_user", f)) )) @defer.inlineCallbacks @@ -212,7 +212,7 @@ class ReceiptsStore(SQLBaseStore): def insert_linearized_receipt_txn(self, txn, room_id, receipt_type, user_id, event_id, data, stream_id): txn.call_after( - self.get_receipts_for_room.invalidate, (room_id, receipt_type) + self.get_receipts_for_room.invalidate, (room_id, receipt_type) ) txn.call_after( self.get_receipts_for_user.invalidate, (user_id, receipt_type) @@ -348,7 +348,6 @@ class ReceiptsStore(SQLBaseStore): # FIXME: This shouldn't invalidate the whole cache txn.call_after(self.get_linearized_receipts_for_room.invalidate_all) - self._simple_delete_txn( txn, table="receipts_graph", From c43b6dcc756c259e479c13d4f0802850d657fed6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Jan 2016 16:14:48 +0000 Subject: [PATCH 125/294] Fix change_password --- synapse/rest/client/v2_alpha/account.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index fa56249a6..d50717270 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -56,10 +56,9 @@ class PasswordRestServlet(RestServlet): if LoginType.PASSWORD in result: # if using password, they should also be logged in requester = yield self.auth.get_user_by_req(request) - requester_user_id = requester.user.to_string() - if requester_user_id.to_string() != result[LoginType.PASSWORD]: + user_id = requester.user.to_string() + if user_id != result[LoginType.PASSWORD]: raise LoginError(400, "", Codes.UNKNOWN) - user_id = requester_user_id elif LoginType.EMAIL_IDENTITY in result: threepid = result[LoginType.EMAIL_IDENTITY] if 'medium' not in threepid or 'address' not in threepid: From d7c85ad91685beb6e40d03979b2b65f007e09670 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Jan 2016 10:19:05 +0000 Subject: [PATCH 126/294] Add another graph contrib --- contrib/graph/graph3.py | 151 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 contrib/graph/graph3.py diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py new file mode 100644 index 000000000..88d92c89d --- /dev/null +++ b/contrib/graph/graph3.py @@ -0,0 +1,151 @@ +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pydot +import cgi +import simplejson as json +import datetime +import argparse + +from synapse.events import FrozenEvent +from synapse.util.frozenutils import unfreeze + + +def make_graph(file_name, room_id, file_prefix, limit): + print "Reading lines" + with open(file_name) as f: + lines = f.readlines() + + print "Read lines" + + events = [FrozenEvent(json.loads(line)) for line in lines] + + print "Loaded events." + + events.sort(key=lambda e: e.depth) + + print "Sorted events" + + if limit: + events = events[-int(limit):] + + node_map = {} + + graph = pydot.Dot(graph_name="Test") + + for event in events: + t = datetime.datetime.fromtimestamp( + float(event.origin_server_ts) / 1000 + ).strftime('%Y-%m-%d %H:%M:%S,%f') + + content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4) + content = content.replace("\n", "
\n") + + print content + content = [] + for key, value in unfreeze(event.get_dict()["content"]).items(): + if value is None: + value = "" + elif isinstance(value, basestring): + pass + else: + value = json.dumps(value) + + content.append( + "%s: %s," % ( + cgi.escape(key, quote=True).encode("ascii", 'xmlcharrefreplace'), + cgi.escape(value, quote=True).encode("ascii", 'xmlcharrefreplace'), + ) + ) + + content = "
\n".join(content) + + print content + + label = ( + "<" + "%(name)s
" + "Type: %(type)s
" + "State key: %(state_key)s
" + "Content: %(content)s
" + "Time: %(time)s
" + "Depth: %(depth)s
" + ">" + ) % { + "name": event.event_id, + "type": event.type, + "state_key": event.get("state_key", None), + "content": content, + "time": t, + "depth": event.depth, + } + + node = pydot.Node( + name=event.event_id, + label=label, + ) + + node_map[event.event_id] = node + graph.add_node(node) + + print "Created Nodes" + + for event in events: + for prev_id, _ in event.prev_events: + try: + end_node = node_map[prev_id] + except: + end_node = pydot.Node( + name=prev_id, + label="<%s>" % (prev_id,), + ) + + node_map[prev_id] = end_node + graph.add_node(end_node) + + edge = pydot.Edge(node_map[event.event_id], end_node) + graph.add_edge(edge) + + print "Created edges" + + graph.write('%s.dot' % file_prefix, format='raw', prog='dot') + + print "Created Dot" + + graph.write_svg("%s.svg" % file_prefix, prog='dot') + + print "Created svg" + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate a PDU graph for a given room by reading " + "from a file with line deliminated events. \n" + "Requires pydot." + ) + parser.add_argument( + "-p", "--prefix", dest="prefix", + help="String to prefix output files with", + default="graph_output" + ) + parser.add_argument( + "-l", "--limit", + help="Only retrieve the last N events.", + ) + parser.add_argument('event_file') + parser.add_argument('room') + + args = parser.parse_args() + + make_graph(args.event_file, args.room, args.prefix, args.limit) From 930ba003f85d58c5ef58c4f4a43e098beee94781 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 21 Jan 2016 11:50:27 +0000 Subject: [PATCH 127/294] Remove member event rule as per SYN-607 --- synapse/push/baserules.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 3b526c4e3..186281dfa 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -218,23 +218,25 @@ BASE_APPEND_UNDERRIDE_RULES = [ } ] }, - { - 'rule_id': 'global/underride/.m.rule.member_event', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'type', - 'pattern': 'm.room.member', - '_id': '_member', - } - ], - 'actions': [ - 'notify', { - 'set_tweak': 'highlight', - 'value': False - } - ] - }, + # This is too simple: https://matrix.org/jira/browse/SYN-607 + # Removing for now + # { + # 'rule_id': 'global/underride/.m.rule.member_event', + # 'conditions': [ + # { + # 'kind': 'event_match', + # 'key': 'type', + # 'pattern': 'm.room.member', + # '_id': '_member', + # } + # ], + # 'actions': [ + # 'notify', { + # 'set_tweak': 'highlight', + # 'value': False + # } + # ] + # }, { 'rule_id': 'global/underride/.m.rule.message', 'conditions': [ From 91a222c66dae86b3e82fb39a1558db89b030a055 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Jan 2016 13:22:26 +0000 Subject: [PATCH 128/294] SYN-606: Peeking does not wake up /events If a real user attempted to first peek into one room, and then another, their room event stream would not be woken up for events in the later room. --- synapse/notifier.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 328548755..262a684f6 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -403,7 +403,9 @@ class Notifier(object): # over /events. # # I am sorry for what I have done. - user_id_for_stream = "_PEEKING_" + user_id_for_stream + user_id_for_stream = "_PEEKING_%s_%s" % ( + explicit_room_id, user_id_for_stream + ) result = yield self.wait_for_events( user_id_for_stream, From c6549117a21fe11257d4475c1edd9feae6cdb685 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Jan 2016 14:02:14 +0000 Subject: [PATCH 129/294] Fix AttributeError --- synapse/http/endpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index a9e024a41..4775f6707 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -88,7 +88,7 @@ class SRVClientEndpoint(object): self.service_name = "_%s._%s.%s" % (service, protocol, domain) if default_port is not None: - self.default_server = self._Server( + self.default_server = _Server( host=domain, port=default_port, priority=0, From 3a00f13436ae981baa9717dc740ea2e6c3d14f3c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Jan 2016 14:55:59 +0000 Subject: [PATCH 130/294] Only compute badge count when necessary. This reverts commit d7265977376eb391007cde55c4b2d9b8f54d452b. --- synapse/push/__init__.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 9a4af2b3c..bbfe9b8a5 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -125,9 +125,6 @@ class Pusher(object): from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') timeout = (300 + random.randint(-60, 60)) * 1000 - # note that we need to get read receipts down the stream as we need to - # wake up when one arrives. we don't need to explicitly look for - # them though. chunk = yield self.evStreamHandler.get_stream( self.user_id, config, timeout=timeout, affect_presence=False ) @@ -135,12 +132,23 @@ class Pusher(object): # limiting to 1 may get 1 event plus 1 presence event, so # pick out the actual event single_event = None + read_receipt = None for c in chunk['chunk']: if 'event_id' in c: # Hmmm... single_event = c + elif c['type'] == 'm.receipt': + read_receipt = c + + have_updated_badge = False + if read_receipt: + for receipt_part in read_receipt['content'].values(): + if 'm.read' in receipt_part: + if self.user_id in receipt_part['m.read'].keys(): + have_updated_badge = True if not single_event: - yield self.update_badge() + if have_updated_badge: + yield self.update_badge() self.last_token = chunk['end'] yield self.store.update_pusher_last_token( self.app_id, @@ -185,6 +193,9 @@ class Pusher(object): yield self.hs.get_pusherpool().remove_pusher( self.app_id, pk, self.user_id ) + else: + if have_updated_badge: + yield self.update_badge() processed = True if not self.alive: From 8f66fe639211988f66778d6b8a40a2a5fd2cfaa1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Jan 2016 15:02:07 +0000 Subject: [PATCH 131/294] Cache get_unread_event_push_actions_by_room_for_user --- synapse/storage/event_push_actions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 6b7cebc9c..aa61cf556 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -15,6 +15,7 @@ from ._base import SQLBaseStore from twisted.internet import defer +from synapse.util.caches.descriptors import cachedInlineCallbacks import logging import ujson as json @@ -46,7 +47,7 @@ class EventPushActionsStore(SQLBaseStore): values ) - @defer.inlineCallbacks + @cachedInlineCallbacks(num_args=3) def get_unread_event_push_actions_by_room_for_user( self, room_id, user_id, last_read_event_id ): From 0e07f2e15de196076d75784f44624861b971f8e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Jan 2016 16:10:37 +0000 Subject: [PATCH 132/294] Only fetch events for rooms and receipts --- synapse/handlers/events.py | 6 +++--- synapse/notifier.py | 6 +++--- synapse/push/__init__.py | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index aca4b6754..254b483da 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -113,10 +113,10 @@ class EventStreamHandler(BaseHandler): @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, as_client_event=True, affect_presence=True, - only_room_events=False, room_id=None, is_guest=False): + only_keys=None, room_id=None, is_guest=False): """Fetches the events stream for a given user. - If `only_room_events` is `True` only room events will be returned. + If `only_keys` is not None, events from keys will be sent down. """ auth_user = UserID.from_string(auth_user_id) @@ -134,7 +134,7 @@ class EventStreamHandler(BaseHandler): events, tokens = yield self.notifier.get_events_for( auth_user, pagin_config, timeout, - only_room_events=only_room_events, + only_keys=only_keys, is_guest=is_guest, explicit_room_id=room_id ) diff --git a/synapse/notifier.py b/synapse/notifier.py index 262a684f6..6eaa65071 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -331,13 +331,13 @@ class Notifier(object): @defer.inlineCallbacks def get_events_for(self, user, pagination_config, timeout, - only_room_events=False, + only_keys=None, is_guest=False, explicit_room_id=None): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any new events to happen before returning. - If `only_room_events` is `True` only room events will be returned. + If `only_keys` is not None, events from keys will be sent down. If explicit_room_id is not set, the user's joined rooms will be polled for events. @@ -367,7 +367,7 @@ class Notifier(object): after_id = getattr(after_token, keyname) if before_id == after_id: continue - if only_room_events and name != "room": + if only_keys and name not in only_keys: continue new_events, new_key = yield source.get_new_events( user=user, diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index bbfe9b8a5..e6a28bd8c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -126,7 +126,8 @@ class Pusher(object): config = PaginationConfig(from_token=from_tok, limit='1') timeout = (300 + random.randint(-60, 60)) * 1000 chunk = yield self.evStreamHandler.get_stream( - self.user_id, config, timeout=timeout, affect_presence=False + self.user_id, config, timeout=timeout, affect_presence=False, + only_keys=("room", "receipt",), ) # limiting to 1 may get 1 event plus 1 presence event, so From f1f81221205cf2ec101f96234050569d6419fd6b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 21 Jan 2016 19:16:25 +0000 Subject: [PATCH 133/294] Change LRUCache to be tree-based so we can delete subtrees. --- synapse/push/push_rule_evaluator.py | 6 +-- synapse/util/caches/descriptors.py | 11 +++++- synapse/util/caches/dictionary_cache.py | 10 ++--- synapse/util/caches/lrucache.py | 43 ++++++++++++++++---- synapse/util/caches/treecache.py | 52 +++++++++++++++++++++++++ tests/storage/test__base.py | 26 ++++++------- tests/util/test_lrucache.py | 44 ++++++++++----------- 7 files changed, 140 insertions(+), 52 deletions(-) create mode 100644 synapse/util/caches/treecache.py diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index dca018af9..27b0de4f6 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -309,14 +309,14 @@ def _flatten_dict(d, prefix=[], result={}): return result -regex_cache = LruCache(5000) +regex_cache = LruCache(5000, 1) def _compile_regex(regex_str): - r = regex_cache.get(regex_str, None) + r = regex_cache.get((regex_str,), None) if r: return r r = re.compile(regex_str, flags=re.IGNORECASE) - regex_cache[regex_str] = r + regex_cache[(regex_str,)] = r return r diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 003305184..af7bf1550 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -38,7 +38,7 @@ class Cache(object): def __init__(self, name, max_entries=1000, keylen=1, lru=True): if lru: - self.cache = LruCache(max_size=max_entries) + self.cache = LruCache(max_size=max_entries, keylen=keylen) self.max_entries = None else: self.cache = OrderedDict() @@ -99,6 +99,15 @@ class Cache(object): self.sequence += 1 self.cache.pop(key, None) + def invalidate_many(self, key): + self.check_thread() + if not isinstance(key, tuple): + raise TypeError( + "The cache key must be a tuple not %r" % (type(key),) + ) + self.sequence += 1 + self.cache.del_multi(key) + def invalidate_all(self): self.check_thread() self.sequence += 1 diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index f92d80542..b7964467e 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -32,7 +32,7 @@ class DictionaryCache(object): """ def __init__(self, name, max_entries=1000): - self.cache = LruCache(max_size=max_entries) + self.cache = LruCache(max_size=max_entries, keylen=1) self.name = name self.sequence = 0 @@ -56,7 +56,7 @@ class DictionaryCache(object): ) def get(self, key, dict_keys=None): - entry = self.cache.get(key, self.sentinel) + entry = self.cache.get((key,), self.sentinel) if entry is not self.sentinel: cache_counter.inc_hits(self.name) @@ -78,7 +78,7 @@ class DictionaryCache(object): # Increment the sequence number so that any SELECT statements that # raced with the INSERT don't update the cache (SYN-369) self.sequence += 1 - self.cache.pop(key, None) + self.cache.pop((key,), None) def invalidate_all(self): self.check_thread() @@ -96,8 +96,8 @@ class DictionaryCache(object): self._update_or_insert(key, value) def _update_or_insert(self, key, value): - entry = self.cache.setdefault(key, DictionaryEntry(False, {})) + entry = self.cache.setdefault((key,), DictionaryEntry(False, {})) entry.value.update(value) def _insert(self, key, value): - self.cache[key] = DictionaryEntry(True, value) + self.cache[(key,)] = DictionaryEntry(True, value) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 0122b0bb3..0feceb298 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -17,11 +17,23 @@ from functools import wraps import threading +from synapse.util.caches.treecache import TreeCache + + +def enumerate_leaves(node, depth): + if depth == 0: + yield node + else: + for n in node.values(): + for m in enumerate_leaves(n, depth - 1): + yield m + class LruCache(object): """Least-recently-used cache.""" - def __init__(self, max_size): - cache = {} + def __init__(self, max_size, keylen): + cache = TreeCache() + self.size = 0 list_root = [] list_root[:] = [list_root, list_root, None, None] @@ -44,6 +56,7 @@ class LruCache(object): prev_node[NEXT] = node next_node[PREV] = node cache[key] = node + self.size += 1 def move_node_to_front(node): prev_node = node[PREV] @@ -62,7 +75,7 @@ class LruCache(object): next_node = node[NEXT] prev_node[NEXT] = next_node next_node[PREV] = prev_node - cache.pop(node[KEY], None) + self.size -= 1 @synchronized def cache_get(key, default=None): @@ -81,8 +94,10 @@ class LruCache(object): node[VALUE] = value else: add_node(key, value) - if len(cache) > max_size: - delete_node(list_root[PREV]) + if self.size > max_size: + todelete = list_root[PREV] + delete_node(todelete) + cache.pop(todelete[KEY], None) @synchronized def cache_set_default(key, value): @@ -91,8 +106,10 @@ class LruCache(object): return node[VALUE] else: add_node(key, value) - if len(cache) > max_size: - delete_node(list_root[PREV]) + if self.size > max_size: + todelete = list_root[PREV] + delete_node(todelete) + cache.pop(todelete[KEY], None) return value @synchronized @@ -100,10 +117,19 @@ class LruCache(object): node = cache.get(key, None) if node: delete_node(node) + cache.pop(node[KEY], None) return node[VALUE] else: return default + @synchronized + def cache_del_multi(key): + popped = cache.pop(key) + if popped is None: + return + for leaf in enumerate_leaves(popped, keylen - len(key)): + delete_node(leaf) + @synchronized def cache_clear(): list_root[NEXT] = list_root @@ -112,7 +138,7 @@ class LruCache(object): @synchronized def cache_len(): - return len(cache) + return self.size @synchronized def cache_contains(key): @@ -123,6 +149,7 @@ class LruCache(object): self.set = cache_set self.setdefault = cache_set_default self.pop = cache_pop + self.del_multi = cache_del_multi self.len = cache_len self.contains = cache_contains self.clear = cache_clear diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py new file mode 100644 index 000000000..1e5f87e6a --- /dev/null +++ b/synapse/util/caches/treecache.py @@ -0,0 +1,52 @@ +SENTINEL = object() + + +class TreeCache(object): + def __init__(self): + self.root = {} + + def __setitem__(self, key, value): + return self.set(key, value) + + def set(self, key, value): + node = self.root + for k in key[:-1]: + node = node.setdefault(k, {}) + node[key[-1]] = value + + def get(self, key, default=None): + node = self.root + for k in key[:-1]: + node = node.get(k, None) + if node is None: + return default + return node.get(key[-1], default) + + def clear(self): + self.root = {} + + def pop(self, key, default=None): + nodes = [] + + node = self.root + for k in key[:-1]: + node = node.get(k, None) + nodes.append(node) # don't add the root node + if node is None: + return default + popped = node.pop(key[-1], SENTINEL) + if popped is SENTINEL: + return default + + node_and_keys = zip(nodes, key) + node_and_keys.reverse() + node_and_keys.append((self.root, None)) + + for i in range(len(node_and_keys) - 1): + n,k = node_and_keys[i] + + if n: + break + node_and_keys[i+1][0].pop(k) + + return popped \ No newline at end of file diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 219288621..c4e4c9b4b 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -56,42 +56,42 @@ class CacheTestCase(unittest.TestCase): def test_eviction(self): cache = Cache("test", max_entries=2) - cache.prefill(1, "one") - cache.prefill(2, "two") - cache.prefill(3, "three") # 1 will be evicted + cache.prefill((1,), "one") + cache.prefill((2,), "two") + cache.prefill((3,), "three") # 1 will be evicted failed = False try: - cache.get(1) + cache.get((1,)) except KeyError: failed = True self.assertTrue(failed) - cache.get(2) - cache.get(3) + cache.get((2,)) + cache.get((3,)) def test_eviction_lru(self): cache = Cache("test", max_entries=2, lru=True) - cache.prefill(1, "one") - cache.prefill(2, "two") + cache.prefill((1,), "one") + cache.prefill((2,), "two") # Now access 1 again, thus causing 2 to be least-recently used - cache.get(1) + cache.get((1,)) - cache.prefill(3, "three") + cache.prefill((3,), "three") failed = False try: - cache.get(2) + cache.get((2,)) except KeyError: failed = True self.assertTrue(failed) - cache.get(1) - cache.get(3) + cache.get((1,)) + cache.get((3,)) class CacheDecoratorTestCase(unittest.TestCase): diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index fbbc5eed1..80c19b944 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -21,34 +21,34 @@ from synapse.util.caches.lrucache import LruCache class LruCacheTestCase(unittest.TestCase): def test_get_set(self): - cache = LruCache(1) - cache["key"] = "value" - self.assertEquals(cache.get("key"), "value") - self.assertEquals(cache["key"], "value") + cache = LruCache(1, 1) + cache[("key",)] = "value" + self.assertEquals(cache.get(("key",)), "value") + self.assertEquals(cache[("key",)], "value") def test_eviction(self): - cache = LruCache(2) - cache[1] = 1 - cache[2] = 2 + cache = LruCache(2, 1) + cache[(1,)] = 1 + cache[(2,)] = 2 - self.assertEquals(cache.get(1), 1) - self.assertEquals(cache.get(2), 2) + self.assertEquals(cache.get((1,)), 1) + self.assertEquals(cache.get((2,)), 2) - cache[3] = 3 + cache[(3,)] = 3 - self.assertEquals(cache.get(1), None) - self.assertEquals(cache.get(2), 2) - self.assertEquals(cache.get(3), 3) + self.assertEquals(cache.get((1,)), None) + self.assertEquals(cache.get((2,)), 2) + self.assertEquals(cache.get((3,)), 3) def test_setdefault(self): - cache = LruCache(1) - self.assertEquals(cache.setdefault("key", 1), 1) - self.assertEquals(cache.get("key"), 1) - self.assertEquals(cache.setdefault("key", 2), 1) - self.assertEquals(cache.get("key"), 1) + cache = LruCache(1, 1) + self.assertEquals(cache.setdefault(("key",), 1), 1) + self.assertEquals(cache.get(("key",)), 1) + self.assertEquals(cache.setdefault(("key",), 2), 1) + self.assertEquals(cache.get(("key",)), 1) def test_pop(self): - cache = LruCache(1) - cache["key"] = 1 - self.assertEquals(cache.pop("key"), 1) - self.assertEquals(cache.pop("key"), None) + cache = LruCache(1, 1) + cache[("key",)] = 1 + self.assertEquals(cache.pop(("key",)), 1) + self.assertEquals(cache.pop(("key",)), None) From 330be18ec5db6e903ac4d5fe97b3e3940fea2a5b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 21 Jan 2016 19:17:32 +0000 Subject: [PATCH 134/294] peppate --- synapse/util/caches/treecache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index 1e5f87e6a..6faf2cf75 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -43,10 +43,10 @@ class TreeCache(object): node_and_keys.append((self.root, None)) for i in range(len(node_and_keys) - 1): - n,k = node_and_keys[i] + n, k = node_and_keys[i] if n: break node_and_keys[i+1][0].pop(k) - return popped \ No newline at end of file + return popped From 4efcaa43c8c69c7fdbaec74d7af2b71dbc6faea6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 10:37:37 +0000 Subject: [PATCH 135/294] Add tests for treecache directly and test del_multi at the LruCache level too. --- tests/util/test_treecache.py | 66 ++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/util/test_treecache.py diff --git a/tests/util/test_treecache.py b/tests/util/test_treecache.py new file mode 100644 index 000000000..9946ceb3f --- /dev/null +++ b/tests/util/test_treecache.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2015, 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import unittest + +from synapse.util.caches.treecache import TreeCache + +class TreeCacheTestCase(unittest.TestCase): + def test_get_set_onelevel(self): + cache = TreeCache() + cache[("a",)] = "A" + cache[("b",)] = "B" + self.assertEquals(cache.get(("a",)), "A") + self.assertEquals(cache.get(("b",)), "B") + + def test_pop_onelevel(self): + cache = TreeCache() + cache[("a",)] = "A" + cache[("b",)] = "B" + self.assertEquals(cache.pop(("a",)), "A") + self.assertEquals(cache.pop(("a",)), None) + self.assertEquals(cache.get(("b",)), "B") + + def test_get_set_twolevel(self): + cache = TreeCache() + cache[("a", "a")] = "AA" + cache[("a", "b")] = "AB" + cache[("b", "a")] = "BA" + self.assertEquals(cache.get(("a", "a")), "AA") + self.assertEquals(cache.get(("a", "b")), "AB") + self.assertEquals(cache.get(("b", "a")), "BA") + + def test_pop_twolevel(self): + cache = TreeCache() + cache[("a", "a")] = "AA" + cache[("a", "b")] = "AB" + cache[("b", "a")] = "BA" + self.assertEquals(cache.pop(("a", "a")), "AA") + self.assertEquals(cache.get(("a", "a")), None) + self.assertEquals(cache.get(("a", "b")), "AB") + self.assertEquals(cache.pop(("b", "a")), "BA") + self.assertEquals(cache.pop(("b", "a")), None) + + def test_pop_mixedlevel(self): + cache = TreeCache() + cache[("a", "a")] = "AA" + cache[("a", "b")] = "AB" + cache[("b", "a")] = "BA" + self.assertEquals(cache.get(("a", "a")), "AA") + cache.pop(("a",)) + self.assertEquals(cache.get(("a", "a")), None) + self.assertEquals(cache.get(("a", "b")), None) + self.assertEquals(cache.get(("b", "a")), "BA") From 975903ae1735f690a387c10b1ffd1a9384353213 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Jan 2016 10:41:30 +0000 Subject: [PATCH 136/294] Sanitize filters --- synapse/api/filtering.py | 48 +++++++++++++++----------- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 24 ++++++------- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index c7f021d1f..5530b8c48 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -28,14 +28,14 @@ class Filtering(object): return result def add_user_filter(self, user_localpart, user_filter): - self._check_valid_filter(user_filter) + self.check_valid_filter(user_filter) return self.store.add_user_filter(user_localpart, user_filter) # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however - def _check_valid_filter(self, user_filter_json): + def check_valid_filter(self, user_filter_json): """Check if the provided filter is valid. This inspects all definitions contained within the filter. @@ -129,52 +129,55 @@ class Filtering(object): class FilterCollection(object): def __init__(self, filter_json): - self.filter_json = filter_json + self._filter_json = filter_json - room_filter_json = self.filter_json.get("room", {}) + room_filter_json = self._filter_json.get("room", {}) - self.room_filter = Filter({ + self._room_filter = Filter({ k: v for k, v in room_filter_json.items() if k in ("rooms", "not_rooms") }) - self.room_timeline_filter = Filter(room_filter_json.get("timeline", {})) - self.room_state_filter = Filter(room_filter_json.get("state", {})) - self.room_ephemeral_filter = Filter(room_filter_json.get("ephemeral", {})) - self.room_account_data = Filter(room_filter_json.get("account_data", {})) - self.presence_filter = Filter(self.filter_json.get("presence", {})) - self.account_data = Filter(self.filter_json.get("account_data", {})) + self._room_timeline_filter = Filter(room_filter_json.get("timeline", {})) + self._room_state_filter = Filter(room_filter_json.get("state", {})) + self._room_ephemeral_filter = Filter(room_filter_json.get("ephemeral", {})) + self._room_account_data = Filter(room_filter_json.get("account_data", {})) + self._presence_filter = Filter(filter_json.get("presence", {})) + self._account_data = Filter(filter_json.get("account_data", {})) - self.include_leave = self.filter_json.get("room", {}).get( + self.include_leave = filter_json.get("room", {}).get( "include_leave", False ) + def get_filter_json(self): + return self._filter_json + def timeline_limit(self): - return self.room_timeline_filter.limit() + return self._room_timeline_filter.limit() def presence_limit(self): - return self.presence_filter.limit() + return self._presence_filter.limit() def ephemeral_limit(self): - return self.room_ephemeral_filter.limit() + return self._room_ephemeral_filter.limit() def filter_presence(self, events): - return self.presence_filter.filter(events) + return self._presence_filter.filter(events) def filter_account_data(self, events): - return self.account_data.filter(events) + return self._account_data.filter(events) def filter_room_state(self, events): - return self.room_state_filter.filter(self.room_filter.filter(events)) + return self._room_state_filter.filter(self._room_filter.filter(events)) def filter_room_timeline(self, events): - return self.room_timeline_filter.filter(self.room_filter.filter(events)) + return self._room_timeline_filter.filter(self._room_filter.filter(events)) def filter_room_ephemeral(self, events): - return self.room_ephemeral_filter.filter(self.room_filter.filter(events)) + return self._room_ephemeral_filter.filter(self._room_filter.filter(events)) def filter_room_account_data(self, events): - return self.room_account_data.filter(self.room_filter.filter(events)) + return self._room_account_data.filter(self._room_filter.filter(events)) class Filter(object): @@ -258,3 +261,6 @@ def _matches_wildcard(actual_value, filter_value): return actual_value.startswith(type_prefix) else: return actual_value == filter_value + + +DEFAULT_FILTER_COLLECTION = FilterCollection({}) diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 7695bebc2..7c94f6ec4 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -59,7 +59,7 @@ class GetFilterRestServlet(RestServlet): filter_id=filter_id, ) - defer.returnValue((200, filter.filter_json)) + defer.returnValue((200, filter.get_filter_json())) except KeyError: raise SynapseError(400, "No such filter") diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 4114a7e43..ab924ad9e 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -24,7 +24,7 @@ from synapse.events import FrozenEvent from synapse.events.utils import ( serialize_event, format_event_for_client_v2_without_room_id, ) -from synapse.api.filtering import FilterCollection +from synapse.api.filtering import FilterCollection, DEFAULT_FILTER_COLLECTION from synapse.api.errors import SynapseError from ._base import client_v2_patterns @@ -113,20 +113,20 @@ class SyncRestServlet(RestServlet): ) ) - if filter_id and filter_id.startswith('{'): - try: - filter_object = json.loads(filter_id) - except: - raise SynapseError(400, "Invalid filter JSON") - self.filtering._check_valid_filter(filter_object) - filter = FilterCollection(filter_object) - else: - try: + if filter_id: + if filter_id.startswith('{'): + try: + filter_object = json.loads(filter_id) + except: + raise SynapseError(400, "Invalid filter JSON") + self.filtering.check_valid_filter(filter_object) + filter = FilterCollection(filter_object) + else: filter = yield self.filtering.get_user_filter( user.localpart, filter_id ) - except: - filter = FilterCollection({}) + else: + filter = DEFAULT_FILTER_COLLECTION sync_config = SyncConfig( user=user, From 8f9c74e9f12050b6680355fc93758b28672e9358 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Jan 2016 10:48:27 +0000 Subject: [PATCH 137/294] Fix tests --- tests/api/test_filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 14cddee67..16ee6bbe6 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -504,4 +504,4 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - self.assertEquals(filter.filter_json, user_filter_json) + self.assertEquals(filter.get_filter_json(), user_filter_json) From 31a051b6771bf720c78246bd6c8875d219ddbc88 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 11:22:00 +0000 Subject: [PATCH 138/294] Test treecache directly --- tests/util/test_lrucache.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index 80c19b944..fca2e9898 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -52,3 +52,22 @@ class LruCacheTestCase(unittest.TestCase): cache[("key",)] = 1 self.assertEquals(cache.pop(("key",)), 1) self.assertEquals(cache.pop(("key",)), None) + + def test_del_multi(self): + cache = LruCache(4, 2) + cache[("animal", "cat")] = "mew" + cache[("animal", "dog")] = "woof" + cache[("vehicles", "car")] = "vroom" + cache[("vehicles", "train")] = "chuff" + + self.assertEquals(len(cache), 4) + + self.assertEquals(cache.get(("animal", "cat")), "mew") + self.assertEquals(cache.get(("vehicles", "car")), "vroom") + cache.del_multi(("animal",)) + self.assertEquals(len(cache), 2) + self.assertEquals(cache.get(("animal", "cat")), None) + self.assertEquals(cache.get(("animal", "dog")), None) + self.assertEquals(cache.get(("vehicles", "car")), "vroom") + self.assertEquals(cache.get(("vehicles", "train")), "chuff") + # Man from del_multi say "Yes". From 8acc5cb60f551f56815c60a33c36d6752f4b0967 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 11:22:32 +0000 Subject: [PATCH 139/294] Add invalidate_many here too --- synapse/util/caches/descriptors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index af7bf1550..f4a2b4e59 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -209,6 +209,7 @@ class CacheDescriptor(object): wrapped.invalidate = self.cache.invalidate wrapped.invalidate_all = self.cache.invalidate_all + wrapped.invalidate_many = self.cache.invalidate_many wrapped.prefill = self.cache.prefill obj.__dict__[self.orig.__name__] = wrapped From 42eae4634f4bab5649298a65889a4b1a3149d586 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 11:22:48 +0000 Subject: [PATCH 140/294] Use new invalidate_many cache invalidation to invalidate the event_push_actions cache appropriately. --- synapse/storage/event_push_actions.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index aa61cf556..6a212c630 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -40,14 +40,20 @@ class EventPushActionsStore(SQLBaseStore): 'actions': json.dumps(actions) }) + def f(txn): + for uid, _, __ in tuples: + txn.call_after( + self.get_unread_event_push_actions_by_room_for_user.invalidate_many, + (event.room_id, uid) + ) + return self._simple_insert_many_txn(txn, "event_push_actions", values) + yield self.runInteraction( "set_actions_for_event_and_users", - self._simple_insert_many_txn, - "event_push_actions", - values + f, ) - @cachedInlineCallbacks(num_args=3) + @cachedInlineCallbacks(num_args=3, lru=True) def get_unread_event_push_actions_by_room_for_user( self, room_id, user_id, last_read_event_id ): @@ -98,6 +104,11 @@ class EventPushActionsStore(SQLBaseStore): @defer.inlineCallbacks def remove_push_actions_for_event_id(self, room_id, event_id): def f(txn): + # Sad that we have to blow away the cache for the whole room here + txn.call_after( + self.get_unread_event_push_actions_by_room_for_user.invalidate_many, + (room_id,) + ) txn.execute( "DELETE FROM event_push_actions WHERE room_id = ? AND event_id = ?", (room_id, event_id) From eaa836e8cafd2254e73b935dc1e471f784b30773 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 11:47:22 +0000 Subject: [PATCH 141/294] Docs for treecache --- synapse/util/caches/treecache.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index 6faf2cf75..2d014e1f9 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -2,6 +2,11 @@ SENTINEL = object() class TreeCache(object): + """ + Tree-based backing store for LruCache. Allows subtrees of data to be deleted + efficiently. + Keys must be tuples. + """ def __init__(self): self.root = {} From 5b142788d22fc5c225b4897a914ddf96cfd6a1f0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 11:49:59 +0000 Subject: [PATCH 142/294] Add __contains__ --- synapse/util/caches/treecache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index 2d014e1f9..3b5886091 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -13,6 +13,9 @@ class TreeCache(object): def __setitem__(self, key, value): return self.set(key, value) + def __contains__(self, key): + return self.get(key, SENTINEL) is not SENTINEL + def set(self, key, value): node = self.root for k in key[:-1]: From 10f76dc5da47c49a4191d8113b3c0615224eb9fd Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 12:10:33 +0000 Subject: [PATCH 143/294] Make LRU cache not default to treecache & add options to use it --- synapse/storage/event_push_actions.py | 2 +- synapse/util/caches/descriptors.py | 34 +++++++++++++++++---------- synapse/util/caches/lrucache.py | 9 +++---- tests/util/test_lrucache.py | 3 ++- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 6a212c630..a05c4f84c 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -53,7 +53,7 @@ class EventPushActionsStore(SQLBaseStore): f, ) - @cachedInlineCallbacks(num_args=3, lru=True) + @cachedInlineCallbacks(num_args=3, lru=True, tree=True) def get_unread_event_push_actions_by_room_for_user( self, room_id, user_id, last_read_event_id ): diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index f4a2b4e59..88e56e330 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -17,6 +17,7 @@ import logging from synapse.util.async import ObservableDeferred from synapse.util import unwrapFirstError from synapse.util.caches.lrucache import LruCache +from synapse.util.caches.treecache import TreeCache from . import caches_by_name, DEBUG_CACHES, cache_counter @@ -36,9 +37,12 @@ _CacheSentinel = object() class Cache(object): - def __init__(self, name, max_entries=1000, keylen=1, lru=True): + def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False): if lru: - self.cache = LruCache(max_size=max_entries, keylen=keylen) + cache_type = TreeCache if tree else dict + self.cache = LruCache( + max_size=max_entries, keylen=keylen, cache_type=cache_type + ) self.max_entries = None else: self.cache = OrderedDict() @@ -131,7 +135,7 @@ class CacheDescriptor(object): which can be used to insert values into the cache specifically, without calling the calculation function. """ - def __init__(self, orig, max_entries=1000, num_args=1, lru=True, + def __init__(self, orig, max_entries=1000, num_args=1, lru=True, tree=False, inlineCallbacks=False): self.orig = orig @@ -143,6 +147,7 @@ class CacheDescriptor(object): self.max_entries = max_entries self.num_args = num_args self.lru = lru + self.tree = tree self.arg_names = inspect.getargspec(orig).args[1:num_args+1] @@ -158,6 +163,7 @@ class CacheDescriptor(object): max_entries=self.max_entries, keylen=self.num_args, lru=self.lru, + tree=self.tree, ) def __get__(self, obj, objtype=None): @@ -331,21 +337,23 @@ class CacheListDescriptor(object): return wrapped -def cached(max_entries=1000, num_args=1, lru=True): - return lambda orig: CacheDescriptor( - orig, - max_entries=max_entries, - num_args=num_args, - lru=lru - ) - - -def cachedInlineCallbacks(max_entries=1000, num_args=1, lru=False): +def cached(max_entries=1000, num_args=1, lru=True, tree=False): return lambda orig: CacheDescriptor( orig, max_entries=max_entries, num_args=num_args, lru=lru, + tree=tree, + ) + + +def cachedInlineCallbacks(max_entries=1000, num_args=1, lru=False, tree=False): + return lambda orig: CacheDescriptor( + orig, + max_entries=max_entries, + num_args=num_args, + lru=lru, + tree=tree, inlineCallbacks=True, ) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 0feceb298..23e86ec11 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -17,8 +17,6 @@ from functools import wraps import threading -from synapse.util.caches.treecache import TreeCache - def enumerate_leaves(node, depth): if depth == 0: @@ -31,8 +29,8 @@ def enumerate_leaves(node, depth): class LruCache(object): """Least-recently-used cache.""" - def __init__(self, max_size, keylen): - cache = TreeCache() + def __init__(self, max_size, keylen, cache_type=dict): + cache = cache_type() self.size = 0 list_root = [] list_root[:] = [list_root, list_root, None, None] @@ -124,6 +122,9 @@ class LruCache(object): @synchronized def cache_del_multi(key): + """ + This will only work if constructed with cache_type=TreeCache + """ popped = cache.pop(key) if popped is None: return diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index fca2e9898..bcad1d425 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -17,6 +17,7 @@ from .. import unittest from synapse.util.caches.lrucache import LruCache +from synapse.util.caches.treecache import TreeCache class LruCacheTestCase(unittest.TestCase): @@ -54,7 +55,7 @@ class LruCacheTestCase(unittest.TestCase): self.assertEquals(cache.pop(("key",)), None) def test_del_multi(self): - cache = LruCache(4, 2) + cache = LruCache(4, 2, cache_type=TreeCache) cache[("animal", "cat")] = "mew" cache[("animal", "dog")] = "woof" cache[("vehicles", "car")] = "vroom" From d552861346d6f2f3d50fa0aff3e239d17cf9b7c0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 12:18:14 +0000 Subject: [PATCH 144/294] Revert all the bits changing keys of eeverything that used LRUCaches to tuples --- synapse/push/push_rule_evaluator.py | 6 ++-- synapse/util/caches/dictionary_cache.py | 10 +++--- synapse/util/caches/lrucache.py | 2 +- tests/storage/test__base.py | 26 +++++++-------- tests/util/test_lrucache.py | 44 ++++++++++++------------- 5 files changed, 44 insertions(+), 44 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 27b0de4f6..dca018af9 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -309,14 +309,14 @@ def _flatten_dict(d, prefix=[], result={}): return result -regex_cache = LruCache(5000, 1) +regex_cache = LruCache(5000) def _compile_regex(regex_str): - r = regex_cache.get((regex_str,), None) + r = regex_cache.get(regex_str, None) if r: return r r = re.compile(regex_str, flags=re.IGNORECASE) - regex_cache[(regex_str,)] = r + regex_cache[regex_str] = r return r diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index b7964467e..f92d80542 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -32,7 +32,7 @@ class DictionaryCache(object): """ def __init__(self, name, max_entries=1000): - self.cache = LruCache(max_size=max_entries, keylen=1) + self.cache = LruCache(max_size=max_entries) self.name = name self.sequence = 0 @@ -56,7 +56,7 @@ class DictionaryCache(object): ) def get(self, key, dict_keys=None): - entry = self.cache.get((key,), self.sentinel) + entry = self.cache.get(key, self.sentinel) if entry is not self.sentinel: cache_counter.inc_hits(self.name) @@ -78,7 +78,7 @@ class DictionaryCache(object): # Increment the sequence number so that any SELECT statements that # raced with the INSERT don't update the cache (SYN-369) self.sequence += 1 - self.cache.pop((key,), None) + self.cache.pop(key, None) def invalidate_all(self): self.check_thread() @@ -96,8 +96,8 @@ class DictionaryCache(object): self._update_or_insert(key, value) def _update_or_insert(self, key, value): - entry = self.cache.setdefault((key,), DictionaryEntry(False, {})) + entry = self.cache.setdefault(key, DictionaryEntry(False, {})) entry.value.update(value) def _insert(self, key, value): - self.cache[(key,)] = DictionaryEntry(True, value) + self.cache[key] = DictionaryEntry(True, value) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 23e86ec11..5f9405c95 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -29,7 +29,7 @@ def enumerate_leaves(node, depth): class LruCache(object): """Least-recently-used cache.""" - def __init__(self, max_size, keylen, cache_type=dict): + def __init__(self, max_size, keylen=1, cache_type=dict): cache = cache_type() self.size = 0 list_root = [] diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index c4e4c9b4b..219288621 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -56,42 +56,42 @@ class CacheTestCase(unittest.TestCase): def test_eviction(self): cache = Cache("test", max_entries=2) - cache.prefill((1,), "one") - cache.prefill((2,), "two") - cache.prefill((3,), "three") # 1 will be evicted + cache.prefill(1, "one") + cache.prefill(2, "two") + cache.prefill(3, "three") # 1 will be evicted failed = False try: - cache.get((1,)) + cache.get(1) except KeyError: failed = True self.assertTrue(failed) - cache.get((2,)) - cache.get((3,)) + cache.get(2) + cache.get(3) def test_eviction_lru(self): cache = Cache("test", max_entries=2, lru=True) - cache.prefill((1,), "one") - cache.prefill((2,), "two") + cache.prefill(1, "one") + cache.prefill(2, "two") # Now access 1 again, thus causing 2 to be least-recently used - cache.get((1,)) + cache.get(1) - cache.prefill((3,), "three") + cache.prefill(3, "three") failed = False try: - cache.get((2,)) + cache.get(2) except KeyError: failed = True self.assertTrue(failed) - cache.get((1,)) - cache.get((3,)) + cache.get(1) + cache.get(3) class CacheDecoratorTestCase(unittest.TestCase): diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index bcad1d425..2cd3d2645 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -22,37 +22,37 @@ from synapse.util.caches.treecache import TreeCache class LruCacheTestCase(unittest.TestCase): def test_get_set(self): - cache = LruCache(1, 1) - cache[("key",)] = "value" - self.assertEquals(cache.get(("key",)), "value") - self.assertEquals(cache[("key",)], "value") + cache = LruCache(1) + cache["key"] = "value" + self.assertEquals(cache.get("key"), "value") + self.assertEquals(cache["key"], "value") def test_eviction(self): - cache = LruCache(2, 1) - cache[(1,)] = 1 - cache[(2,)] = 2 + cache = LruCache(2) + cache[1] = 1 + cache[2] = 2 - self.assertEquals(cache.get((1,)), 1) - self.assertEquals(cache.get((2,)), 2) + self.assertEquals(cache.get(1), 1) + self.assertEquals(cache.get(2), 2) - cache[(3,)] = 3 + cache[3] = 3 - self.assertEquals(cache.get((1,)), None) - self.assertEquals(cache.get((2,)), 2) - self.assertEquals(cache.get((3,)), 3) + self.assertEquals(cache.get(1), None) + self.assertEquals(cache.get(2), 2) + self.assertEquals(cache.get(3), 3) def test_setdefault(self): - cache = LruCache(1, 1) - self.assertEquals(cache.setdefault(("key",), 1), 1) - self.assertEquals(cache.get(("key",)), 1) - self.assertEquals(cache.setdefault(("key",), 2), 1) - self.assertEquals(cache.get(("key",)), 1) + cache = LruCache(1) + self.assertEquals(cache.setdefault("key", 1), 1) + self.assertEquals(cache.get("key"), 1) + self.assertEquals(cache.setdefault("key", 2), 1) + self.assertEquals(cache.get("key"), 1) def test_pop(self): - cache = LruCache(1, 1) - cache[("key",)] = 1 - self.assertEquals(cache.pop(("key",)), 1) - self.assertEquals(cache.pop(("key",)), None) + cache = LruCache(1) + cache["key"] = 1 + self.assertEquals(cache.pop("key"), 1) + self.assertEquals(cache.pop("key"), None) def test_del_multi(self): cache = LruCache(4, 2, cache_type=TreeCache) From cd80019eec69172f9753e0da25ad22d2f7c434db Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 12:21:13 +0000 Subject: [PATCH 145/294] docs --- synapse/util/caches/lrucache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 5f9405c95..cc6c25a71 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -28,7 +28,11 @@ def enumerate_leaves(node, depth): class LruCache(object): - """Least-recently-used cache.""" + """ + Least-recently-used cache. + Supports del_multi only if cache_type=TreeCache + If cache_type=TreeCache, all keys must be tuples. + """ def __init__(self, max_size, keylen=1, cache_type=dict): cache = cache_type() self.size = 0 From 7cd418d38e472563d8df6546b3227dcfafe3e145 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 13:40:37 +0000 Subject: [PATCH 146/294] Don't add the member functiopn if we're not using treecache --- synapse/util/caches/lrucache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index cc6c25a71..e6a66dc04 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -17,6 +17,8 @@ from functools import wraps import threading +from synapse.util.caches.treecache import TreeCache + def enumerate_leaves(node, depth): if depth == 0: @@ -154,7 +156,8 @@ class LruCache(object): self.set = cache_set self.setdefault = cache_set_default self.pop = cache_pop - self.del_multi = cache_del_multi + if cache_type is TreeCache: + self.del_multi = cache_del_multi self.len = cache_len self.contains = cache_contains self.clear = cache_clear From 52bdd1b8342e0bc20739d277c3c42bddbac77891 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 14:58:19 +0000 Subject: [PATCH 147/294] Overlay the push_rules_enabled map for users, otherwise they won't be able to disable server default rules. --- synapse/push/bulk_push_rule_evaluator.py | 14 ++++++++++++ synapse/storage/push_rule.py | 29 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index b91c165e2..baa32a4e1 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -36,6 +36,7 @@ def decode_rule_json(rule): @defer.inlineCallbacks def _get_rules(room_id, user_ids, store): rules_by_user = yield store.bulk_get_push_rules(user_ids) + rules_enabled_by_user = yield store.bulk_get_push_rules_enabled(user_ids) rules_by_user = { uid: baserules.list_with_base_rules([ @@ -44,6 +45,19 @@ def _get_rules(room_id, user_ids, store): ]) for uid in user_ids } + + # We apply the rules-enabled map here: bulk_get_push_rules doesn't + # fetch disabled rules, but this won't account for any server default + # rules the user has disabled, so we need to do this too. + for uid in user_ids: + user_enabled_map = rules_enabled_by_user[uid] + + for rule in rules_by_user[uid]: + rule_id = rule['rule_id'] + + if rule_id in user_enabled_map: + rule['enabled'] = user_enabled_map[rule_id] + defer.returnValue(rules_by_user) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 2adfefd99..35ec7e8ce 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -93,6 +93,35 @@ class PushRuleStore(SQLBaseStore): results.setdefault(row['user_name'], []).append(row) defer.returnValue(results) + @defer.inlineCallbacks + def bulk_get_push_rules_enabled(self, user_ids): + if not user_ids: + defer.returnValue({}) + + batch_size = 100 + + def f(txn, user_ids_to_fetch): + sql = ( + "SELECT user_name, rule_id, enabled" + " FROM push_rules_enable" + " WHERE user_name" + " IN (" + ",".join("?" for _ in user_ids_to_fetch) + ")" + ) + txn.execute(sql, user_ids_to_fetch) + return self.cursor_to_dict(txn) + + results = {} + + chunks = [user_ids[i:i+batch_size] for i in xrange(0, len(user_ids), batch_size)] + for batch_user_ids in chunks: + rows = yield self.runInteraction( + "bulk_get_push_rules_enabled", f, batch_user_ids + ) + + for row in rows: + results.setdefault(row['user_name'], {})[row['rule_id']] = row['enabled'] + defer.returnValue(results) + @defer.inlineCallbacks def add_push_rule(self, before, after, **kwargs): vals = kwargs From 7959e8b76475527c58c1dda364779a84c6e16368 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Jan 2016 14:59:49 +0000 Subject: [PATCH 148/294] Underscores are allowed in user ids --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 1e99c1303..c11b98d0b 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -52,7 +52,7 @@ class RegistrationHandler(BaseHandler): if urllib.quote(localpart.encode('utf-8')) != localpart: raise SynapseError( 400, - "User ID can only contain characters a-z, 0-9, or '-./'", + "User ID can only contain characters a-z, 0-9, or '_-./'", Codes.INVALID_USERNAME ) From 7065b75bfd52a87771ad36537a374826dc45496a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 15:13:44 +0000 Subject: [PATCH 149/294] Don't crash if a user has no push rule enabled entries --- synapse/push/bulk_push_rule_evaluator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index baa32a4e1..06250d2d9 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -50,6 +50,9 @@ def _get_rules(room_id, user_ids, store): # fetch disabled rules, but this won't account for any server default # rules the user has disabled, so we need to do this too. for uid in user_ids: + if uid not in rules_enabled_by_user: + continue + user_enabled_map = rules_enabled_by_user[uid] for rule in rules_by_user[uid]: From db6e26bb8c29556b43ca2e8c52510158a6f71204 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Jan 2016 16:03:55 +0000 Subject: [PATCH 150/294] Don't mutate cached values --- synapse/push/bulk_push_rule_evaluator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 06250d2d9..ec917c239 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -55,11 +55,15 @@ def _get_rules(room_id, user_ids, store): user_enabled_map = rules_enabled_by_user[uid] - for rule in rules_by_user[uid]: + for i, rule in enumerate(rules_by_user[uid]): rule_id = rule['rule_id'] if rule_id in user_enabled_map: - rule['enabled'] = user_enabled_map[rule_id] + if rule.get('enabled', True) != bool(user_enabled_map[rule_id]): + # Rules are cached across users. + rule = dict(rule) + rule['enabled'] = bool(user_enabled_map[rule_id]) + rules_by_user[uid][i] = rule defer.returnValue(rules_by_user) From b4a41aa542203c03bb8a6c93097b94bc5d167265 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 16:56:48 +0000 Subject: [PATCH 151/294] Don't add notifications to the table unless there's actually a 'notify' action --- synapse/push/bulk_push_rule_evaluator.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 06250d2d9..c4cc85eb4 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -135,8 +135,16 @@ class BulkPushRuleEvaluator: evaluator, rule['conditions'], uid, display_name, condition_cache ) if matches: + notify = False + actions = [] + for a in rule['actions']: + if a != 'dont_notify': + actions.append(a) + elif a == 'notify': + notify = True + actions = [x for x in rule['actions'] if x != 'dont_notify'] - if actions: + if actions and notify: actions_by_user[uid] = actions break defer.returnValue(actions_by_user) From 60965bd7e5a27d3952116c431f54786380643d05 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 17:21:15 +0000 Subject: [PATCH 152/294] Revert b4a41aa542203c03bb8a6c93097b94bc5d167265 as it's just broken. --- synapse/push/bulk_push_rule_evaluator.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index c4cc85eb4..06250d2d9 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -135,16 +135,8 @@ class BulkPushRuleEvaluator: evaluator, rule['conditions'], uid, display_name, condition_cache ) if matches: - notify = False - actions = [] - for a in rule['actions']: - if a != 'dont_notify': - actions.append(a) - elif a == 'notify': - notify = True - actions = [x for x in rule['actions'] if x != 'dont_notify'] - if actions and notify: + if actions: actions_by_user[uid] = actions break defer.returnValue(actions_by_user) From 3fe8c5673605074035f132185a143bcaded1a734 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Jan 2016 17:21:58 +0000 Subject: [PATCH 153/294] Better fix for actions with both dont_notify and tweaks --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 06250d2d9..5abab80e6 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -136,7 +136,7 @@ class BulkPushRuleEvaluator: ) if matches: actions = [x for x in rule['actions'] if x != 'dont_notify'] - if actions: + if actions and 'notify' in actions: actions_by_user[uid] = actions break defer.returnValue(actions_by_user) From 4021f95261ebdcca0ec2c3c91e8dd442a85c5ed4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 10:10:44 +0000 Subject: [PATCH 154/294] Move logic from rest/ to handlers/ --- synapse/api/filtering.py | 22 ++-- synapse/handlers/sync.py | 189 +++++++++++++++++++++------ synapse/rest/client/v2_alpha/sync.py | 57 ++++---- 3 files changed, 181 insertions(+), 87 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 5530b8c48..116060ee7 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -190,18 +190,16 @@ class Filter(object): Returns: bool: True if the event matches """ - if isinstance(event, dict): - return self.check_fields( - event.get("room_id", None), - event.get("sender", None), - event.get("type", None), - ) - else: - return self.check_fields( - getattr(event, "room_id", None), - getattr(event, "sender", None), - event.type, - ) + sender = event.get("sender", None) + if not sender: + # Presence events have their 'sender' in content.user_id + sender = event.get("conntent", {}).get("user_id", None) + + return self.check_fields( + event.get("room_id", None), + sender, + event.get("type", None), + ) def check_fields(self, room_id, sender, event_type): """Checks whether the filter matches the given event fields. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 53e1eb050..9b5b4d2c9 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -17,6 +17,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes +from synapse.api.filtering import DEFAULT_FILTER_COLLECTION from synapse.util import unwrapFirstError from twisted.internet import defer @@ -29,7 +30,7 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "filter", + "filter_collection", "is_guest", ]) @@ -129,6 +130,11 @@ class SyncHandler(BaseHandler): self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() + @defer.inlineCallbacks + def get_sync_for_user(self, sync_config, since_token=None, timeout=0, + filter_collection=DEFAULT_FILTER_COLLECTION): + pass + @defer.inlineCallbacks def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0, full_state=False): @@ -142,8 +148,9 @@ class SyncHandler(BaseHandler): if timeout == 0 or since_token is None or full_state: # we are going to return immediately, so don't bother calling # notifier.wait_for_events. - result = yield self.current_sync_for_user(sync_config, since_token, - full_state=full_state) + result = yield self.current_sync_for_user( + sync_config, since_token, full_state=full_state, + ) defer.returnValue(result) else: def current_sync_callback(before_token, after_token): @@ -151,7 +158,7 @@ class SyncHandler(BaseHandler): result = yield self.notifier.wait_for_events( sync_config.user.to_string(), timeout, current_sync_callback, - from_token=since_token + from_token=since_token, ) defer.returnValue(result) @@ -205,7 +212,7 @@ class SyncHandler(BaseHandler): ) membership_list = (Membership.INVITE, Membership.JOIN) - if sync_config.filter.include_leave: + if sync_config.filter_collection.include_leave: membership_list += (Membership.LEAVE, Membership.BAN) room_list = yield self.store.get_rooms_for_user_where_membership_is( @@ -266,9 +273,17 @@ class SyncHandler(BaseHandler): deferreds, consumeErrors=True ).addErrback(unwrapFirstError) + account_data_for_user = sync_config.filter_collection.filter_account_data( + self.account_data_for_user(account_data) + ) + + presence = sync_config.filter_collection.filter_presence( + presence + ) + defer.returnValue(SyncResult( presence=presence, - account_data=self.account_data_for_user(account_data), + account_data=account_data_for_user, joined=joined, invited=invited, archived=archived, @@ -302,14 +317,31 @@ class SyncHandler(BaseHandler): current_state = yield self.get_state_at(room_id, now_token) + current_state = { + (e.type, e.state_key): e + for e in sync_config.filter_collection.filter_room_state( + current_state.values() + ) + } + + account_data = self.account_data_for_room( + room_id, tags_by_room, account_data_by_room + ) + + account_data = sync_config.filter_collection.filter_room_account_data( + account_data + ) + + ephemeral = sync_config.filter_collection.filter_room_ephemeral( + ephemeral_by_room.get(room_id, []) + ) + defer.returnValue(JoinedSyncResult( room_id=room_id, timeline=batch, state=current_state, - ephemeral=ephemeral_by_room.get(room_id, []), - account_data=self.account_data_for_room( - room_id, tags_by_room, account_data_by_room - ), + ephemeral=ephemeral, + account_data=account_data, unread_notifications=unread_notifications, )) @@ -365,7 +397,7 @@ class SyncHandler(BaseHandler): typing, typing_key = yield typing_source.get_new_events( user=sync_config.user, from_key=typing_key, - limit=sync_config.filter.ephemeral_limit(), + limit=sync_config.filter_collection.ephemeral_limit(), room_ids=room_ids, is_guest=sync_config.is_guest, ) @@ -388,7 +420,7 @@ class SyncHandler(BaseHandler): receipts, receipt_key = yield receipt_source.get_new_events( user=sync_config.user, from_key=receipt_key, - limit=sync_config.filter.ephemeral_limit(), + limit=sync_config.filter_collection.ephemeral_limit(), room_ids=room_ids, is_guest=sync_config.is_guest, ) @@ -419,13 +451,26 @@ class SyncHandler(BaseHandler): leave_state = yield self.store.get_state_for_event(leave_event_id) + leave_state = { + (e.type, e.state_key): e + for e in sync_config.filter_collection.filter_room_state( + leave_state.values() + ) + } + + account_data = self.account_data_for_room( + room_id, tags_by_room, account_data_by_room + ) + + account_data = sync_config.filter_collection.filter_room_account_data( + account_data + ) + defer.returnValue(ArchivedSyncResult( room_id=room_id, timeline=batch, state=leave_state, - account_data=self.account_data_for_room( - room_id, tags_by_room, account_data_by_room - ), + account_data=account_data, )) @defer.inlineCallbacks @@ -444,7 +489,7 @@ class SyncHandler(BaseHandler): presence, presence_key = yield presence_source.get_new_events( user=sync_config.user, from_key=since_token.presence_key, - limit=sync_config.filter.presence_limit(), + limit=sync_config.filter_collection.presence_limit(), room_ids=room_ids, is_guest=sync_config.is_guest, ) @@ -473,7 +518,7 @@ class SyncHandler(BaseHandler): sync_config.user ) - timeline_limit = sync_config.filter.timeline_limit() + timeline_limit = sync_config.filter_collection.timeline_limit() room_events, _ = yield self.store.get_room_events_stream( sync_config.user.to_string(), @@ -538,6 +583,27 @@ class SyncHandler(BaseHandler): # the timeline is inherently limited if we've just joined limited = True + recents = sync_config.filter_collection.filter_room_timeline(recents) + + state = { + (e.type, e.state_key): e + for e in sync_config.filter_collection.filter_room_state( + state.values() + ) + } + + acc_data = self.account_data_for_room( + room_id, tags_by_room, account_data_by_room + ) + + acc_data = sync_config.filter_collection.filter_room_account_data( + acc_data + ) + + ephemeral = sync_config.filter_collection.filter_room_ephemeral( + ephemeral_by_room.get(room_id, []) + ) + room_sync = JoinedSyncResult( room_id=room_id, timeline=TimelineBatch( @@ -546,10 +612,8 @@ class SyncHandler(BaseHandler): limited=limited, ), state=state, - ephemeral=ephemeral_by_room.get(room_id, []), - account_data=self.account_data_for_room( - room_id, tags_by_room, account_data_by_room - ), + ephemeral=ephemeral, + account_data=acc_data, unread_notifications={}, ) logger.debug("Result for room %s: %r", room_id, room_sync) @@ -603,9 +667,17 @@ class SyncHandler(BaseHandler): for event in invite_events ] + account_data_for_user = sync_config.filter_collection.filter_account_data( + self.account_data_for_user(account_data) + ) + + presence = sync_config.filter_collection.filter_presence( + presence + ) + defer.returnValue(SyncResult( presence=presence, - account_data=self.account_data_for_user(account_data), + account_data=account_data_for_user, joined=joined, invited=invited, archived=archived, @@ -621,7 +693,7 @@ class SyncHandler(BaseHandler): limited = True recents = [] filtering_factor = 2 - timeline_limit = sync_config.filter.timeline_limit() + timeline_limit = sync_config.filter_collection.timeline_limit() load_limit = max(timeline_limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key @@ -634,9 +706,9 @@ class SyncHandler(BaseHandler): from_token=since_token.room_key if since_token else None, end_token=end_key, ) - (room_key, _) = keys + room_key, _ = keys end_key = "s" + room_key.split('-')[-1] - loaded_recents = sync_config.filter.filter_room_timeline(events) + loaded_recents = sync_config.filter_collection.filter_room_timeline(events) loaded_recents = yield self._filter_events_for_client( sync_config.user.to_string(), loaded_recents, @@ -684,21 +756,28 @@ class SyncHandler(BaseHandler): logger.debug("Recents %r", batch) - current_state = yield self.get_state_at(room_id, now_token) + if batch.limited: + current_state = yield self.get_state_at(room_id, now_token) - state_at_previous_sync = yield self.get_state_at( - room_id, stream_position=since_token - ) + state_at_previous_sync = yield self.get_state_at( + room_id, stream_position=since_token + ) - state = yield self.compute_state_delta( - since_token=since_token, - previous_state=state_at_previous_sync, - current_state=current_state, - ) + state = yield self.compute_state_delta( + since_token=since_token, + previous_state=state_at_previous_sync, + current_state=current_state, + ) + else: + state = { + (event.type, event.state_key): event + for event in batch.events if event.is_state() + } just_joined = yield self.check_joined_room(sync_config, state) if just_joined: state = yield self.get_state_at(room_id, now_token) + # batch.limited = True notifs = yield self.unread_notifs_for_room_id( room_id, sync_config, all_ephemeral_by_room @@ -711,14 +790,29 @@ class SyncHandler(BaseHandler): 1 for notif in notifs if _action_has_highlight(notif["actions"]) ]) + state = { + (e.type, e.state_key): e + for e in sync_config.filter_collection.filter_room_state(state.values()) + } + + account_data = self.account_data_for_room( + room_id, tags_by_room, account_data_by_room + ) + + account_data = sync_config.filter_collection.filter_room_account_data( + account_data + ) + + ephemeral = sync_config.filter_collection.filter_room_ephemeral( + ephemeral_by_room.get(room_id, []) + ) + room_sync = JoinedSyncResult( room_id=room_id, timeline=batch, state=state, - ephemeral=ephemeral_by_room.get(room_id, []), - account_data=self.account_data_for_room( - room_id, tags_by_room, account_data_by_room - ), + ephemeral=ephemeral, + account_data=account_data, unread_notifications=unread_notifications, ) @@ -765,13 +859,26 @@ class SyncHandler(BaseHandler): current_state=state_events_at_leave, ) + state_events_delta = { + (e.type, e.state_key): e + for e in sync_config.filter_collection.filter_room_state( + state_events_delta.values() + ) + } + + account_data = self.account_data_for_room( + leave_event.room_id, tags_by_room, account_data_by_room + ) + + account_data = sync_config.filter_collection.filter_room_account_data( + account_data + ) + room_sync = ArchivedSyncResult( room_id=leave_event.room_id, timeline=batch, state=state_events_delta, - account_data=self.account_data_for_room( - leave_event.room_id, tags_by_room, account_data_by_room - ), + account_data=account_data, ) logger.debug("Room sync: %r", room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index ab924ad9e..07b5b5dfd 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -130,7 +130,7 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - filter=filter, + filter_collection=filter, is_guest=requester.is_guest, ) @@ -154,23 +154,21 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() joined = self.encode_joined( - sync_result.joined, filter, time_now, requester.access_token_id + sync_result.joined, time_now, requester.access_token_id ) invited = self.encode_invited( - sync_result.invited, filter, time_now, requester.access_token_id + sync_result.invited, time_now, requester.access_token_id ) archived = self.encode_archived( - sync_result.archived, filter, time_now, requester.access_token_id + sync_result.archived, time_now, requester.access_token_id ) response_content = { - "account_data": self.encode_account_data( - sync_result.account_data, filter, time_now - ), + "account_data": {"events": sync_result.account_data}, "presence": self.encode_presence( - sync_result.presence, filter, time_now + sync_result.presence, time_now ), "rooms": { "join": joined, @@ -182,24 +180,20 @@ class SyncRestServlet(RestServlet): defer.returnValue((200, response_content)) - def encode_presence(self, events, filter, time_now): + def encode_presence(self, events, time_now): formatted = [] for event in events: event = copy.deepcopy(event) event['sender'] = event['content'].pop('user_id') formatted.append(event) - return {"events": filter.filter_presence(formatted)} + return {"events": formatted} - def encode_account_data(self, events, filter, time_now): - return {"events": filter.filter_account_data(events)} - - def encode_joined(self, rooms, filter, time_now, token_id): + def encode_joined(self, rooms, time_now, token_id): """ Encode the joined rooms in a sync result :param list[synapse.handlers.sync.JoinedSyncResult] rooms: list of sync results for rooms this user is joined to - :param FilterCollection filter: filters to apply to the results :param int time_now: current time - used as a baseline for age calculations :param int token_id: ID of the user's auth token - used for namespacing @@ -211,18 +205,17 @@ class SyncRestServlet(RestServlet): joined = {} for room in rooms: joined[room.room_id] = self.encode_room( - room, filter, time_now, token_id + room, time_now, token_id ) return joined - def encode_invited(self, rooms, filter, time_now, token_id): + def encode_invited(self, rooms, time_now, token_id): """ Encode the invited rooms in a sync result :param list[synapse.handlers.sync.InvitedSyncResult] rooms: list of sync results for rooms this user is joined to - :param FilterCollection filter: filters to apply to the results :param int time_now: current time - used as a baseline for age calculations :param int token_id: ID of the user's auth token - used for namespacing @@ -237,7 +230,9 @@ class SyncRestServlet(RestServlet): room.invite, time_now, token_id=token_id, event_format=format_event_for_client_v2_without_room_id, ) - invited_state = invite.get("unsigned", {}).pop("invite_room_state", []) + unsigned = dict(invite.get("unsigned", {})) + invite["unsigned"] = unsigned + invited_state = list(unsigned.pop("invite_room_state", [])) invited_state.append(invite) invited[room.room_id] = { "invite_state": {"events": invited_state} @@ -245,13 +240,12 @@ class SyncRestServlet(RestServlet): return invited - def encode_archived(self, rooms, filter, time_now, token_id): + def encode_archived(self, rooms, time_now, token_id): """ Encode the archived rooms in a sync result :param list[synapse.handlers.sync.ArchivedSyncResult] rooms: list of sync results for rooms this user is joined to - :param FilterCollection filter: filters to apply to the results :param int time_now: current time - used as a baseline for age calculations :param int token_id: ID of the user's auth token - used for namespacing @@ -263,17 +257,16 @@ class SyncRestServlet(RestServlet): joined = {} for room in rooms: joined[room.room_id] = self.encode_room( - room, filter, time_now, token_id, joined=False + room, time_now, token_id, joined=False ) return joined @staticmethod - def encode_room(room, filter, time_now, token_id, joined=True): + def encode_room(room, time_now, token_id, joined=True): """ :param JoinedSyncResult|ArchivedSyncResult room: sync result for a single room - :param FilterCollection filter: filters to apply to the results :param int time_now: current time - used as a baseline for age calculations :param int token_id: ID of the user's auth token - used for namespacing @@ -292,19 +285,17 @@ class SyncRestServlet(RestServlet): ) state_dict = room.state - timeline_events = filter.filter_room_timeline(room.timeline.events) + timeline_events = room.timeline.events state_dict = SyncRestServlet._rollback_state_for_timeline( state_dict, timeline_events) - state_events = filter.filter_room_state(state_dict.values()) + state_events = state_dict.values() serialized_state = [serialize(e) for e in state_events] serialized_timeline = [serialize(e) for e in timeline_events] - account_data = filter.filter_room_account_data( - room.account_data - ) + account_data = room.account_data result = { "timeline": { @@ -317,7 +308,7 @@ class SyncRestServlet(RestServlet): } if joined: - ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) + ephemeral_events = room.ephemeral result["ephemeral"] = {"events": ephemeral_events} result["unread_notifications"] = room.unread_notifications @@ -334,8 +325,6 @@ class SyncRestServlet(RestServlet): :param list[synapse.events.EventBase] timeline: the event timeline :return: updated state dictionary """ - logger.debug("Processing state dict %r; timeline %r", state, - [e.get_dict() for e in timeline]) result = state.copy() @@ -357,8 +346,8 @@ class SyncRestServlet(RestServlet): # the event graph, and the state is no longer valid. Really, # the event shouldn't be in the timeline. We're going to ignore # it for now, however. - logger.warn("Found state event %r in timeline which doesn't " - "match state dictionary", timeline_event) + logger.debug("Found state event %r in timeline which doesn't " + "match state dictionary", timeline_event) continue prev_event_id = timeline_event.unsigned.get("replaces_state", None) From 8c6012a4af4973b0a53af65a31cbdb92a3dec5a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 13:12:35 +0000 Subject: [PATCH 155/294] Fix tests --- synapse/api/filtering.py | 2 +- tests/api/test_filtering.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 116060ee7..6c13ada5d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -193,7 +193,7 @@ class Filter(object): sender = event.get("sender", None) if not sender: # Presence events have their 'sender' in content.user_id - sender = event.get("conntent", {}).get("user_id", None) + sender = event.get("content", {}).get("user_id", None) return self.check_fields( event.get("room_id", None), diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 16ee6bbe6..1a4e439d3 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -13,26 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple from tests import unittest from twisted.internet import defer -from mock import Mock, NonCallableMock +from mock import Mock from tests.utils import ( MockHttpResource, DeferredMockCallable, setup_test_homeserver ) from synapse.types import UserID -from synapse.api.filtering import FilterCollection, Filter +from synapse.api.filtering import Filter +from synapse.events import FrozenEvent user_localpart = "test_user" # MockEvent = namedtuple("MockEvent", "sender type room_id") def MockEvent(**kwargs): - ev = NonCallableMock(spec_set=kwargs.keys()) - ev.configure_mock(**kwargs) - return ev + return FrozenEvent(kwargs) class FilteringTestCase(unittest.TestCase): From ddd25def01e3909a34c52954100763bb2a91f648 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 13:36:02 +0000 Subject: [PATCH 156/294] Implement a _simple_select_many_batch --- synapse/storage/_base.py | 67 ++++++++++++++++++++++++++++++++++++ synapse/storage/presence.py | 33 +++++++++--------- synapse/storage/push_rule.py | 63 +++++++++++---------------------- 3 files changed, 104 insertions(+), 59 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 183a75238..897c5d8d7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -629,6 +629,73 @@ class SQLBaseStore(object): return self.cursor_to_dict(txn) + @defer.inlineCallbacks + def _simple_select_many_batch(self, table, column, iterable, retcols, + keyvalues={}, desc="_simple_select_many_batch", + batch_size=100): + """Executes a SELECT query on the named table, which may return zero or + more rows, returning the result as a list of dicts. + + Filters rows by if value of `column` is in `iterable`. + + Args: + txn : Transaction object + table : string giving the table name + column : column name to test for inclusion against `iterable` + iterable : list + keyvalues : dict of column names and values to select the rows with + retcols : list of strings giving the names of the columns to return + """ + results = [] + + chunks = [iterable[i:i+batch_size] for i in xrange(0, len(iterable), batch_size)] + for chunk in chunks: + rows = yield self.runInteraction( + desc, + self._simple_select_many_txn, + table, column, chunk, keyvalues, retcols + ) + + results.extend(rows) + + defer.returnValue(results) + + def _simple_select_many_txn(self, txn, table, column, iterable, keyvalues, retcols): + """Executes a SELECT query on the named table, which may return zero or + more rows, returning the result as a list of dicts. + + Filters rows by if value of `column` is in `iterable`. + + Args: + txn : Transaction object + table : string giving the table name + column : column name to test for inclusion against `iterable` + iterable : list + keyvalues : dict of column names and values to select the rows with + retcols : list of strings giving the names of the columns to return + """ + sql = "SELECT %s FROM %s" % (", ".join(retcols), table) + + clauses = [] + values = [] + clauses.append( + "%s IN (%s)" % (column, ",".join("?" for _ in iterable)) + ) + values.extend(iterable) + + for key, value in keyvalues.items(): + clauses.append("%s = ?" % (key,)) + values.append(value) + + if clauses: + sql = "%s WHERE %s" % ( + sql, + " AND ".join(clauses), + ) + + txn.execute(sql, values) + return self.cursor_to_dict(txn) + def _simple_update_one(self, table, keyvalues, updatevalues, desc="_simple_update_one"): """Executes an UPDATE query on the named table, setting new values for diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 1095d52ac..9b3aecaf8 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -48,24 +48,25 @@ class PresenceStore(SQLBaseStore): desc="get_presence_state", ) - @cachedList(get_presence_state.cache, list_name="user_localparts") + @cachedList(get_presence_state.cache, list_name="user_localparts", + inlineCallbacks=True) def get_presence_states(self, user_localparts): - def f(txn): - results = {} - for user_localpart in user_localparts: - res = self._simple_select_one_txn( - txn, - table="presence", - keyvalues={"user_id": user_localpart}, - retcols=["state", "status_msg", "mtime"], - allow_none=True, - ) - if res: - results[user_localpart] = res + rows = yield self._simple_select_many_batch( + table="presence", + column="user_id", + iterable=user_localparts, + retcols=("user_id", "state", "status_msg", "mtime",), + desc="get_presence_states", + ) - return results - - return self.runInteraction("get_presence_states", f) + defer.returnValue({ + row["user_id"]: { + "state": row["state"], + "status_msg": row["status_msg"], + "mtime": row["mtime"], + } + for row in rows + }) def set_presence_state(self, user_localpart, new_state): res = self._simple_update_one( diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 35ec7e8ce..1f51c90ee 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -65,32 +65,20 @@ class PushRuleStore(SQLBaseStore): if not user_ids: defer.returnValue({}) - batch_size = 100 - - def f(txn, user_ids_to_fetch): - sql = ( - "SELECT pr.*" - " FROM push_rules AS pr" - " LEFT JOIN push_rules_enable AS pre" - " ON pr.user_name = pre.user_name AND pr.rule_id = pre.rule_id" - " WHERE pr.user_name" - " IN (" + ",".join("?" for _ in user_ids_to_fetch) + ")" - " AND (pre.enabled IS NULL OR pre.enabled = 1)" - " ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC" - ) - txn.execute(sql, user_ids_to_fetch) - return self.cursor_to_dict(txn) - results = {} - chunks = [user_ids[i:i+batch_size] for i in xrange(0, len(user_ids), batch_size)] - for batch_user_ids in chunks: - rows = yield self.runInteraction( - "bulk_get_push_rules", f, batch_user_ids - ) + rows = yield self._simple_select_many_batch( + table="push_rules", + column="user_name", + iterable=user_ids, + retcols=("*",), + desc="bulk_get_push_rules", + ) - for row in rows: - results.setdefault(row['user_name'], []).append(row) + rows.sort(key=lambda e: (-e["priority_class"], -e["priority"])) + + for row in rows: + results.setdefault(row['user_name'], []).append(row) defer.returnValue(results) @defer.inlineCallbacks @@ -98,28 +86,17 @@ class PushRuleStore(SQLBaseStore): if not user_ids: defer.returnValue({}) - batch_size = 100 - - def f(txn, user_ids_to_fetch): - sql = ( - "SELECT user_name, rule_id, enabled" - " FROM push_rules_enable" - " WHERE user_name" - " IN (" + ",".join("?" for _ in user_ids_to_fetch) + ")" - ) - txn.execute(sql, user_ids_to_fetch) - return self.cursor_to_dict(txn) - results = {} - chunks = [user_ids[i:i+batch_size] for i in xrange(0, len(user_ids), batch_size)] - for batch_user_ids in chunks: - rows = yield self.runInteraction( - "bulk_get_push_rules_enabled", f, batch_user_ids - ) - - for row in rows: - results.setdefault(row['user_name'], {})[row['rule_id']] = row['enabled'] + rows = yield self._simple_select_many_batch( + table="push_rules_enable", + column="user_name", + iterable=user_ids, + retcols=("user_name", "rule_id", "enabled",), + desc="bulk_get_push_rules_enabled", + ) + for row in rows: + results.setdefault(row['user_name'], {})[row['rule_id']] = row['enabled'] defer.returnValue(results) @defer.inlineCallbacks From d59c58bc95e3ca6ea4ad9a7e62c6bc3f5f724978 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 13:13:46 +0000 Subject: [PATCH 157/294] Remove weird stuff --- synapse/handlers/sync.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9b5b4d2c9..84b059917 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -130,11 +130,6 @@ class SyncHandler(BaseHandler): self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() - @defer.inlineCallbacks - def get_sync_for_user(self, sync_config, since_token=None, timeout=0, - filter_collection=DEFAULT_FILTER_COLLECTION): - pass - @defer.inlineCallbacks def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0, full_state=False): @@ -777,7 +772,6 @@ class SyncHandler(BaseHandler): just_joined = yield self.check_joined_room(sync_config, state) if just_joined: state = yield self.get_state_at(room_id, now_token) - # batch.limited = True notifs = yield self.unread_notifs_for_room_id( room_id, sync_config, all_ephemeral_by_room From 53cb17366391a13e8d8297c9f4fb3f77fd6b85b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 13:53:05 +0000 Subject: [PATCH 158/294] Push: Use storage apis that are cached --- synapse/push/__init__.py | 30 +++++++++++++----------------- synapse/storage/roommember.py | 1 + 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index e6a28bd8c..9bc0b356f 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig from synapse.types import StreamToken -from synapse.api.constants import Membership import synapse.util.async import push_rule_evaluator as push_rule_evaluator @@ -296,31 +295,28 @@ class Pusher(object): @defer.inlineCallbacks def _get_badge_count(self): - room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=self.user_id, - membership_list=(Membership.INVITE, Membership.JOIN) - ) + invites, joins = yield defer.gatherResults([ + self.store.get_invites_for_user(self.user_id), + self.store.get_rooms_for_user(self.user_id), + ], consumeErrors=True) my_receipts_by_room = yield self.store.get_receipts_for_user( self.user_id, "m.read", ) - badge = 0 + badge = len(invites) - for r in room_list: - if r.membership == Membership.INVITE: - badge += 1 - else: - if r.room_id in my_receipts_by_room: - last_unread_event_id = my_receipts_by_room[r.room_id] + for r in joins: + if r.room_id in my_receipts_by_room: + last_unread_event_id = my_receipts_by_room[r.room_id] - notifs = yield ( - self.store.get_unread_event_push_actions_by_room_for_user( - r.room_id, self.user_id, last_unread_event_id - ) + notifs = yield ( + self.store.get_unread_event_push_actions_by_room_for_user( + r.room_id, self.user_id, last_unread_event_id ) - badge += len(notifs) + ) + badge += len(notifs) defer.returnValue(badge) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 68ac88905..edfecced0 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -110,6 +110,7 @@ class RoomMemberStore(SQLBaseStore): membership=membership, ).addCallback(self._get_events) + @cached() def get_invites_for_user(self, user_id): """ Get all the invite events for a user Args: From 86896408b0a3c0d0c82356c8cc0bdaf3fe236b45 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 15:30:32 +0000 Subject: [PATCH 159/294] Add index to event_push_actions --- synapse/storage/schema/delta/28/event_push_actions.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/schema/delta/28/event_push_actions.sql b/synapse/storage/schema/delta/28/event_push_actions.sql index bdf6ae3f2..4d519849d 100644 --- a/synapse/storage/schema/delta/28/event_push_actions.sql +++ b/synapse/storage/schema/delta/28/event_push_actions.sql @@ -24,3 +24,4 @@ CREATE TABLE IF NOT EXISTS event_push_actions( CREATE INDEX event_push_actions_room_id_event_id_user_id_profile_tag on event_push_actions(room_id, event_id, user_id, profile_tag); +CREATE INDEX event_push_actions_room_id_user_id on event_push_actions(room_id, user_id); From dc2647cd3d1e8891399f309344e375a25142e43e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 15:48:54 +0000 Subject: [PATCH 160/294] PEP8 --- synapse/handlers/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 84b059917..328c049b0 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -17,7 +17,6 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes -from synapse.api.filtering import DEFAULT_FILTER_COLLECTION from synapse.util import unwrapFirstError from twisted.internet import defer From 1ebf5e3d03a2a2ce9ff278b2eac07acc0f7cde66 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 15:53:36 +0000 Subject: [PATCH 161/294] Correct docstring --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 897c5d8d7..304ebdc82 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -639,7 +639,6 @@ class SQLBaseStore(object): Filters rows by if value of `column` is in `iterable`. Args: - txn : Transaction object table : string giving the table name column : column name to test for inclusion against `iterable` iterable : list From aea5da0ef6f4d3907ace2c1fdba743312118660c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Jan 2016 15:59:29 +0000 Subject: [PATCH 162/294] Guard against empty iterables --- synapse/storage/_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 304ebdc82..90d7aee94 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -647,6 +647,9 @@ class SQLBaseStore(object): """ results = [] + if not iterable: + defer.returnValue(results) + chunks = [iterable[i:i+batch_size] for i in xrange(0, len(iterable), batch_size)] for chunk in chunks: rows = yield self.runInteraction( @@ -673,6 +676,9 @@ class SQLBaseStore(object): keyvalues : dict of column names and values to select the rows with retcols : list of strings giving the names of the columns to return """ + if not iterable: + return [] + sql = "SELECT %s FROM %s" % (", ".join(retcols), table) clauses = [] From e18257f0e52cf587ab4d7bb8937dbcaefeb26e54 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 25 Jan 2016 16:51:56 +0000 Subject: [PATCH 163/294] Add missing yield in push_rules set enabled --- synapse/rest/client/v1/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2272d66dc..cb3ec2387 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -52,7 +52,7 @@ class PushRuleRestServlet(ClientV1RestServlet): content = _parse_json(request) if 'attr' in spec: - self.set_rule_attr(requester.user.to_string(), spec, content) + yield self.set_rule_attr(requester.user.to_string(), spec, content) defer.returnValue((200, {})) try: @@ -228,7 +228,7 @@ class PushRuleRestServlet(ClientV1RestServlet): # bools directly, so let's not break them. raise SynapseError(400, "Value for 'enabled' must be boolean") namespaced_rule_id = _namespaced_rule_id_from_spec(spec) - self.hs.get_datastore().set_push_rule_enabled( + return self.hs.get_datastore().set_push_rule_enabled( user_id, namespaced_rule_id, val ) else: From 766c24b2e6efbb5d250f5a36809caf3026140593 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 26 Jan 2016 10:21:41 +0000 Subject: [PATCH 164/294] Only notify for messages in one to one rooms, not every event Fixes the fact that candidate events and hangups generated notifications. --- synapse/push/baserules.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 186281dfa..0832c77cb 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -173,6 +173,12 @@ BASE_APPEND_UNDERRIDE_RULES = [ 'kind': 'room_member_count', 'is': '2', '_id': 'member_count', + }, + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.room.message', + '_id': '_message', } ], 'actions': [ From 9959d9ece84d85dae3ed06b22e3f234575b93fd1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Jan 2016 13:52:29 +0000 Subject: [PATCH 165/294] Remove redundated BaseHomeServer --- synapse/app/homeserver.py | 139 ++++------- synapse/federation/__init__.py | 9 +- synapse/federation/replication.py | 2 - synapse/federation/transport/__init__.py | 52 ---- synapse/federation/transport/client.py | 4 + synapse/federation/transport/server.py | 82 +++--- synapse/server.py | 106 ++++---- tests/federation/__init__.py | 0 tests/federation/test_federation.py | 303 ----------------------- tests/rest/client/v1/test_presence.py | 18 +- tests/test_types.py | 5 +- tests/utils.py | 18 ++ 12 files changed, 192 insertions(+), 546 deletions(-) delete mode 100644 tests/federation/__init__.py delete mode 100644 tests/federation/test_federation.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6928d9d3e..795c655ae 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -50,16 +50,14 @@ from twisted.cred import checkers, portal from twisted.internet import reactor, task, defer from twisted.application import service -from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper from twisted.web.static import File from twisted.web.server import Site, GzipEncoderFactory, Request -from synapse.http.server import JsonResource, RootRedirect +from synapse.http.server import RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource -from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, SERVER_KEY_PREFIX, MEDIA_PREFIX, STATIC_PREFIX, @@ -69,6 +67,7 @@ from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.metrics.resource import MetricsResource, METRICS_PREFIX +from synapse.federation.transport.server import TransportLayerServer from synapse import events @@ -95,80 +94,37 @@ def gz_wrap(r): return EncodingResourceWrapper(r, [GzipEncoderFactory()]) +def build_resource_for_web_client(hs): + webclient_path = hs.get_config().web_client_location + if not webclient_path: + try: + import syweb + except ImportError: + quit_with_error( + "Could not find a webclient.\n\n" + "Please either install the matrix-angular-sdk or configure\n" + "the location of the source to serve via the configuration\n" + "option `web_client_location`\n\n" + "To install the `matrix-angular-sdk` via pip, run:\n\n" + " pip install '%(dep)s'\n" + "\n" + "You can also disable hosting of the webclient via the\n" + "configuration option `web_client`\n" + % {"dep": DEPENDENCY_LINKS["matrix-angular-sdk"]} + ) + syweb_path = os.path.dirname(syweb.__file__) + webclient_path = os.path.join(syweb_path, "webclient") + # GZip is disabled here due to + # https://twistedmatrix.com/trac/ticket/7678 + # (It can stay enabled for the API resources: they call + # write() with the whole body and then finish() straight + # after and so do not trigger the bug. + # GzipFile was removed in commit 184ba09 + # return GzipFile(webclient_path) # TODO configurable? + return File(webclient_path) # TODO configurable? + + class SynapseHomeServer(HomeServer): - - def build_http_client(self): - return MatrixFederationHttpClient(self) - - def build_client_resource(self): - return ClientRestResource(self) - - def build_resource_for_federation(self): - return JsonResource(self) - - def build_resource_for_web_client(self): - webclient_path = self.get_config().web_client_location - if not webclient_path: - try: - import syweb - except ImportError: - quit_with_error( - "Could not find a webclient.\n\n" - "Please either install the matrix-angular-sdk or configure\n" - "the location of the source to serve via the configuration\n" - "option `web_client_location`\n\n" - "To install the `matrix-angular-sdk` via pip, run:\n\n" - " pip install '%(dep)s'\n" - "\n" - "You can also disable hosting of the webclient via the\n" - "configuration option `web_client`\n" - % {"dep": DEPENDENCY_LINKS["matrix-angular-sdk"]} - ) - syweb_path = os.path.dirname(syweb.__file__) - webclient_path = os.path.join(syweb_path, "webclient") - # GZip is disabled here due to - # https://twistedmatrix.com/trac/ticket/7678 - # (It can stay enabled for the API resources: they call - # write() with the whole body and then finish() straight - # after and so do not trigger the bug. - # GzipFile was removed in commit 184ba09 - # return GzipFile(webclient_path) # TODO configurable? - return File(webclient_path) # TODO configurable? - - def build_resource_for_static_content(self): - # This is old and should go away: not going to bother adding gzip - return File( - os.path.join(os.path.dirname(synapse.__file__), "static") - ) - - def build_resource_for_content_repo(self): - return ContentRepoResource( - self, self.config.uploads_path, self.auth, self.content_addr - ) - - def build_resource_for_media_repository(self): - return MediaRepositoryResource(self) - - def build_resource_for_server_key(self): - return LocalKey(self) - - def build_resource_for_server_key_v2(self): - return KeyApiV2Resource(self) - - def build_resource_for_metrics(self): - if self.get_config().enable_metrics: - return MetricsResource(self) - else: - return None - - def build_db_pool(self): - name = self.db_config["name"] - - return adbapi.ConnectionPool( - name, - **self.db_config.get("args", {}) - ) - def _listener_http(self, config, listener_config): port = listener_config["port"] bind_address = listener_config.get("bind_address", "") @@ -178,13 +134,11 @@ class SynapseHomeServer(HomeServer): if tls and config.no_tls: return - metrics_resource = self.get_resource_for_metrics() - resources = {} for res in listener_config["resources"]: for name in res["names"]: if name == "client": - client_resource = self.get_client_resource() + client_resource = ClientRestResource(self) if res["compress"]: client_resource = gz_wrap(client_resource) @@ -198,31 +152,35 @@ class SynapseHomeServer(HomeServer): if name == "federation": resources.update({ - FEDERATION_PREFIX: self.get_resource_for_federation(), + FEDERATION_PREFIX: TransportLayerServer(self), }) if name in ["static", "client"]: resources.update({ - STATIC_PREFIX: self.get_resource_for_static_content(), + STATIC_PREFIX: File( + os.path.join(os.path.dirname(synapse.__file__), "static") + ), }) if name in ["media", "federation", "client"]: resources.update({ - MEDIA_PREFIX: self.get_resource_for_media_repository(), - CONTENT_REPO_PREFIX: self.get_resource_for_content_repo(), + MEDIA_PREFIX: MediaRepositoryResource(self), + CONTENT_REPO_PREFIX: ContentRepoResource( + self, self.config.uploads_path, self.auth, self.content_addr + ), }) if name in ["keys", "federation"]: resources.update({ - SERVER_KEY_PREFIX: self.get_resource_for_server_key(), - SERVER_KEY_V2_PREFIX: self.get_resource_for_server_key_v2(), + SERVER_KEY_PREFIX: LocalKey(self), + SERVER_KEY_V2_PREFIX: KeyApiV2Resource(self), }) if name == "webclient": - resources[WEB_CLIENT_PREFIX] = self.get_resource_for_web_client() + resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) - if name == "metrics" and metrics_resource: - resources[METRICS_PREFIX] = metrics_resource + if name == "metrics" and self.get_config().enable_metrics: + resources[METRICS_PREFIX] = MetricsResource(self) root_resource = create_resource_tree(resources) if tls: @@ -675,7 +633,7 @@ def _resource_id(resource, path_seg): the mapping should looks like _resource_id(A,C) = B. Args: - resource (Resource): The *parent* Resource + resource (Resource): The *parent* Resourceb path_seg (str): The name of the child Resource to be attached. Returns: str: A unique string which can be a key to the child Resource. @@ -684,7 +642,7 @@ def _resource_id(resource, path_seg): def run(hs): - PROFILE_SYNAPSE = False + PROFILE_SYNAPSE = True if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile @@ -761,6 +719,7 @@ def run(hs): auto_close_fds=False, verbose=True, logger=logger, + chdir=os.path.dirname(os.path.abspath(__file__)), ) daemon.start() diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py index 0bfb79d09..979fdf243 100644 --- a/synapse/federation/__init__.py +++ b/synapse/federation/__init__.py @@ -17,15 +17,10 @@ """ from .replication import ReplicationLayer -from .transport import TransportLayer +from .transport.client import TransportLayerClient def initialize_http_replication(homeserver): - transport = TransportLayer( - homeserver, - homeserver.hostname, - server=homeserver.get_resource_for_federation(), - client=homeserver.get_http_client() - ) + transport = TransportLayerClient(homeserver) return ReplicationLayer(homeserver, transport) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 6e0be8ef1..3e062a5ea 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -54,8 +54,6 @@ class ReplicationLayer(FederationClient, FederationServer): self.keyring = hs.get_keyring() self.transport_layer = transport_layer - self.transport_layer.register_received_handler(self) - self.transport_layer.register_request_handler(self) self.federation_client = self diff --git a/synapse/federation/transport/__init__.py b/synapse/federation/transport/__init__.py index 155a7d587..d9fcc520a 100644 --- a/synapse/federation/transport/__init__.py +++ b/synapse/federation/transport/__init__.py @@ -20,55 +20,3 @@ By default this is done over HTTPS (and all home servers are required to support HTTPS), however individual pairings of servers may decide to communicate over a different (albeit still reliable) protocol. """ - -from .server import TransportLayerServer -from .client import TransportLayerClient - -from synapse.util.ratelimitutils import FederationRateLimiter - - -class TransportLayer(TransportLayerServer, TransportLayerClient): - """This is a basic implementation of the transport layer that translates - transactions and other requests to/from HTTP. - - Attributes: - server_name (str): Local home server host - - server (synapse.http.server.HttpServer): the http server to - register listeners on - - client (synapse.http.client.HttpClient): the http client used to - send requests - - request_handler (TransportRequestHandler): The handler to fire when we - receive requests for data. - - received_handler (TransportReceivedHandler): The handler to fire when - we receive data. - """ - - def __init__(self, homeserver, server_name, server, client): - """ - Args: - server_name (str): Local home server host - server (synapse.protocol.http.HttpServer): the http server to - register listeners on - client (synapse.protocol.http.HttpClient): the http client used to - send requests - """ - self.keyring = homeserver.get_keyring() - self.clock = homeserver.get_clock() - self.server_name = server_name - self.server = server - self.client = client - self.request_handler = None - self.received_handler = None - - self.ratelimiter = FederationRateLimiter( - self.clock, - window_size=homeserver.config.federation_rc_window_size, - sleep_limit=homeserver.config.federation_rc_sleep_limit, - sleep_msec=homeserver.config.federation_rc_sleep_delay, - reject_limit=homeserver.config.federation_rc_reject_limit, - concurrent_requests=homeserver.config.federation_rc_concurrent, - ) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 949d01dea..2b5d40ea7 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -28,6 +28,10 @@ logger = logging.getLogger(__name__) class TransportLayerClient(object): """Sends federation HTTP requests to other servers""" + def __init__(self, hs): + self.server_name = hs.hostname + self.client = hs.get_http_client() + @log_function def get_room_state(self, destination, room_id, event_id): """ Requests all state for a given room from the given server at the diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 8dca0a7f6..65e054f7d 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -17,7 +17,8 @@ from twisted.internet import defer from synapse.api.urls import FEDERATION_PREFIX as PREFIX from synapse.api.errors import Codes, SynapseError -from synapse.util.logutils import log_function +from synapse.http.server import JsonResource +from synapse.util.ratelimitutils import FederationRateLimiter import functools import logging @@ -28,9 +29,41 @@ import re logger = logging.getLogger(__name__) -class TransportLayerServer(object): +class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" + def __init__(self, hs): + self.hs = hs + self.clock = hs.get_clock() + + super(TransportLayerServer, self).__init__(hs) + + self.authenticator = Authenticator(hs) + self.ratelimiter = FederationRateLimiter( + self.clock, + window_size=hs.config.federation_rc_window_size, + sleep_limit=hs.config.federation_rc_sleep_limit, + sleep_msec=hs.config.federation_rc_sleep_delay, + reject_limit=hs.config.federation_rc_reject_limit, + concurrent_requests=hs.config.federation_rc_concurrent, + ) + + self.register_servlets() + + def register_servlets(self): + register_servlets( + self.hs, + resource=self, + ratelimiter=self.ratelimiter, + authenticator=self.authenticator, + ) + + +class Authenticator(object): + def __init__(self, hs): + self.keyring = hs.get_keyring() + self.server_name = hs.hostname + # A method just so we can pass 'self' as the authenticator to the Servlets @defer.inlineCallbacks def authenticate_request(self, request): @@ -98,37 +131,9 @@ class TransportLayerServer(object): defer.returnValue((origin, content)) - @log_function - def register_received_handler(self, handler): - """ Register a handler that will be fired when we receive data. - - Args: - handler (TransportReceivedHandler) - """ - FederationSendServlet( - handler, - authenticator=self, - ratelimiter=self.ratelimiter, - server_name=self.server_name, - ).register(self.server) - - @log_function - def register_request_handler(self, handler): - """ Register a handler that will be fired when we get asked for data. - - Args: - handler (TransportRequestHandler) - """ - for servletclass in SERVLET_CLASSES: - servletclass( - handler, - authenticator=self, - ratelimiter=self.ratelimiter, - ).register(self.server) - class BaseFederationServlet(object): - def __init__(self, handler, authenticator, ratelimiter): + def __init__(self, handler, authenticator, ratelimiter, server_name): self.handler = handler self.authenticator = authenticator self.ratelimiter = ratelimiter @@ -172,7 +177,9 @@ class FederationSendServlet(BaseFederationServlet): PATH = "/send/([^/]*)/" def __init__(self, handler, server_name, **kwargs): - super(FederationSendServlet, self).__init__(handler, **kwargs) + super(FederationSendServlet, self).__init__( + handler, server_name=server_name, **kwargs + ) self.server_name = server_name # This is when someone is trying to send us a bunch of data. @@ -432,6 +439,7 @@ class On3pidBindServlet(BaseFederationServlet): SERVLET_CLASSES = ( + FederationSendServlet, FederationPullServlet, FederationEventServlet, FederationStateServlet, @@ -451,3 +459,13 @@ SERVLET_CLASSES = ( FederationThirdPartyInviteExchangeServlet, On3pidBindServlet, ) + + +def register_servlets(hs, resource, authenticator, ratelimiter): + for servletclass in SERVLET_CLASSES: + servletclass( + handler=hs.get_replication_layer(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) diff --git a/synapse/server.py b/synapse/server.py index 4a5796b98..a59e46ca2 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -20,6 +20,8 @@ # Imports required for the default HomeServer() implementation from twisted.web.client import BrowserLikePolicyForHTTPS +from twisted.enterprise import adbapi + from synapse.federation import initialize_http_replication from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory from synapse.notifier import Notifier @@ -36,8 +38,10 @@ from synapse.push.pusherpool import PusherPool from synapse.events.builder import EventBuilderFactory from synapse.api.filtering import Filtering +from synapse.http.matrixfederationclient import MatrixFederationHttpClient -class BaseHomeServer(object): + +class HomeServer(object): """A basic homeserver object without lazy component builders. This will need all of the components it requires to either be passed as @@ -102,36 +106,6 @@ class BaseHomeServer(object): for depname in kwargs: setattr(self, depname, kwargs[depname]) - @classmethod - def _make_dependency_method(cls, depname): - def _get(self): - if hasattr(self, depname): - return getattr(self, depname) - - if hasattr(self, "build_%s" % (depname)): - # Prevent cyclic dependencies from deadlocking - if depname in self._building: - raise ValueError("Cyclic dependency while building %s" % ( - depname, - )) - self._building[depname] = 1 - - builder = getattr(self, "build_%s" % (depname)) - dep = builder() - setattr(self, depname, dep) - - del self._building[depname] - - return dep - - raise NotImplementedError( - "%s has no %s nor a builder for it" % ( - type(self).__name__, depname, - ) - ) - - setattr(BaseHomeServer, "get_%s" % (depname), _get) - def get_ip_from_request(self, request): # X-Forwarded-For is handled by our custom request type. return request.getClientIP() @@ -142,24 +116,6 @@ class BaseHomeServer(object): def is_mine_id(self, string): return string.split(":", 1)[1] == self.hostname -# Build magic accessors for every dependency -for depname in BaseHomeServer.DEPENDENCIES: - BaseHomeServer._make_dependency_method(depname) - - -class HomeServer(BaseHomeServer): - """A homeserver object that will construct most of its dependencies as - required. - - It still requires the following to be specified by the caller: - resource_for_client - resource_for_web_client - resource_for_federation - resource_for_content_repo - http_client - db_pool - """ - def build_clock(self): return Clock() @@ -224,3 +180,55 @@ class HomeServer(BaseHomeServer): def build_pusherpool(self): return PusherPool(self) + + def build_http_client(self): + return MatrixFederationHttpClient(self) + + def build_db_pool(self): + name = self.db_config["name"] + + return adbapi.ConnectionPool( + name, + **self.db_config.get("args", {}) + ) + + +def _make_dependency_method(depname): + def _get(hs): + try: + return getattr(hs, depname) + except AttributeError: + pass + + try: + builder = getattr(hs, "build_%s" % (depname)) + except AttributeError: + builder = None + + if builder: + # Prevent cyclic dependencies from deadlocking + if depname in hs._building: + raise ValueError("Cyclic dependency while building %s" % ( + depname, + )) + hs._building[depname] = 1 + + dep = builder() + setattr(hs, depname, dep) + + del hs._building[depname] + + return dep + + raise NotImplementedError( + "%s has no %s nor a builder for it" % ( + type(hs).__name__, depname, + ) + ) + + setattr(HomeServer, "get_%s" % (depname), _get) + + +# Build magic accessors for every dependency +for depname in HomeServer.DEPENDENCIES: + _make_dependency_method(depname) diff --git a/tests/federation/__init__.py b/tests/federation/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py deleted file mode 100644 index f2c2ee412..000000000 --- a/tests/federation/test_federation.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2014-2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# trial imports -from twisted.internet import defer -from tests import unittest - -# python imports -from mock import Mock, ANY - -from ..utils import MockHttpResource, MockClock, setup_test_homeserver - -from synapse.federation import initialize_http_replication -from synapse.events import FrozenEvent - - -def make_pdu(prev_pdus=[], **kwargs): - """Provide some default fields for making a PduTuple.""" - pdu_fields = { - "state_key": None, - "prev_events": prev_pdus, - } - pdu_fields.update(kwargs) - - return FrozenEvent(pdu_fields) - - -class FederationTestCase(unittest.TestCase): - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource() - self.mock_http_client = Mock(spec=[ - "get_json", - "put_json", - ]) - self.mock_persistence = Mock(spec=[ - "prep_send_transaction", - "delivered_txn", - "get_received_txn_response", - "set_received_txn_response", - "get_destination_retry_timings", - "get_auth_chain", - ]) - self.mock_persistence.get_received_txn_response.return_value = ( - defer.succeed(None) - ) - - retry_timings_res = { - "destination": "", - "retry_last_ts": 0, - "retry_interval": 0, - } - self.mock_persistence.get_destination_retry_timings.return_value = ( - defer.succeed(retry_timings_res) - ) - self.mock_persistence.get_auth_chain.return_value = [] - self.clock = MockClock() - hs = yield setup_test_homeserver( - resource_for_federation=self.mock_resource, - http_client=self.mock_http_client, - datastore=self.mock_persistence, - clock=self.clock, - keyring=Mock(), - ) - self.federation = initialize_http_replication(hs) - self.distributor = hs.get_distributor() - - @defer.inlineCallbacks - def test_get_state(self): - mock_handler = Mock(spec=[ - "get_state_for_pdu", - ]) - - self.federation.set_handler(mock_handler) - - mock_handler.get_state_for_pdu.return_value = defer.succeed([]) - - # Empty context initially - (code, response) = yield self.mock_resource.trigger( - "GET", - "/_matrix/federation/v1/state/my-context/", - None - ) - self.assertEquals(200, code) - self.assertFalse(response["pdus"]) - - # Now lets give the context some state - mock_handler.get_state_for_pdu.return_value = ( - defer.succeed([ - make_pdu( - event_id="the-pdu-id", - origin="red", - user_id="@a:red", - room_id="my-context", - type="m.topic", - origin_server_ts=123456789000, - depth=1, - content={"topic": "The topic"}, - state_key="", - power_level=1000, - prev_state="last-pdu-id", - ), - ]) - ) - - (code, response) = yield self.mock_resource.trigger( - "GET", - "/_matrix/federation/v1/state/my-context/", - None - ) - self.assertEquals(200, code) - self.assertEquals(1, len(response["pdus"])) - - @defer.inlineCallbacks - def test_get_pdu(self): - mock_handler = Mock(spec=[ - "get_persisted_pdu", - ]) - - self.federation.set_handler(mock_handler) - - mock_handler.get_persisted_pdu.return_value = ( - defer.succeed(None) - ) - - (code, response) = yield self.mock_resource.trigger( - "GET", - "/_matrix/federation/v1/event/abc123def456/", - None - ) - self.assertEquals(404, code) - - # Now insert such a PDU - mock_handler.get_persisted_pdu.return_value = ( - defer.succeed( - make_pdu( - event_id="abc123def456", - origin="red", - user_id="@a:red", - room_id="my-context", - type="m.text", - origin_server_ts=123456789001, - depth=1, - content={"text": "Here is the message"}, - ) - ) - ) - - (code, response) = yield self.mock_resource.trigger( - "GET", - "/_matrix/federation/v1/event/abc123def456/", - None - ) - self.assertEquals(200, code) - self.assertEquals(1, len(response["pdus"])) - self.assertEquals("m.text", response["pdus"][0]["type"]) - - @defer.inlineCallbacks - def test_send_pdu(self): - self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") - ) - - pdu = make_pdu( - event_id="abc123def456", - origin="red", - user_id="@a:red", - room_id="my-context", - type="m.text", - origin_server_ts=123456789001, - depth=1, - content={"text": "Here is the message"}, - ) - - yield self.federation.send_pdu(pdu, ["remote"]) - - self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin_server_ts": 1000000, - "origin": "test", - "pdus": [ - pdu.get_pdu_json(), - ], - 'pdu_failures': [], - }, - json_data_callback=ANY, - long_retries=True, - ) - - @defer.inlineCallbacks - def test_send_edu(self): - self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") - ) - - yield self.federation.send_edu( - destination="remote", - edu_type="m.test", - content={"testing": "content here"}, - ) - - # MockClock ensures we can guess these timestamps - self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin": "test", - "origin_server_ts": 1000000, - "pdus": [], - "edus": [ - { - "edu_type": "m.test", - "content": {"testing": "content here"}, - } - ], - 'pdu_failures': [], - }, - json_data_callback=ANY, - long_retries=True, - ) - - @defer.inlineCallbacks - def test_recv_edu(self): - recv_observer = Mock() - recv_observer.return_value = defer.succeed(()) - - self.federation.register_edu_handler("m.test", recv_observer) - - yield self.mock_resource.trigger( - "PUT", - "/_matrix/federation/v1/send/1001000/", - """{ - "origin": "remote", - "origin_server_ts": 1001000, - "pdus": [], - "edus": [ - { - "origin": "remote", - "destination": "test", - "edu_type": "m.test", - "content": {"testing": "reply here"} - } - ] - }""" - ) - - recv_observer.assert_called_with( - "remote", {"testing": "reply here"} - ) - - @defer.inlineCallbacks - def test_send_query(self): - self.mock_http_client.get_json.return_value = defer.succeed( - {"your": "response"} - ) - - response = yield self.federation.make_query( - destination="remote", - query_type="a-question", - args={"one": "1", "two": "2"}, - ) - - self.assertEquals({"your": "response"}, response) - - self.mock_http_client.get_json.assert_called_with( - destination="remote", - path="/_matrix/federation/v1/query/a-question", - args={"one": "1", "two": "2"}, - retry_on_dns_fail=True, - ) - - @defer.inlineCallbacks - def test_recv_query(self): - recv_handler = Mock() - recv_handler.return_value = defer.succeed({"another": "response"}) - - self.federation.register_query_handler("a-question", recv_handler) - - code, response = yield self.mock_resource.trigger( - "GET", - "/_matrix/federation/v1/query/a-question?three=3&four=4", - None - ) - - self.assertEquals(200, code) - self.assertEquals({"another": "response"}, response) - - recv_handler.assert_called_with( - {"three": "3", "four": "4"} - ) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 90b911f87..8d7cfd79a 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -280,6 +280,15 @@ class PresenceEventStreamTestCase(unittest.TestCase): } EventSources.SOURCE_TYPES["presence"] = PresenceEventSource + clock = Mock(spec=[ + "call_later", + "cancel_call_later", + "time_msec", + "looping_call", + ]) + + clock.time_msec.return_value = 1000000 + hs = yield setup_test_homeserver( http_client=None, resource_for_client=self.mock_resource, @@ -289,16 +298,9 @@ class PresenceEventStreamTestCase(unittest.TestCase): "get_presence_list", "get_rooms_for_user", ]), - clock=Mock(spec=[ - "call_later", - "cancel_call_later", - "time_msec", - "looping_call", - ]), + clock=clock, ) - hs.get_clock().time_msec.return_value = 1000000 - def _get_user_by_req(req=None, allow_guest=False): return Requester(UserID.from_string(myid), "", False) diff --git a/tests/test_types.py b/tests/test_types.py index b9534329e..24d61dbe5 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -16,10 +16,10 @@ from tests import unittest from synapse.api.errors import SynapseError -from synapse.server import BaseHomeServer +from synapse.server import HomeServer from synapse.types import UserID, RoomAlias -mock_homeserver = BaseHomeServer(hostname="my.domain") +mock_homeserver = HomeServer(hostname="my.domain") class UserIDTestCase(unittest.TestCase): @@ -34,7 +34,6 @@ class UserIDTestCase(unittest.TestCase): with self.assertRaises(SynapseError): UserID.from_string("") - def test_build(self): user = UserID("5678efgh", "my.domain") diff --git a/tests/utils.py b/tests/utils.py index 358b5b72b..d75d492cb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,6 +19,8 @@ from synapse.api.constants import EventTypes from synapse.storage.prepare_database import prepare_database from synapse.storage.engines import create_engine from synapse.server import HomeServer +from synapse.federation.transport import server +from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.logcontext import LoggingContext @@ -80,6 +82,22 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): hs.build_handlers = swap_out_hash_for_testing(hs.build_handlers) + fed = kargs.get("resource_for_federation", None) + if fed: + server.register_servlets( + hs, + resource=fed, + authenticator=server.Authenticator(hs), + ratelimiter=FederationRateLimiter( + hs.get_clock(), + window_size=hs.config.federation_rc_window_size, + sleep_limit=hs.config.federation_rc_sleep_limit, + sleep_msec=hs.config.federation_rc_sleep_delay, + reject_limit=hs.config.federation_rc_reject_limit, + concurrent_requests=hs.config.federation_rc_concurrent + ), + ) + defer.returnValue(hs) From 87f9477b105b4e8216d1df186492ec6d9872967f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Jan 2016 15:51:06 +0000 Subject: [PATCH 166/294] Add a Homeserver.setup method. This is for setting up dependencies that require work on startup. This is useful for the DataStore that wants to read a bunch from the database before initiliazing. --- synapse/app/homeserver.py | 33 +++++++++++------- synapse/server.py | 32 +++++++++-------- synapse/storage/__init__.py | 45 +++++++++++++++++++++--- synapse/storage/_base.py | 49 +++++++++++++-------------- synapse/storage/events.py | 14 +++----- synapse/storage/receipts.py | 8 +++-- synapse/storage/stream.py | 13 ------- synapse/storage/tags.py | 7 ---- synapse/storage/util/id_generators.py | 36 +++++--------------- 9 files changed, 121 insertions(+), 116 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 795c655ae..fb76be58a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -254,6 +254,17 @@ class SynapseHomeServer(HomeServer): except IncorrectDatabaseSetup as e: quit_with_error(e.message) + def get_db_conn(self): + db_conn = self.database_engine.module.connect( + **{ + k: v for k, v in self.db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) + + self.database_engine.on_new_connection(db_conn) + return db_conn + def quit_with_error(error_string): message_lines = error_string.split("\n") @@ -390,13 +401,7 @@ def setup(config_options): logger.info("Preparing database: %s...", config.database_config['name']) try: - db_conn = database_engine.module.connect( - **{ - k: v for k, v in config.database_config.get("args", {}).items() - if not k.startswith("cp_") - } - ) - + db_conn = hs.get_db_conn() database_engine.prepare_database(db_conn) hs.run_startup_checks(db_conn, database_engine) @@ -411,13 +416,17 @@ def setup(config_options): logger.info("Database prepared in %s.", config.database_config['name']) + hs.setup() hs.start_listening() - hs.get_pusherpool().start() - hs.get_state_handler().start_caching() - hs.get_datastore().start_profiling() - hs.get_datastore().start_doing_background_updates() - hs.get_replication_layer().start_get_pdu_cache() + def start(): + hs.get_pusherpool().start() + hs.get_state_handler().start_caching() + hs.get_datastore().start_profiling() + hs.get_datastore().start_doing_background_updates() + hs.get_replication_layer().start_get_pdu_cache() + + reactor.callWhenRunning(start) return hs diff --git a/synapse/server.py b/synapse/server.py index a59e46ca2..006e91b37 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -21,6 +21,7 @@ # Imports required for the default HomeServer() implementation from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.enterprise import adbapi +from twisted.internet import defer from synapse.federation import initialize_http_replication from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory @@ -28,7 +29,7 @@ from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler -from synapse.storage import DataStore +from synapse.storage import get_datastore from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.streams.events import EventSources @@ -40,6 +41,11 @@ from synapse.api.filtering import Filtering from synapse.http.matrixfederationclient import MatrixFederationHttpClient +import logging + + +logger = logging.getLogger(__name__) + class HomeServer(object): """A basic homeserver object without lazy component builders. @@ -102,10 +108,19 @@ class HomeServer(object): self.hostname = hostname self._building = {} + self.clock = Clock() + self.distributor = Distributor() + self.ratelimiter = Ratelimiter() + # Other kwargs are explicit dependencies for depname in kwargs: setattr(self, depname, kwargs[depname]) + def setup(self): + logger.info("Setting up.") + self.datastore = get_datastore(self) + logger.info("Finished setting up.") + def get_ip_from_request(self, request): # X-Forwarded-For is handled by our custom request type. return request.getClientIP() @@ -116,15 +131,9 @@ class HomeServer(object): def is_mine_id(self, string): return string.split(":", 1)[1] == self.hostname - def build_clock(self): - return Clock() - def build_replication_layer(self): return initialize_http_replication(self) - def build_datastore(self): - return DataStore(self) - def build_handlers(self): return Handlers(self) @@ -135,10 +144,9 @@ class HomeServer(object): return Auth(self) def build_http_client_context_factory(self): - config = self.get_config() return ( InsecureInterceptableContextFactory() - if config.use_insecure_ssl_client_just_for_testing_do_not_use + if self.config.use_insecure_ssl_client_just_for_testing_do_not_use else BrowserLikePolicyForHTTPS() ) @@ -157,15 +165,9 @@ class HomeServer(object): def build_state_handler(self): return StateHandler(self) - def build_distributor(self): - return Distributor() - def build_event_sources(self): return EventSources(self) - def build_ratelimiter(self): - return Ratelimiter() - def build_keyring(self): return Keyring(self) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7a3f6c466..c8cab45f7 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -46,6 +46,9 @@ from .tags import TagsStore from .account_data import AccountDataStore +from util.id_generators import IdGenerator, StreamIdGenerator + + import logging @@ -58,6 +61,22 @@ logger = logging.getLogger(__name__) LAST_SEEN_GRANULARITY = 120*1000 +def get_datastore(hs): + logger.info("getting called!") + + conn = hs.get_db_conn() + try: + cur = conn.cursor() + cur.execute("SELECT MIN(stream_ordering) FROM events",) + rows = cur.fetchall() + min_token = rows[0][0] if rows and rows[0] and rows[0][0] else -1 + min_token = min(min_token, -1) + + return DataStore(conn, hs, min_token) + finally: + conn.close() + + class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, PresenceStore, TransactionStore, @@ -79,18 +98,36 @@ class DataStore(RoomMemberStore, RoomStore, EventPushActionsStore ): - def __init__(self, hs): - super(DataStore, self).__init__(hs) + def __init__(self, db_conn, hs, min_stream_token): self.hs = hs - self.min_token_deferred = self._get_min_token() - self.min_token = None + self.min_stream_token = min_stream_token self.client_ip_last_seen = Cache( name="client_ip_last_seen", keylen=4, ) + self._stream_id_gen = StreamIdGenerator( + db_conn, "events", "stream_ordering" + ) + self._receipts_id_gen = StreamIdGenerator( + db_conn, "receipts_linearized", "stream_id" + ) + self._account_data_id_gen = StreamIdGenerator( + db_conn, "account_data_max_stream_id", "stream_id" + ) + + self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) + self._state_groups_id_gen = IdGenerator("state_groups", "id", self) + self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) + self._refresh_tokens_id_gen = IdGenerator("refresh_tokens", "id", self) + self._pushers_id_gen = IdGenerator("pushers", "id", self) + self._push_rule_id_gen = IdGenerator("push_rules", "id", self) + self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) + + super(DataStore, self).__init__(hs) + @defer.inlineCallbacks def insert_client_ip(self, user, access_token, ip, user_agent): now = int(self._clock.time_msec()) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 90d7aee94..5e7732054 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -15,13 +15,11 @@ import logging from synapse.api.errors import StoreError -from synapse.util.logutils import log_function from synapse.util.logcontext import preserve_context_over_fn, LoggingContext from synapse.util.caches.dictionary_cache import DictionaryCache from synapse.util.caches.descriptors import Cache import synapse.metrics -from util.id_generators import IdGenerator, StreamIdGenerator from twisted.internet import defer @@ -175,16 +173,6 @@ class SQLBaseStore(object): self.database_engine = hs.database_engine - self._stream_id_gen = StreamIdGenerator("events", "stream_ordering") - self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) - self._state_groups_id_gen = IdGenerator("state_groups", "id", self) - self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) - self._refresh_tokens_id_gen = IdGenerator("refresh_tokens", "id", self) - self._pushers_id_gen = IdGenerator("pushers", "id", self) - self._push_rule_id_gen = IdGenerator("push_rules", "id", self) - self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) - self._receipts_id_gen = StreamIdGenerator("receipts_linearized", "stream_id") - def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -345,7 +333,8 @@ class SQLBaseStore(object): defer.returnValue(result) - def cursor_to_dict(self, cursor): + @staticmethod + def cursor_to_dict(cursor): """Converts a SQL cursor into an list of dicts. Args: @@ -402,8 +391,8 @@ class SQLBaseStore(object): if not or_ignore: raise - @log_function - def _simple_insert_txn(self, txn, table, values): + @staticmethod + def _simple_insert_txn(txn, table, values): keys, vals = zip(*values.items()) sql = "INSERT INTO %s (%s) VALUES(%s)" % ( @@ -414,7 +403,8 @@ class SQLBaseStore(object): txn.execute(sql, vals) - def _simple_insert_many_txn(self, txn, table, values): + @staticmethod + def _simple_insert_many_txn(txn, table, values): if not values: return @@ -537,9 +527,10 @@ class SQLBaseStore(object): table, keyvalues, retcol, allow_none=allow_none, ) - def _simple_select_one_onecol_txn(self, txn, table, keyvalues, retcol, + @classmethod + def _simple_select_one_onecol_txn(cls, txn, table, keyvalues, retcol, allow_none=False): - ret = self._simple_select_onecol_txn( + ret = cls._simple_select_onecol_txn( txn, table=table, keyvalues=keyvalues, @@ -554,7 +545,8 @@ class SQLBaseStore(object): else: raise StoreError(404, "No row found") - def _simple_select_onecol_txn(self, txn, table, keyvalues, retcol): + @staticmethod + def _simple_select_onecol_txn(txn, table, keyvalues, retcol): sql = ( "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" ) % { @@ -603,7 +595,8 @@ class SQLBaseStore(object): table, keyvalues, retcols ) - def _simple_select_list_txn(self, txn, table, keyvalues, retcols): + @classmethod + def _simple_select_list_txn(cls, txn, table, keyvalues, retcols): """Executes a SELECT query on the named table, which may return zero or more rows, returning the result as a list of dicts. @@ -627,7 +620,7 @@ class SQLBaseStore(object): ) txn.execute(sql) - return self.cursor_to_dict(txn) + return cls.cursor_to_dict(txn) @defer.inlineCallbacks def _simple_select_many_batch(self, table, column, iterable, retcols, @@ -662,7 +655,8 @@ class SQLBaseStore(object): defer.returnValue(results) - def _simple_select_many_txn(self, txn, table, column, iterable, keyvalues, retcols): + @classmethod + def _simple_select_many_txn(cls, txn, table, column, iterable, keyvalues, retcols): """Executes a SELECT query on the named table, which may return zero or more rows, returning the result as a list of dicts. @@ -699,7 +693,7 @@ class SQLBaseStore(object): ) txn.execute(sql, values) - return self.cursor_to_dict(txn) + return cls.cursor_to_dict(txn) def _simple_update_one(self, table, keyvalues, updatevalues, desc="_simple_update_one"): @@ -726,7 +720,8 @@ class SQLBaseStore(object): table, keyvalues, updatevalues, ) - def _simple_update_one_txn(self, txn, table, keyvalues, updatevalues): + @staticmethod + def _simple_update_one_txn(txn, table, keyvalues, updatevalues): update_sql = "UPDATE %s SET %s WHERE %s" % ( table, ", ".join("%s = ?" % (k,) for k in updatevalues), @@ -743,7 +738,8 @@ class SQLBaseStore(object): if txn.rowcount > 1: raise StoreError(500, "More than one row matched") - def _simple_select_one_txn(self, txn, table, keyvalues, retcols, + @staticmethod + def _simple_select_one_txn(txn, table, keyvalues, retcols, allow_none=False): select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), @@ -784,7 +780,8 @@ class SQLBaseStore(object): raise StoreError(500, "more than one row matched") return self.runInteraction(desc, func) - def _simple_delete_txn(self, txn, table, keyvalues): + @staticmethod + def _simple_delete_txn(txn, table, keyvalues): sql = "DELETE FROM %s WHERE %s" % ( table, " AND ".join("%s = ?" % (k, ) for k in keyvalues) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index ba368a3ec..298cb9bad 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -66,11 +66,9 @@ class EventsStore(SQLBaseStore): return if backfilled: - if not self.min_token_deferred.called: - yield self.min_token_deferred - start = self.min_token - 1 - self.min_token -= len(events_and_contexts) + 1 - stream_orderings = range(start, self.min_token, -1) + start = self.min_stream_token - 1 + self.min_stream_token -= len(events_and_contexts) + 1 + stream_orderings = range(start, self.min_stream_token, -1) @contextmanager def stream_ordering_manager(): @@ -107,10 +105,8 @@ class EventsStore(SQLBaseStore): is_new_state=True, current_state=None): stream_ordering = None if backfilled: - if not self.min_token_deferred.called: - yield self.min_token_deferred - self.min_token -= 1 - stream_ordering = self.min_token + self.min_stream_token -= 1 + stream_ordering = self.min_stream_token if stream_ordering is None: stream_ordering_manager = yield self._stream_id_gen.get_next(self) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index c4232bdc6..c0593e23e 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -31,7 +31,9 @@ class ReceiptsStore(SQLBaseStore): def __init__(self, hs): super(ReceiptsStore, self).__init__(hs) - self._receipts_stream_cache = _RoomStreamChangeCache() + self._receipts_stream_cache = _RoomStreamChangeCache( + self._receipts_id_gen.get_max_token(None) + ) @cached(num_args=2) def get_receipts_for_room(self, room_id, receipt_type): @@ -377,11 +379,11 @@ class _RoomStreamChangeCache(object): may have changed since that key. If the key is too old then the cache will simply return all rooms. """ - def __init__(self, size_of_cache=10000): + def __init__(self, current_key, size_of_cache=10000): self._size_of_cache = size_of_cache self._room_to_key = {} self._cache = sorteddict() - self._earliest_key = None + self._earliest_key = current_key self.name = "ReceiptsRoomChangeCache" caches_by_name[self.name] = self._cache diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 02b1913e2..e31bad258 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -444,19 +444,6 @@ class StreamStore(SQLBaseStore): rows = txn.fetchall() return rows[0][0] if rows else 0 - @defer.inlineCallbacks - def _get_min_token(self): - row = yield self._execute( - "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" - ) - - self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 - self.min_token = min(self.min_token, -1) - - logger.debug("min_token is: %s", self.min_token) - - defer.returnValue(self.min_token) - @staticmethod def _set_before_and_after(events, rows): for event, row in zip(events, rows): diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index ed9c91e5e..4c39e07cb 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -16,7 +16,6 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached from twisted.internet import defer -from .util.id_generators import StreamIdGenerator import ujson as json import logging @@ -25,12 +24,6 @@ logger = logging.getLogger(__name__) class TagsStore(SQLBaseStore): - def __init__(self, hs): - super(TagsStore, self).__init__(hs) - - self._account_data_id_gen = StreamIdGenerator( - "account_data_max_stream_id", "stream_id" - ) def get_max_account_data_stream_id(self): """Get the current max stream id for the private user data stream diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index f58bf7fd2..5c522f4ab 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -72,28 +72,24 @@ class StreamIdGenerator(object): with stream_id_gen.get_next_txn(txn) as stream_id: # ... persist event ... """ - def __init__(self, table, column): + def __init__(self, db_conn, table, column): self.table = table self.column = column self._lock = threading.Lock() - self._current_max = None + cur = db_conn.cursor() + self._current_max = self._get_or_compute_current_max(cur) + cur.close() + self._unfinished_ids = deque() - @defer.inlineCallbacks def get_next(self, store): """ Usage: with yield stream_id_gen.get_next as stream_id: # ... persist event ... """ - if not self._current_max: - yield store.runInteraction( - "_compute_current_max", - self._get_or_compute_current_max, - ) - with self._lock: self._current_max += 1 next_id = self._current_max @@ -108,21 +104,14 @@ class StreamIdGenerator(object): with self._lock: self._unfinished_ids.remove(next_id) - defer.returnValue(manager()) + return manager() - @defer.inlineCallbacks def get_next_mult(self, store, n): """ Usage: with yield stream_id_gen.get_next(store, n) as stream_ids: # ... persist events ... """ - if not self._current_max: - yield store.runInteraction( - "_compute_current_max", - self._get_or_compute_current_max, - ) - with self._lock: next_ids = range(self._current_max + 1, self._current_max + n + 1) self._current_max += n @@ -139,24 +128,17 @@ class StreamIdGenerator(object): for next_id in next_ids: self._unfinished_ids.remove(next_id) - defer.returnValue(manager()) + return manager() - @defer.inlineCallbacks def get_max_token(self, store): """Returns the maximum stream id such that all stream ids less than or equal to it have been successfully persisted. """ - if not self._current_max: - yield store.runInteraction( - "_compute_current_max", - self._get_or_compute_current_max, - ) - with self._lock: if self._unfinished_ids: - defer.returnValue(self._unfinished_ids[0] - 1) + return self._unfinished_ids[0] - 1 - defer.returnValue(self._current_max) + return self._current_max def _get_or_compute_current_max(self, txn): with self._lock: From 9fda8b519358a4bf2702b24a45a7744b399ef127 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Jan 2016 18:27:23 +0000 Subject: [PATCH 167/294] Don't turn on profiling --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 795c655ae..f881a8d26 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -642,7 +642,7 @@ def _resource_id(resource, path_seg): def run(hs): - PROFILE_SYNAPSE = True + PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile From 8c94833b72f27d18a57e424a0f4f0c823c3a0aa1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 27 Jan 2016 10:24:20 +0000 Subject: [PATCH 168/294] Fix adding push rules relative to other rules --- synapse/rest/client/v1/push_rule.py | 15 ++++++++++----- synapse/storage/push_rule.py | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index cb3ec2387..96633a176 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -66,11 +66,12 @@ class PushRuleRestServlet(ClientV1RestServlet): raise SynapseError(400, e.message) before = request.args.get("before", None) - if before and len(before): - before = before[0] + if before: + before = _namespaced_rule_id(spec, before[0]) + after = request.args.get("after", None) - if after and len(after): - after = after[0] + if after: + after = _namespaced_rule_id(spec, after[0]) try: yield self.hs.get_datastore().add_push_rule( @@ -452,11 +453,15 @@ def _strip_device_condition(rule): def _namespaced_rule_id_from_spec(spec): + return _namespaced_rule_id(spec, spec['rule_id']) + + +def _namespaced_rule_id(spec, rule_id): if spec['scope'] == 'global': scope = 'global' else: scope = 'device/%s' % (spec['profile_tag']) - return "%s/%s/%s" % (scope, spec['template'], spec['rule_id']) + return "%s/%s/%s" % (scope, spec['template'], rule_id) def _rule_id_from_namespaced(in_rule_id): diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 1f51c90ee..f9a48171b 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -130,7 +130,8 @@ class PushRuleStore(SQLBaseStore): def _add_push_rule_relative_txn(self, txn, user_id, **kwargs): after = kwargs.pop("after", None) - relative_to_rule = kwargs.pop("before", after) + before = kwargs.pop("before", None) + relative_to_rule = before or after res = self._simple_select_one_txn( txn, From 03f4569dc3ee6379778633f8c295edde9f2a212c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Jan 2016 10:35:30 +0000 Subject: [PATCH 169/294] Bump version and changelog --- CHANGES.rst | 45 +++++++++++++++++++++++++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cb317c6a8..dd4863c3e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,48 @@ +Changes in synapse v0.12.1-rc1 (2016-01-xx) +=========================================== + +Features: + +* Add unread notification counts in ``/sync`` (PR #456) +* Add support for inviting 3pids in ``/createRoom`` (PR #460) +* Add ability for guest accounts to upgrade (PR #462) +* Add ``/versions`` API (PR #468) +* Add ``event`` to ``/context`` API (PR #492) +* Add specific error code for invalid user names in ``/register`` (PR #499) +* Add support for push badge counts (PR #507) +* Add support for non-guest users to peek in rooms using ``/events`` (PR #510) + +Changes: + +* Change ``/sync`` so that guest users only get rooms they've joined (PR #469) +* Change to require unbanning before other membership changes (PR #501) +* Change default push rules to notify for all messages (PR #486) +* Change default push rules to not notify on membership changes (PR #514) +* Change default push rules to only notify messages in one to one rooms + (PR #529) +* Change ``/sync`` to reject requests with a ``from`` query param (PR #512) +* Change server manhole to use SSH rather than telnet (PR #473) +* Change server to require AS users to be registered before use (PR #487) +* Change server not to start when ASes are invalidly configured (PR #494) +* Change server to require ID and ``as_token`` to be unique for AS's (PR #496) +* Change maximum pagination limit to 1000 (PR #497) + +Bug fixes: + +* Fix bug where ``/sync`` didn't return when something under the leave key + changed (PR #461) +* Fix bug where we returned smaller rather than larger than requested + thumbnails when ``method=crop`` (PR #464) +* Fix thumbnails API to only return cropped thumbnails when asking for a + cropped thumbnail (PR #475) +* Fix bug where we occasionally still logged access tokens (PR #477) +* Fix bug where ``/events`` would always return immediately for guest users + (PR #480) +* Fix bug where ``/sync`` unexpectedly returned old left rooms (PR #481) +* Fix enabling and disabling push rules (P #498) +* Fix bug where ``/register`` returned 500 when given unicode username + (PR #513) + Changes in synapse v0.12.0 (2016-01-04) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index 800707913..aacacda4f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.12.0" +__version__ = "0.12.1-rc1" From 97b364cb25179cefe38f0cb51ddde3ccb5757072 Mon Sep 17 00:00:00 2001 From: Florent VIOLLEAU Date: Wed, 27 Jan 2016 14:11:05 +0100 Subject: [PATCH 170/294] Update documentation Signed-off-by: Florent VIOLLEAU --- AUTHORS.rst | 3 +++ README.rst | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index f19d17d24..07d4bee2a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -51,3 +51,6 @@ Steven Hammerton Mads Robin Christensen * CentOS 7 installation instructions. + +Florent Violleau + * Add Raspberry Pi installation instructions and general troubleshooting items \ No newline at end of file diff --git a/README.rst b/README.rst index 297e72f1a..39a338c79 100644 --- a/README.rst +++ b/README.rst @@ -125,6 +125,15 @@ Installing prerequisites on Mac OS X:: sudo easy_install pip sudo pip install virtualenv +Installing prerequisites on Raspbian:: + + sudo apt-get install build-essential python2.7-dev libffi-dev \ + python-pip python-setuptools sqlite3 \ + libssl-dev python-virtualenv libjpeg-dev + sudo pip install --upgrade pip + sudo pip install --upgrade ndg-httpsclient + sudo pip install --upgrade virtualenv + To install the synapse homeserver run:: virtualenv -p python2.7 ~/.synapse @@ -310,6 +319,18 @@ may need to manually upgrade it:: sudo pip install --upgrade pip +Installing may fail with ``Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)``. +You can fix this by manually upgrading pip and virtualenv:: + + sudo pip install --upgrade virtualenv + +You can next rerun ``virtualenv -p python2.7 synapse`` to update the virtual env. + +Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.`` +You can fix this by manually installing ndg-httpsclient:: + + pip install --upgrade ndg-httpsclient + Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``. You can fix this by upgrading setuptools:: @@ -544,4 +565,4 @@ sphinxcontrib-napoleon:: Building internal API documentation:: python setup.py build_sphinx - + \ No newline at end of file From d6d60b4d6cfb4f4b02a83bddb04530a6fbdf52a3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Jan 2016 17:02:10 +0000 Subject: [PATCH 171/294] Federation: drop events which cause SynapseErrors ... rather than rejecting any attempt to federate channels which contain such events. --- synapse/handlers/federation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6c19d6ae8..2ce1e9d6c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1186,7 +1186,13 @@ class FederationHandler(BaseHandler): try: self.auth.check(e, auth_events=auth_for_e) - except AuthError as err: + except SynapseError as err: + # we may get SynapseErrors here as well as AuthErrors. For + # instance, there are a couple of (ancient) events in some + # rooms whose senders do not have the correct sigil; these + # cause SynapseErrors in auth.check. We don't want to give up + # the attempt to federate altogether in such cases. + logger.warn( "Rejecting %s because %s", e.event_id, err.msg From 2152b320c5cd11caca0a039279e48262d6d88408 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Jan 2016 17:09:17 +0000 Subject: [PATCH 172/294] PEP 8 --- synapse/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index 006e91b37..e013a349c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -21,7 +21,6 @@ # Imports required for the default HomeServer() implementation from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.enterprise import adbapi -from twisted.internet import defer from synapse.federation import initialize_http_replication from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory From 0487c9441f1439bd02cb4d107a4fcacfe5dbe75d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Jan 2016 17:25:07 +0000 Subject: [PATCH 173/294] Fix tests --- tests/storage/test_appservice.py | 6 +++--- tests/storage/test_registration.py | 3 +-- tests/utils.py | 8 ++++++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 5abecdf6e..ed8af10d8 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -439,7 +439,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): f2 = self._write_config(suffix="2") config = Mock(app_service_config_files=[f1, f2]) - hs = yield setup_test_homeserver(config=config) + hs = yield setup_test_homeserver(config=config, datastore=Mock()) ApplicationServiceStore(hs) @@ -449,7 +449,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): f2 = self._write_config(id="id", suffix="2") config = Mock(app_service_config_files=[f1, f2]) - hs = yield setup_test_homeserver(config=config) + hs = yield setup_test_homeserver(config=config, datastore=Mock()) with self.assertRaises(ConfigError) as cm: ApplicationServiceStore(hs) @@ -465,7 +465,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): f2 = self._write_config(as_token="as_token", suffix="2") config = Mock(app_service_config_files=[f1, f2]) - hs = yield setup_test_homeserver(config=config) + hs = yield setup_test_homeserver(config=config, datastore=Mock()) with self.assertRaises(ConfigError) as cm: ApplicationServiceStore(hs) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index a35efcc71..7b3b4c13b 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -18,7 +18,6 @@ from tests import unittest from twisted.internet import defer from synapse.api.errors import StoreError -from synapse.storage.registration import RegistrationStore from synapse.util import stringutils from tests.utils import setup_test_homeserver @@ -31,7 +30,7 @@ class RegistrationStoreTestCase(unittest.TestCase): hs = yield setup_test_homeserver() self.db_pool = hs.get_db_pool() - self.store = RegistrationStore(hs) + self.store = hs.get_datastore() self.user_id = "@my-user:test" self.tokens = ["AbCdEfGhIjKlMnOpQrStUvWxYz", diff --git a/tests/utils.py b/tests/utils.py index d75d492cb..43cc2b30c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -60,8 +60,10 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): name, db_pool=db_pool, config=config, version_string="Synapse/tests", database_engine=create_engine("sqlite3"), + get_db_conn=db_pool.get_db_conn, **kargs ) + hs.setup() else: hs = HomeServer( name, db_pool=None, datastore=datastore, config=config, @@ -280,6 +282,12 @@ class SQLiteMemoryDbPool(ConnectionPool, object): lambda conn: prepare_database(conn, engine) ) + def get_db_conn(self): + conn = self.connect() + engine = create_engine("sqlite3") + prepare_database(conn, engine) + return conn + class MemoryDataStore(object): From f93ecf87837567a540904e5d6a4280cf453ce7a8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Jan 2016 18:27:23 +0000 Subject: [PATCH 174/294] Don't turn on profiling --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fb76be58a..504557b2f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -651,7 +651,7 @@ def _resource_id(resource, path_seg): def run(hs): - PROFILE_SYNAPSE = True + PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile From b97f6626b6f9b91498d06a7ae113b9d20f1fc2ef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Jan 2016 09:54:30 +0000 Subject: [PATCH 175/294] Add cache to room stream --- synapse/handlers/sync.py | 42 +++++-- synapse/storage/events.py | 2 + synapse/storage/receipts.py | 65 +---------- synapse/storage/stream.py | 133 +++++++++++++++++++++++ synapse/util/caches/room_change_cache.py | 86 +++++++++++++++ 5 files changed, 254 insertions(+), 74 deletions(-) create mode 100644 synapse/util/caches/room_change_cache.py diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 328c049b0..1fdf97831 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -514,13 +514,6 @@ class SyncHandler(BaseHandler): timeline_limit = sync_config.filter_collection.timeline_limit() - room_events, _ = yield self.store.get_room_events_stream( - sync_config.user.to_string(), - from_key=since_token.room_key, - to_key=now_token.room_key, - limit=timeline_limit + 1, - ) - tags_by_room = yield self.store.get_updated_tags( sync_config.user.to_string(), since_token.account_data_key, @@ -533,6 +526,32 @@ class SyncHandler(BaseHandler): ) ) + rooms_changed = yield self.store.get_room_changes_for_user( + sync_config.user.to_string(), since_token.room_key, now_token.room_key + ) + + room_to_events = yield self.store.get_room_events_stream_for_rooms( + room_ids=room_ids, + from_key=since_token.room_key, + to_key=now_token.room_key, + limit=timeline_limit + 1, + ) + + room_events = [ + event + for events, _ in room_to_events.values() + for event in events + ] + + room_events.extend(rooms_changed) + + # room_events, _ = yield self.store.get_room_events_stream( + # sync_config.user.to_string(), + # from_key=since_token.room_key, + # to_key=now_token.room_key, + # limit=timeline_limit + 1, + # ) + joined = [] archived = [] if len(room_events) <= timeline_limit: @@ -694,14 +713,12 @@ class SyncHandler(BaseHandler): end_key = room_key while limited and len(recents) < timeline_limit and max_repeat: - events, keys = yield self.store.get_recent_events_for_room( + events, end_key = yield self.store.get_recent_room_events_stream_for_room( room_id, limit=load_limit + 1, - from_token=since_token.room_key if since_token else None, - end_token=end_key, + from_key=since_token.room_key if since_token else None, + to_key=end_key, ) - room_key, _ = keys - end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter_collection.filter_room_timeline(events) loaded_recents = yield self._filter_events_for_client( sync_config.user.to_string(), @@ -712,6 +729,7 @@ class SyncHandler(BaseHandler): recents = loaded_recents if len(events) <= load_limit: limited = False + break max_repeat -= 1 if len(recents) > timeline_limit: diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 298cb9bad..d96ea3a30 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -128,6 +128,8 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) + logger.info("Invalidating %r at %r", event.room_id, stream_ordering) + self._events_stream_cache.room_has_changed(None, event.room_id, stream_ordering) except _RollbackButIsFineException: pass diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index c0593e23e..b7a4e7774 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -16,6 +16,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList, cached from synapse.util.caches import cache_counter, caches_by_name +from synapse.util.caches.room_change_cache import RoomStreamChangeCache from twisted.internet import defer @@ -31,8 +32,8 @@ class ReceiptsStore(SQLBaseStore): def __init__(self, hs): super(ReceiptsStore, self).__init__(hs) - self._receipts_stream_cache = _RoomStreamChangeCache( - self._receipts_id_gen.get_max_token(None) + self._receipts_stream_cache = RoomStreamChangeCache( + "ReceiptsRoomChangeCache", self._receipts_id_gen.get_max_token(None) ) @cached(num_args=2) @@ -370,63 +371,3 @@ class ReceiptsStore(SQLBaseStore): "data": json.dumps(data), } ) - - -class _RoomStreamChangeCache(object): - """Keeps track of the stream_id of the latest change in rooms. - - Given a list of rooms and stream key, it will give a subset of rooms that - may have changed since that key. If the key is too old then the cache - will simply return all rooms. - """ - def __init__(self, current_key, size_of_cache=10000): - self._size_of_cache = size_of_cache - self._room_to_key = {} - self._cache = sorteddict() - self._earliest_key = current_key - self.name = "ReceiptsRoomChangeCache" - caches_by_name[self.name] = self._cache - - @defer.inlineCallbacks - def get_rooms_changed(self, store, room_ids, key): - """Returns subset of room ids that have had new receipts since the - given key. If the key is too old it will just return the given list. - """ - if key > (yield self._get_earliest_key(store)): - keys = self._cache.keys() - i = keys.bisect_right(key) - - result = set( - self._cache[k] for k in keys[i:] - ).intersection(room_ids) - - cache_counter.inc_hits(self.name) - else: - result = room_ids - cache_counter.inc_misses(self.name) - - defer.returnValue(result) - - @defer.inlineCallbacks - def room_has_changed(self, store, room_id, key): - """Informs the cache that the room has been changed at the given key. - """ - if key > (yield self._get_earliest_key(store)): - old_key = self._room_to_key.get(room_id, None) - if old_key: - key = max(key, old_key) - self._cache.pop(old_key, None) - self._cache[key] = room_id - - while len(self._cache) > self._size_of_cache: - k, r = self._cache.popitem() - self._earliest_key = max(k, self._earliest_key) - self._room_to_key.pop(r, None) - - @defer.inlineCallbacks - def _get_earliest_key(self, store): - if self._earliest_key is None: - self._earliest_key = yield store.get_max_receipt_stream_id() - self._earliest_key = int(self._earliest_key) - - defer.returnValue(self._earliest_key) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e31bad258..3a32a0019 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -37,6 +37,7 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks +from synapse.util.caches.room_change_cache import RoomStreamChangeCache from synapse.api.constants import EventTypes from synapse.types import RoomStreamToken from synapse.util.logutils import log_function @@ -77,6 +78,12 @@ def upper_bound(token): class StreamStore(SQLBaseStore): + def __init__(self, hs): + super(StreamStore, self).__init__(hs) + + self._events_stream_cache = RoomStreamChangeCache( + "EventsRoomStreamChangeCache", self._stream_id_gen.get_max_token(None) + ) @defer.inlineCallbacks def get_appservice_room_stream(self, service, from_key, to_key, limit=0): @@ -157,6 +164,132 @@ class StreamStore(SQLBaseStore): results = yield self.runInteraction("get_appservice_room_stream", f) defer.returnValue(results) + @defer.inlineCallbacks + def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0): + from_id = RoomStreamToken.parse_stream_token(from_key).stream + + room_ids = yield self._events_stream_cache.get_rooms_changed( + self, room_ids, from_id + ) + + if not room_ids: + defer.returnValue({}) + + results = {} + room_ids = list(room_ids) + for rm_ids in (room_ids[i:i+20] for i in xrange(0, len(room_ids), 20)): + res = yield defer.gatherResults([ + self.get_recent_room_events_stream_for_room( + room_id, from_key, to_key, limit + ).addCallback(lambda r, rm: (rm, r), room_id) + for room_id in room_ids + ]) + results.update(dict(res)) + + defer.returnValue(results) + + @defer.inlineCallbacks + def get_recent_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0): + if from_key is not None: + from_id = RoomStreamToken.parse_stream_token(from_key).stream + else: + from_id = None + to_id = RoomStreamToken.parse_stream_token(to_key).stream + + if from_key == to_key: + defer.returnValue(([], from_key)) + + has_changed = yield self._events_stream_cache.get_room_has_changed( + room_id, from_id + ) + + if not has_changed: + defer.returnValue(([], from_key)) + + def f(txn): + if from_id is not None: + sql = ( + "SELECT event_id, stream_ordering FROM events WHERE" + " room_id = ?" + " AND not outlier" + " AND stream_ordering > ? AND stream_ordering <= ?" + " ORDER BY stream_ordering DESC LIMIT ?" + ) + txn.execute(sql, (room_id, from_id, to_id, limit)) + else: + sql = ( + "SELECT event_id, stream_ordering FROM events WHERE" + " room_id = ?" + " AND not outlier" + " AND stream_ordering <= ?" + " ORDER BY stream_ordering DESC LIMIT ?" + ) + txn.execute(sql, (room_id, to_id, limit)) + + rows = self.cursor_to_dict(txn) + + ret = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) + + ret.reverse() + + self._set_before_and_after(ret, rows) + + if rows: + key = "s%d" % min(r["stream_ordering"] for r in rows) + else: + # Assume we didn't get anything because there was nothing to + # get. + key = from_key + + return ret, key + res = yield self.runInteraction("get_recent_room_events_stream_for_room", f) + defer.returnValue(res) + + def get_room_changes_for_user(self, user_id, from_key, to_key): + if from_key is not None: + from_id = RoomStreamToken.parse_stream_token(from_key).stream + else: + from_id = None + to_id = RoomStreamToken.parse_stream_token(to_key).stream + + if from_key == to_key: + return defer.succeed([]) + + def f(txn): + if from_id is not None: + sql = ( + "SELECT m.event_id, stream_ordering FROM events AS e, room_memberships AS m" + " WHERE e.event_id = m.event_id" + " AND m.user_id = ?" + " AND e.stream_ordering > ? AND e.stream_ordering <= ?" + " ORDER BY e.stream_ordering ASC" + ) + txn.execute(sql, (user_id, from_id, to_id,)) + else: + sql = ( + "SELECT m.event_id, stream_ordering FROM events AS e, room_memberships AS m" + " WHERE e.event_id = m.event_id" + " AND m.user_id = ?" + " AND stream_ordering <= ?" + " ORDER BY stream_ordering ASC" + ) + txn.execute(sql, (user_id, to_id,)) + rows = self.cursor_to_dict(txn) + + ret = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) + + return ret + + return self.runInteraction("get_room_changes_for_user", f) + @log_function def get_room_events_stream( self, diff --git a/synapse/util/caches/room_change_cache.py b/synapse/util/caches/room_change_cache.py new file mode 100644 index 000000000..3a873c9c3 --- /dev/null +++ b/synapse/util/caches/room_change_cache.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.util.caches import cache_counter, caches_by_name + + +from blist import sorteddict +import logging + + +logger = logging.getLogger(__name__) + + +class RoomStreamChangeCache(object): + """Keeps track of the stream_id of the latest change in rooms. + + Given a list of rooms and stream key, it will give a subset of rooms that + may have changed since that key. If the key is too old then the cache + will simply return all rooms. + """ + def __init__(self, name, current_key, size_of_cache=10000): + self._size_of_cache = size_of_cache + self._room_to_key = {} + self._cache = sorteddict() + self._earliest_known_key = current_key + self.name = name + caches_by_name[self.name] = self._cache + + def get_room_has_changed(self, room_id, key): + if key <= self._earliest_known_key: + return True + + room_key = self._room_to_key.get(room_id, None) + if room_key is None: + return True + + if key < room_key: + return True + + return False + + def get_rooms_changed(self, store, room_ids, key): + """Returns subset of room ids that have had new things since the + given key. If the key is too old it will just return the given list. + """ + if key > self._earliest_known_key: + keys = self._cache.keys() + i = keys.bisect_right(key) + + result = set( + self._cache[k] for k in keys[i:] + ).intersection(room_ids) + + cache_counter.inc_hits(self.name) + else: + result = room_ids + cache_counter.inc_misses(self.name) + + return result + + def room_has_changed(self, store, room_id, key): + """Informs the cache that the room has been changed at the given key. + """ + if key > self._earliest_known_key: + old_key = self._room_to_key.get(room_id, None) + if old_key: + key = max(key, old_key) + self._cache.pop(old_key, None) + self._cache[key] = room_id + + while len(self._cache) > self._size_of_cache: + k, r = self._cache.popitem() + self._earliest_key = max(k, self._earliest_key) + self._room_to_key.pop(r, None) From aca3193efb8c5f9f20049f61c96e5ff12f328b05 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Jan 2016 17:06:52 +0000 Subject: [PATCH 176/294] Use the same path for incremental with gap or without gap --- synapse/handlers/sync.py | 352 ++++++++++++++++---------------------- synapse/storage/events.py | 1 - synapse/storage/stream.py | 6 +- 3 files changed, 147 insertions(+), 212 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1fdf97831..f5e20d6a6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -72,7 +72,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ ) -class ArchivedSyncResult(collections.namedtuple("JoinedSyncResult", [ +class ArchivedSyncResult(collections.namedtuple("ArchivedSyncResult", [ "room_id", # str "timeline", # TimelineBatch "state", # dict[(str, str), FrozenEvent] @@ -429,44 +429,20 @@ class SyncHandler(BaseHandler): defer.returnValue((now_token, ephemeral_by_room)) - @defer.inlineCallbacks def full_state_sync_for_archived_room(self, room_id, sync_config, leave_event_id, leave_token, timeline_since_token, tags_by_room, account_data_by_room): """Sync a room for a client which is starting without any state Returns: - A Deferred JoinedSyncResult. + A Deferred ArchivedSyncResult. """ - batch = yield self.load_filtered_recents( - room_id, sync_config, leave_token, since_token=timeline_since_token + return self.incremental_sync_for_archived_room( + sync_config, room_id, leave_event_id, timeline_since_token, tags_by_room, + account_data_by_room, full_state=True, leave_token=leave_token, ) - leave_state = yield self.store.get_state_for_event(leave_event_id) - - leave_state = { - (e.type, e.state_key): e - for e in sync_config.filter_collection.filter_room_state( - leave_state.values() - ) - } - - account_data = self.account_data_for_room( - room_id, tags_by_room, account_data_by_room - ) - - account_data = sync_config.filter_collection.filter_room_account_data( - account_data - ) - - defer.returnValue(ArchivedSyncResult( - room_id=room_id, - timeline=batch, - state=leave_state, - account_data=account_data, - )) - @defer.inlineCallbacks def incremental_sync_with_gap(self, sync_config, since_token): """ Get the incremental delta needed to bring the client up to @@ -512,173 +488,127 @@ class SyncHandler(BaseHandler): sync_config.user ) + user_id = sync_config.user.to_string() + timeline_limit = sync_config.filter_collection.timeline_limit() tags_by_room = yield self.store.get_updated_tags( - sync_config.user.to_string(), + user_id, since_token.account_data_key, ) account_data, account_data_by_room = ( yield self.store.get_updated_account_data_for_user( - sync_config.user.to_string(), + user_id, since_token.account_data_key, ) ) + # Get a list of membership change events that have happened. rooms_changed = yield self.store.get_room_changes_for_user( - sync_config.user.to_string(), since_token.room_key, now_token.room_key + user_id, since_token.room_key, now_token.room_key ) + mem_change_events_by_room_id = {} + for event in rooms_changed: + mem_change_events_by_room_id.setdefault(event.room_id, []).append(event) + + newly_joined_rooms = [] + archived = [] + invited = [] + for room_id, events in mem_change_events_by_room_id.items(): + non_joins = [e for e in events if e.membership != Membership.JOIN] + has_join = len(non_joins) != len(events) + + # We want to figure out if we joined the room at some point since + # the last sync (even if we have since left). This is to make sure + # we do send down the room, and with full state, where necessary + if room_id in joined_room_ids or has_join: + old_state = yield self.get_state_at(room_id, since_token) + old_mem_ev = old_state.get((EventTypes.Member, user_id), None) + if not old_mem_ev or old_mem_ev.membership != Membership.JOIN: + newly_joined_rooms.append(room_id) + + if room_id in joined_room_ids: + continue + + if not non_joins: + continue + + # Only bother if we're still currently invited + should_invite = non_joins[-1].membership == Membership.INVITE + if should_invite: + room_sync = InvitedSyncResult(room_id, invite=non_joins[-1]) + if room_sync: + invited.append(room_sync) + + # Always include leave/ban events. Just take the last one. + # TODO: How do we handle ban -> leave in same batch? + leave_events = [ + e for e in non_joins + if e.membership in (Membership.LEAVE, Membership.BAN) + ] + + if leave_events: + leave_event = leave_events[-1] + room_sync = yield self.incremental_sync_for_archived_room( + sync_config, room_id, leave_event.event_id, since_token, + tags_by_room, account_data_by_room, + full_state=room_id in newly_joined_rooms + ) + if room_sync: + archived.append(room_sync) + + # Get all events for rooms we're currently joined to. room_to_events = yield self.store.get_room_events_stream_for_rooms( - room_ids=room_ids, + room_ids=joined_room_ids, from_key=since_token.room_key, to_key=now_token.room_key, limit=timeline_limit + 1, ) - room_events = [ - event - for events, _ in room_to_events.values() - for event in events - ] - - room_events.extend(rooms_changed) - - # room_events, _ = yield self.store.get_room_events_stream( - # sync_config.user.to_string(), - # from_key=since_token.room_key, - # to_key=now_token.room_key, - # limit=timeline_limit + 1, - # ) - joined = [] - archived = [] - if len(room_events) <= timeline_limit: - # There is no gap in any of the rooms. Therefore we can just - # partition the new events by room and return them. - logger.debug("Got %i events for incremental sync - not limited", - len(room_events)) + # We loop through all room ids, even if there are no new events, in case + # there are non room events taht we need to notify about. + for room_id in joined_room_ids: + room_entry = room_to_events.get(room_id, None) - invite_events = [] - leave_events = [] - events_by_room_id = {} - for event in room_events: - events_by_room_id.setdefault(event.room_id, []).append(event) - if event.room_id not in joined_room_ids: - if (event.type == EventTypes.Member - and event.state_key == sync_config.user.to_string()): - if event.membership == Membership.INVITE: - invite_events.append(event) - elif event.membership in (Membership.LEAVE, Membership.BAN): - leave_events.append(event) + if room_entry: + events, start_key = room_entry - for room_id in joined_room_ids: - recents = events_by_room_id.get(room_id, []) - logger.debug("Events for room %s: %r", room_id, recents) - state = { - (event.type, event.state_key): event - for event in recents if event.is_state()} - limited = False + prev_batch_token = now_token.copy_and_replace("room_key", start_key) - if recents: - prev_batch = now_token.copy_and_replace( - "room_key", recents[0].internal_metadata.before - ) - else: - prev_batch = now_token + newly_joined_room = room_id in newly_joined_rooms + full_state = newly_joined_room - just_joined = yield self.check_joined_room(sync_config, state) - if just_joined: - logger.debug("User has just joined %s: needs full state", - room_id) - state = yield self.get_state_at(room_id, now_token) - # the timeline is inherently limited if we've just joined - limited = True - - recents = sync_config.filter_collection.filter_room_timeline(recents) - - state = { - (e.type, e.state_key): e - for e in sync_config.filter_collection.filter_room_state( - state.values() - ) - } - - acc_data = self.account_data_for_room( - room_id, tags_by_room, account_data_by_room + batch = yield self.load_filtered_recents( + room_id, sync_config, prev_batch_token, + since_token=since_token, + recents=events, + newly_joined_room=newly_joined_room, ) - - acc_data = sync_config.filter_collection.filter_room_account_data( - acc_data + else: + batch = TimelineBatch( + events=[], + prev_batch=since_token, + limited=False, ) + full_state = False - ephemeral = sync_config.filter_collection.filter_room_ephemeral( - ephemeral_by_room.get(room_id, []) - ) - - room_sync = JoinedSyncResult( - room_id=room_id, - timeline=TimelineBatch( - events=recents, - prev_batch=prev_batch, - limited=limited, - ), - state=state, - ephemeral=ephemeral, - account_data=acc_data, - unread_notifications={}, - ) - logger.debug("Result for room %s: %r", room_id, room_sync) - - if room_sync: - notifs = yield self.unread_notifs_for_room_id( - room_id, sync_config, all_ephemeral_by_room - ) - - if notifs is not None: - notif_dict = room_sync.unread_notifications - notif_dict["notification_count"] = len(notifs) - notif_dict["highlight_count"] = len([ - 1 for notif in notifs - if _action_has_highlight(notif["actions"]) - ]) - - joined.append(room_sync) - - else: - logger.debug("Got %i events for incremental sync - hit limit", - len(room_events)) - - invite_events = yield self.store.get_invites_for_user( - sync_config.user.to_string() - ) - - leave_events = yield self.store.get_leave_and_ban_events_for_user( - sync_config.user.to_string() - ) - - for room_id in joined_room_ids: - room_sync = yield self.incremental_sync_with_gap_for_room( - room_id, sync_config, since_token, now_token, - ephemeral_by_room, tags_by_room, account_data_by_room, - all_ephemeral_by_room=all_ephemeral_by_room, - ) - if room_sync: - joined.append(room_sync) - - for leave_event in leave_events: - room_sync = yield self.incremental_sync_for_archived_room( - sync_config, leave_event, since_token, tags_by_room, - account_data_by_room + room_sync = yield self.incremental_sync_with_gap_for_room( + room_id=room_id, + sync_config=sync_config, + since_token=since_token, + now_token=now_token, + ephemeral_by_room=ephemeral_by_room, + tags_by_room=tags_by_room, + account_data_by_room=account_data_by_room, + all_ephemeral_by_room=all_ephemeral_by_room, + batch=batch, + full_state=full_state, ) if room_sync: - archived.append(room_sync) - - invited = [ - InvitedSyncResult(room_id=event.room_id, invite=event) - for event in invite_events - ] + joined.append(room_sync) account_data_for_user = sync_config.filter_collection.filter_account_data( self.account_data_for_user(account_data) @@ -699,12 +629,10 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def load_filtered_recents(self, room_id, sync_config, now_token, - since_token=None): + since_token=None, recents=None, newly_joined_room=False): """ :returns a Deferred TimelineBatch """ - limited = True - recents = [] filtering_factor = 2 timeline_limit = sync_config.filter_collection.timeline_limit() load_limit = max(timeline_limit * filtering_factor, 100) @@ -712,11 +640,27 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key end_key = room_key + limited = recents is None or newly_joined_room or timeline_limit < len(recents) + + if recents is not None: + recents = sync_config.filter_collection.filter_room_timeline(recents) + recents = yield self._filter_events_for_client( + sync_config.user.to_string(), + recents, + is_peeking=sync_config.is_guest, + ) + else: + recents = [] + + since_key = None + if since_token and not newly_joined_room: + since_key = since_token.room_key + while limited and len(recents) < timeline_limit and max_repeat: - events, end_key = yield self.store.get_recent_room_events_stream_for_room( + events, end_key = yield self.store.get_room_events_stream_for_room( room_id, limit=load_limit + 1, - from_key=since_token.room_key if since_token else None, + from_key=since_key, to_key=end_key, ) loaded_recents = sync_config.filter_collection.filter_room_timeline(events) @@ -727,6 +671,7 @@ class SyncHandler(BaseHandler): ) loaded_recents.extend(recents) recents = loaded_recents + if len(events) <= load_limit: limited = False break @@ -742,7 +687,9 @@ class SyncHandler(BaseHandler): ) defer.returnValue(TimelineBatch( - events=recents, prev_batch=prev_batch_token, limited=limited + events=recents, + prev_batch=prev_batch_token, + limited=limited or newly_joined_room )) @defer.inlineCallbacks @@ -750,24 +697,8 @@ class SyncHandler(BaseHandler): since_token, now_token, ephemeral_by_room, tags_by_room, account_data_by_room, - all_ephemeral_by_room): - """ Get the incremental delta needed to bring the client up to date for - the room. Gives the client the most recent events and the changes to - state. - Returns: - A Deferred JoinedSyncResult - """ - logger.debug("Doing incremental sync for room %s between %s and %s", - room_id, since_token, now_token) - - # TODO(mjark): Check for redactions we might have missed. - - batch = yield self.load_filtered_recents( - room_id, sync_config, now_token, since_token, - ) - - logger.debug("Recents %r", batch) - + all_ephemeral_by_room, + batch, full_state=False): if batch.limited: current_state = yield self.get_state_at(room_id, now_token) @@ -832,43 +763,48 @@ class SyncHandler(BaseHandler): defer.returnValue(room_sync) @defer.inlineCallbacks - def incremental_sync_for_archived_room(self, sync_config, leave_event, + def incremental_sync_for_archived_room(self, sync_config, room_id, leave_event_id, since_token, tags_by_room, - account_data_by_room): + account_data_by_room, full_state, + leave_token=None): """ Get the incremental delta needed to bring the client up to date for the archived room. Returns: A Deferred ArchivedSyncResult """ - stream_token = yield self.store.get_stream_token_for_event( - leave_event.event_id - ) + if not leave_token: + stream_token = yield self.store.get_stream_token_for_event( + leave_event_id + ) - leave_token = since_token.copy_and_replace("room_key", stream_token) + leave_token = since_token.copy_and_replace("room_key", stream_token) - if since_token.is_after(leave_token): + if since_token and since_token.is_after(leave_token): defer.returnValue(None) batch = yield self.load_filtered_recents( - leave_event.room_id, sync_config, leave_token, since_token, + room_id, sync_config, leave_token, since_token, ) logger.debug("Recents %r", batch) state_events_at_leave = yield self.store.get_state_for_event( - leave_event.event_id + leave_event_id ) - state_at_previous_sync = yield self.get_state_at( - leave_event.room_id, stream_position=since_token - ) + if not full_state: + state_at_previous_sync = yield self.get_state_at( + room_id, stream_position=since_token + ) - state_events_delta = yield self.compute_state_delta( - since_token=since_token, - previous_state=state_at_previous_sync, - current_state=state_events_at_leave, - ) + state_events_delta = yield self.compute_state_delta( + since_token=since_token, + previous_state=state_at_previous_sync, + current_state=state_events_at_leave, + ) + else: + state_events_delta = state_events_at_leave state_events_delta = { (e.type, e.state_key): e @@ -878,7 +814,7 @@ class SyncHandler(BaseHandler): } account_data = self.account_data_for_room( - leave_event.room_id, tags_by_room, account_data_by_room + room_id, tags_by_room, account_data_by_room ) account_data = sync_config.filter_collection.filter_room_account_data( @@ -886,7 +822,7 @@ class SyncHandler(BaseHandler): ) room_sync = ArchivedSyncResult( - room_id=leave_event.room_id, + room_id=room_id, timeline=batch, state=state_events_delta, account_data=account_data, diff --git a/synapse/storage/events.py b/synapse/storage/events.py index d96ea3a30..0dd1daaa2 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -128,7 +128,6 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) - logger.info("Invalidating %r at %r", event.room_id, stream_ordering) self._events_stream_cache.room_has_changed(None, event.room_id, stream_ordering) except _RollbackButIsFineException: pass diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3a32a0019..563e289c4 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -179,7 +179,7 @@ class StreamStore(SQLBaseStore): room_ids = list(room_ids) for rm_ids in (room_ids[i:i+20] for i in xrange(0, len(room_ids), 20)): res = yield defer.gatherResults([ - self.get_recent_room_events_stream_for_room( + self.get_room_events_stream_for_room( room_id, from_key, to_key, limit ).addCallback(lambda r, rm: (rm, r), room_id) for room_id in room_ids @@ -189,7 +189,7 @@ class StreamStore(SQLBaseStore): defer.returnValue(results) @defer.inlineCallbacks - def get_recent_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0): + def get_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0): if from_key is not None: from_id = RoomStreamToken.parse_stream_token(from_key).stream else: @@ -246,7 +246,7 @@ class StreamStore(SQLBaseStore): key = from_key return ret, key - res = yield self.runInteraction("get_recent_room_events_stream_for_room", f) + res = yield self.runInteraction("get_room_events_stream_for_room", f) defer.returnValue(res) def get_room_changes_for_user(self, user_id, from_key, to_key): From e7febf4fbb1f1beb11e7a03252f6844f84af7f30 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Jan 2016 17:11:04 +0000 Subject: [PATCH 177/294] PEP8 --- synapse/storage/events.py | 4 +++- synapse/storage/receipts.py | 2 -- synapse/storage/stream.py | 9 ++++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0dd1daaa2..80187722e 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -128,7 +128,9 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) - self._events_stream_cache.room_has_changed(None, event.room_id, stream_ordering) + self._events_stream_cache.room_has_changed( + None, event.room_id, stream_ordering + ) except _RollbackButIsFineException: pass diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index b7a4e7774..7118368d9 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -15,12 +15,10 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList, cached -from synapse.util.caches import cache_counter, caches_by_name from synapse.util.caches.room_change_cache import RoomStreamChangeCache from twisted.internet import defer -from blist import sorteddict import logging import ujson as json diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 563e289c4..0b2225179 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -262,7 +262,8 @@ class StreamStore(SQLBaseStore): def f(txn): if from_id is not None: sql = ( - "SELECT m.event_id, stream_ordering FROM events AS e, room_memberships AS m" + "SELECT m.event_id, stream_ordering FROM events AS e," + " room_memberships AS m" " WHERE e.event_id = m.event_id" " AND m.user_id = ?" " AND e.stream_ordering > ? AND e.stream_ordering <= ?" @@ -271,7 +272,8 @@ class StreamStore(SQLBaseStore): txn.execute(sql, (user_id, from_id, to_id,)) else: sql = ( - "SELECT m.event_id, stream_ordering FROM events AS e, room_memberships AS m" + "SELECT m.event_id, stream_ordering FROM events AS e," + " room_memberships AS m" " WHERE e.event_id = m.event_id" " AND m.user_id = ?" " AND stream_ordering <= ?" @@ -307,7 +309,8 @@ class StreamStore(SQLBaseStore): "SELECT c.room_id FROM history_visibility AS h" " INNER JOIN current_state_events AS c" " ON h.event_id = c.event_id" - " WHERE c.room_id IN (%s) AND h.history_visibility = 'world_readable'" % ( + " WHERE c.room_id IN (%s)" + " AND h.history_visibility = 'world_readable'" % ( ",".join(map(lambda _: "?", room_ids)) ) ) From 5fc9b17518008e08a420b94a8f77955440c52b2b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Jan 2016 17:39:20 +0000 Subject: [PATCH 178/294] No chdir --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 504557b2f..56a34bd50 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -728,7 +728,6 @@ def run(hs): auto_close_fds=False, verbose=True, logger=logger, - chdir=os.path.dirname(os.path.abspath(__file__)), ) daemon.start() From 5cba88ea7c96e5e8a9f3bc1a28cf3414b3083d60 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Jan 2016 17:42:45 +0000 Subject: [PATCH 179/294] Make it possible to paginate forwards from stream tokens In order that we can fill the gap after a /sync, make it possible to paginate forwards from a stream token. --- synapse/handlers/message.py | 43 +++++++++++++++--------------- tests/rest/client/v1/test_rooms.py | 16 +++++++++-- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index ff800f8af..b73ad6214 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -105,8 +105,6 @@ class MessageHandler(BaseHandler): room_token = pagin_config.from_token.room_key room_token = RoomStreamToken.parse(room_token) - if room_token.topological is None: - raise SynapseError(400, "Invalid token") pagin_config.from_token = pagin_config.from_token.copy_and_replace( "room_key", str(room_token) @@ -117,27 +115,28 @@ class MessageHandler(BaseHandler): membership, member_event_id = yield self._check_in_room_or_world_readable( room_id, user_id ) - if membership == Membership.LEAVE: - # If they have left the room then clamp the token to be before - # they left the room. - leave_token = yield self.store.get_topological_token_for_event( - member_event_id + + if source_config.direction == 'b': + # if we're going backwards, we might need to backfill. This + # requires that we have a topo token. + if room_token.topological is None: + raise SynapseError(400, "Invalid token: cannot paginate " + "backwards from a stream token") + + if membership == Membership.LEAVE: + # If they have left the room then clamp the token to be before + # they left the room, to save the effort of loading from the + # database. + leave_token = yield self.store.get_topological_token_for_event( + member_event_id + ) + leave_token = RoomStreamToken.parse(leave_token) + if leave_token.topological < room_token.topological: + source_config.from_key = str(leave_token) + + yield self.hs.get_handlers().federation_handler.maybe_backfill( + room_id, room_token.topological ) - leave_token = RoomStreamToken.parse(leave_token) - if leave_token.topological < room_token.topological: - source_config.from_key = str(leave_token) - - if source_config.direction == "f": - if source_config.to_key is None: - source_config.to_key = str(leave_token) - else: - to_token = RoomStreamToken.parse(source_config.to_key) - if leave_token.topological < to_token.topological: - source_config.to_key = str(leave_token) - - yield self.hs.get_handlers().federation_handler.maybe_backfill( - room_id, room_token.topological - ) events, next_key = yield data_source.get_pagination_rows( requester.user, source_config, room_id diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index cd03106e8..2fe6f695f 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -1045,8 +1045,20 @@ class RoomMessageListTestCase(RestTestCase): self.assertTrue("end" in response) @defer.inlineCallbacks - def test_stream_token_is_rejected(self): + def test_stream_token_is_rejected_for_back_pagination(self): (code, response) = yield self.mock_resource.trigger_get( - "/rooms/%s/messages?access_token=x&from=s0_0_0_0" % + "/rooms/%s/messages?access_token=x&from=s0_0_0_0_0&dir=b" % self.room_id) self.assertEquals(400, code) + + @defer.inlineCallbacks + def test_stream_token_is_accepted_for_fwd_pagianation(self): + token = "s0_0_0_0_0" + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/messages?access_token=x&from=%s" % + (self.room_id, token)) + self.assertEquals(200, code) + self.assertTrue("start" in response) + self.assertEquals(token, response['start']) + self.assertTrue("chunk" in response) + self.assertTrue("end" in response) \ No newline at end of file From a6477d5933a11d5e93a3d6fad284db511e0a1ea0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 09:19:55 +0000 Subject: [PATCH 180/294] Remove chdir --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f881a8d26..a83d3f394 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -719,7 +719,6 @@ def run(hs): auto_close_fds=False, verbose=True, logger=logger, - chdir=os.path.dirname(os.path.abspath(__file__)), ) daemon.start() From c5e7c0e436d9073c07887676817bb5a45314aea5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 09:58:45 +0000 Subject: [PATCH 181/294] Up get_rooms_for_user cache size --- synapse/storage/roommember.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index edfecced0..1d3e004c9 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -241,7 +241,7 @@ class RoomMemberStore(SQLBaseStore): return rows - @cached() + @cached(max_entries=5000) def get_rooms_for_user(self, user_id): return self.get_rooms_for_user_where_membership_is( user_id, membership_list=[Membership.JOIN], From ba8931829b0b601eb14049c92e0f21a10772576d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 11:34:17 +0000 Subject: [PATCH 182/294] Return correct type of token --- synapse/storage/stream.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 0b2225179..28721e699 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -236,7 +236,7 @@ class StreamStore(SQLBaseStore): ret.reverse() - self._set_before_and_after(ret, rows) + self._set_before_and_after(ret, rows, topo_order=False) if rows: key = "s%d" % min(r["stream_ordering"] for r in rows) @@ -581,10 +581,13 @@ class StreamStore(SQLBaseStore): return rows[0][0] if rows else 0 @staticmethod - def _set_before_and_after(events, rows): + def _set_before_and_after(events, rows, topo_order=True): for event, row in zip(events, rows): stream = row["stream_ordering"] - topo = event.depth + if topo_order: + topo = event.depth + else: + topo = None internal = event.internal_metadata internal.before = str(RoomStreamToken(topo, stream - 1)) internal.after = str(RoomStreamToken(topo, stream)) From 4e7948b47a3f197682de82fc0cda07ebb08a581d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 11:52:34 +0000 Subject: [PATCH 183/294] Allow paginating backwards from stream token --- synapse/handlers/message.py | 15 +++++++++------ synapse/storage/stream.py | 16 ++++++++++++++-- tests/rest/client/v1/test_rooms.py | 9 +-------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b73ad6214..82c8cb5f0 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import SynapseError, AuthError, Codes +from synapse.api.errors import AuthError, Codes from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -119,9 +119,12 @@ class MessageHandler(BaseHandler): if source_config.direction == 'b': # if we're going backwards, we might need to backfill. This # requires that we have a topo token. - if room_token.topological is None: - raise SynapseError(400, "Invalid token: cannot paginate " - "backwards from a stream token") + if room_token.topological: + max_topo = room_token.topological + else: + max_topo = yield self.store.get_max_topological_token_for_stream_and_room( + room_id, room_token.stream + ) if membership == Membership.LEAVE: # If they have left the room then clamp the token to be before @@ -131,11 +134,11 @@ class MessageHandler(BaseHandler): member_event_id ) leave_token = RoomStreamToken.parse(leave_token) - if leave_token.topological < room_token.topological: + if leave_token.topological < max_topo: source_config.from_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( - room_id, room_token.topological + room_id, max_topo ) events, next_key = yield data_source.get_pagination_rows( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 28721e699..5096b4686 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -234,10 +234,10 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) - ret.reverse() - self._set_before_and_after(ret, rows, topo_order=False) + ret.reverse() + if rows: key = "s%d" % min(r["stream_ordering"] for r in rows) else: @@ -570,6 +570,18 @@ class StreamStore(SQLBaseStore): row["topological_ordering"], row["stream_ordering"],) ) + def get_max_topological_token_for_stream_and_room(self, room_id, stream_key): + sql = ( + "SELECT max(topological_ordering) FROM events" + " WHERE room_id = ? AND stream_ordering < ?" + ) + return self._execute( + "get_max_topological_token_for_stream_and_room", None, + sql, room_id, stream_key, + ).addCallback( + lambda r: r[0][0] if r else 0 + ) + def _get_max_topological_txn(self, txn): txn.execute( "SELECT MAX(topological_ordering) FROM events" diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 2fe6f695f..ad5dd3bd6 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -1044,13 +1044,6 @@ class RoomMessageListTestCase(RestTestCase): self.assertTrue("chunk" in response) self.assertTrue("end" in response) - @defer.inlineCallbacks - def test_stream_token_is_rejected_for_back_pagination(self): - (code, response) = yield self.mock_resource.trigger_get( - "/rooms/%s/messages?access_token=x&from=s0_0_0_0_0&dir=b" % - self.room_id) - self.assertEquals(400, code) - @defer.inlineCallbacks def test_stream_token_is_accepted_for_fwd_pagianation(self): token = "s0_0_0_0_0" @@ -1061,4 +1054,4 @@ class RoomMessageListTestCase(RestTestCase): self.assertTrue("start" in response) self.assertEquals(token, response['start']) self.assertTrue("chunk" in response) - self.assertTrue("end" in response) \ No newline at end of file + self.assertTrue("end" in response) From 3c6518ddbfb6f74eb5c0d1d0e4e8159e4ae45b04 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 14:03:48 +0000 Subject: [PATCH 184/294] Amalgamate incremental and full sync for user --- synapse/handlers/sync.py | 56 ++++++++++++---------------------------- 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f5e20d6a6..2bd83e4b5 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,46 +298,19 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, since_token=timeline_since_token ) - notifs = yield self.unread_notifs_for_room_id( - room_id, sync_config, ephemeral_by_room + room_sync = yield self.incremental_sync_with_gap_for_room( + room_id, sync_config, + now_token=now_token, + since_token=timeline_since_token, + ephemeral_by_room=ephemeral_by_room, + tags_by_room=tags_by_room, + account_data_by_room=account_data_by_room, + all_ephemeral_by_room=ephemeral_by_room, + batch=batch, + full_state=True, ) - unread_notifications = {} - if notifs is not None: - unread_notifications["notification_count"] = len(notifs) - unread_notifications["highlight_count"] = len([ - 1 for notif in notifs if _action_has_highlight(notif["actions"]) - ]) - - current_state = yield self.get_state_at(room_id, now_token) - - current_state = { - (e.type, e.state_key): e - for e in sync_config.filter_collection.filter_room_state( - current_state.values() - ) - } - - account_data = self.account_data_for_room( - room_id, tags_by_room, account_data_by_room - ) - - account_data = sync_config.filter_collection.filter_room_account_data( - account_data - ) - - ephemeral = sync_config.filter_collection.filter_room_ephemeral( - ephemeral_by_room.get(room_id, []) - ) - - defer.returnValue(JoinedSyncResult( - room_id=room_id, - timeline=batch, - state=current_state, - ephemeral=ephemeral, - account_data=account_data, - unread_notifications=unread_notifications, - )) + defer.returnValue(room_sync) def account_data_for_user(self, account_data): account_data_events = [] @@ -635,7 +608,7 @@ class SyncHandler(BaseHandler): """ filtering_factor = 2 timeline_limit = sync_config.filter_collection.timeline_limit() - load_limit = max(timeline_limit * filtering_factor, 100) + load_limit = timeline_limit * filtering_factor max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key end_key = room_key @@ -699,7 +672,10 @@ class SyncHandler(BaseHandler): account_data_by_room, all_ephemeral_by_room, batch, full_state=False): - if batch.limited: + if full_state: + state = yield self.get_state_at(room_id, now_token) + + elif batch.limited: current_state = yield self.get_state_at(room_id, now_token) state_at_previous_sync = yield self.get_state_at( From 571a5663993fe68e85da76f9df47c2f215a84403 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 14:11:16 +0000 Subject: [PATCH 185/294] Change load limit params --- synapse/handlers/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 2bd83e4b5..51ec4702d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -608,8 +608,8 @@ class SyncHandler(BaseHandler): """ filtering_factor = 2 timeline_limit = sync_config.filter_collection.timeline_limit() - load_limit = timeline_limit * filtering_factor - max_repeat = 3 # Only try a few times per room, otherwise + load_limit = max(timeline_limit * filtering_factor, 10) + max_repeat = 5 # Only try a few times per room, otherwise room_key = now_token.room_key end_key = room_key From 9101193242c59a9b56f550be05509c76e87b3bca Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 28 Jan 2016 14:18:45 +0000 Subject: [PATCH 186/294] Quot all the things --- jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins.sh b/jenkins.sh index e2bb706c7..804bbd49f 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -52,7 +52,7 @@ RUN_POSTGRES="" for port in $(($PORT_BASE + 1)) $(($PORT_BASE + 2)); do if psql synapse_jenkins_$port <<< ""; then - RUN_POSTGRES=$RUN_POSTGRES:$port + RUN_POSTGRES="$RUN_POSTGRES:$port" cat > localhost-$port/database.yaml << EOF name: psycopg2 args: @@ -62,7 +62,7 @@ EOF done # Run if both postgresql databases exist -if test $RUN_POSTGRES = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then +if test "$RUN_POSTGRES" = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then echo >&2 "Running sytest with PostgreSQL"; $TOX_BIN/pip install psycopg2 ./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \ From 7ed2bbeb11d99ef97672497879e480f91db9b99b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 14:32:05 +0000 Subject: [PATCH 187/294] Clean up a bit. Add comment --- synapse/app/homeserver.py | 13 +++++++------ synapse/server.py | 4 ++-- synapse/storage/__init__.py | 27 +++++++++------------------ 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 504557b2f..65562222c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -255,12 +255,13 @@ class SynapseHomeServer(HomeServer): quit_with_error(e.message) def get_db_conn(self): - db_conn = self.database_engine.module.connect( - **{ - k: v for k, v in self.db_config.get("args", {}).items() - if not k.startswith("cp_") - } - ) + # Any param beginning with cp_ is a parameter for adbapi, and should + # not be passed to the database engine. + db_params = { + k: v for k, v in self.db_config.get("args", {}).items() + if not k.startswith("cp_") + } + db_conn = self.database_engine.module.connect(**db_params) self.database_engine.on_new_connection(db_conn) return db_conn diff --git a/synapse/server.py b/synapse/server.py index e013a349c..5fee7fe13 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -28,7 +28,7 @@ from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler -from synapse.storage import get_datastore +from synapse.storage import DataStore from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.streams.events import EventSources @@ -117,7 +117,7 @@ class HomeServer(object): def setup(self): logger.info("Setting up.") - self.datastore = get_datastore(self) + self.datastore = DataStore(self.get_db_conn(), self) logger.info("Finished setting up.") def get_ip_from_request(self, request): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c8cab45f7..eb8884230 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -61,22 +61,6 @@ logger = logging.getLogger(__name__) LAST_SEEN_GRANULARITY = 120*1000 -def get_datastore(hs): - logger.info("getting called!") - - conn = hs.get_db_conn() - try: - cur = conn.cursor() - cur.execute("SELECT MIN(stream_ordering) FROM events",) - rows = cur.fetchall() - min_token = rows[0][0] if rows and rows[0] and rows[0][0] else -1 - min_token = min(min_token, -1) - - return DataStore(conn, hs, min_token) - finally: - conn.close() - - class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, PresenceStore, TransactionStore, @@ -98,10 +82,17 @@ class DataStore(RoomMemberStore, RoomStore, EventPushActionsStore ): - def __init__(self, db_conn, hs, min_stream_token): + def __init__(self, db_conn, hs): self.hs = hs - self.min_stream_token = min_stream_token + cur = db_conn.cursor() + try: + cur.execute("SELECT MIN(stream_ordering) FROM events",) + rows = cur.fetchall() + self.min_stream_token = rows[0][0] if rows and rows[0] and rows[0][0] else -1 + self.min_stream_token = min(self.min_stream_token, -1) + finally: + cur.close() self.client_ip_last_seen = Cache( name="client_ip_last_seen", From 0935802f1e811f56121e968547fb25141ac43daa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 28 Jan 2016 14:47:03 +0000 Subject: [PATCH 188/294] Pin pynacl to 0.3.0 Something has gone wrong in the packaging of 1.* which causes it not to compile. --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index d37dc1147..a87628ed8 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -22,7 +22,7 @@ REQUIREMENTS = { "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"], "signedjson>=1.0.0": ["signedjson>=1.0.0"], - "pynacl>=0.3.0": ["nacl>=0.3.0", "nacl.bindings"], + "pynacl==0.3.0": ["nacl==0.3.0", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "Twisted>=15.1.0": ["twisted>=15.1.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], From 49c328a8927a913c4905f0c136fc6ce90e361b6e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 28 Jan 2016 14:57:40 +0000 Subject: [PATCH 189/294] Prune on fetching So we don't try to checkout a stale ref --- jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins.sh b/jenkins.sh index 804bbd49f..ed3bdf80f 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -26,7 +26,7 @@ TOX_BIN=$WORKSPACE/.tox/py27/bin if [[ ! -e .sytest-base ]]; then git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror else - (cd .sytest-base; git fetch) + (cd .sytest-base; git fetch -p) fi rm -rf sytest From e1941442d442fe62570551071edfd936304697e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 15:02:37 +0000 Subject: [PATCH 190/294] Invalidate caches properly. Remove unused arg --- synapse/storage/events.py | 9 ++++++--- synapse/storage/receipts.py | 10 ++++++---- synapse/storage/stream.py | 2 +- synapse/util/caches/room_change_cache.py | 4 ++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 80187722e..2d2270b29 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -128,9 +128,6 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) - self._events_stream_cache.room_has_changed( - None, event.room_id, stream_ordering - ) except _RollbackButIsFineException: pass @@ -213,6 +210,12 @@ class EventsStore(SQLBaseStore): for event, _ in events_and_contexts: txn.call_after(self._invalidate_get_event_cache, event.event_id) + if not backfilled: + txn.call_after( + self._events_stream_cache.room_has_changed, + event.room_id, event.internal_metadata.stream_ordering, + ) + depth_updates = {} for event, _ in events_and_contexts: if event.internal_metadata.is_outlier(): diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 7118368d9..5ffbfdec5 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -78,7 +78,7 @@ class ReceiptsStore(SQLBaseStore): if from_key: room_ids = yield self._receipts_stream_cache.get_rooms_changed( - self, room_ids, from_key + room_ids, from_key ) results = yield self._get_linearized_receipts_for_rooms( @@ -221,6 +221,11 @@ class ReceiptsStore(SQLBaseStore): # FIXME: This shouldn't invalidate the whole cache txn.call_after(self.get_linearized_receipts_for_room.invalidate_all) + txn.call_after( + self._receipts_stream_cache.room_has_changed, + room_id, stream_id + ) + # We don't want to clobber receipts for more recent events, so we # have to compare orderings of existing receipts sql = ( @@ -308,9 +313,6 @@ class ReceiptsStore(SQLBaseStore): stream_id_manager = yield self._receipts_id_gen.get_next(self) with stream_id_manager as stream_id: - yield self._receipts_stream_cache.room_has_changed( - self, room_id, stream_id - ) have_persisted = yield self.runInteraction( "insert_linearized_receipt", self.insert_linearized_receipt_txn, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 5096b4686..67e7e6a76 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -169,7 +169,7 @@ class StreamStore(SQLBaseStore): from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = yield self._events_stream_cache.get_rooms_changed( - self, room_ids, from_id + room_ids, from_id ) if not room_ids: diff --git a/synapse/util/caches/room_change_cache.py b/synapse/util/caches/room_change_cache.py index 3a873c9c3..eb2ab5f1e 100644 --- a/synapse/util/caches/room_change_cache.py +++ b/synapse/util/caches/room_change_cache.py @@ -51,7 +51,7 @@ class RoomStreamChangeCache(object): return False - def get_rooms_changed(self, store, room_ids, key): + def get_rooms_changed(self, room_ids, key): """Returns subset of room ids that have had new things since the given key. If the key is too old it will just return the given list. """ @@ -70,7 +70,7 @@ class RoomStreamChangeCache(object): return result - def room_has_changed(self, store, room_id, key): + def room_has_changed(self, room_id, key): """Informs the cache that the room has been changed at the given key. """ if key > self._earliest_known_key: From c23a8c783382a0789c757e16e104cf08654e6cf8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 15:55:26 +0000 Subject: [PATCH 191/294] Ensure keys to RoomStreamChangeCache are ints --- synapse/storage/stream.py | 11 ++++++----- synapse/util/caches/room_change_cache.py | 6 ++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 67e7e6a76..6a724193e 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -199,12 +199,13 @@ class StreamStore(SQLBaseStore): if from_key == to_key: defer.returnValue(([], from_key)) - has_changed = yield self._events_stream_cache.get_room_has_changed( - room_id, from_id - ) + if from_id: + has_changed = yield self._events_stream_cache.get_room_has_changed( + room_id, from_id + ) - if not has_changed: - defer.returnValue(([], from_key)) + if not has_changed: + defer.returnValue(([], from_key)) def f(txn): if from_id is not None: diff --git a/synapse/util/caches/room_change_cache.py b/synapse/util/caches/room_change_cache.py index eb2ab5f1e..e8bfedd72 100644 --- a/synapse/util/caches/room_change_cache.py +++ b/synapse/util/caches/room_change_cache.py @@ -39,6 +39,8 @@ class RoomStreamChangeCache(object): caches_by_name[self.name] = self._cache def get_room_has_changed(self, room_id, key): + assert type(key) is int + if key <= self._earliest_known_key: return True @@ -55,6 +57,8 @@ class RoomStreamChangeCache(object): """Returns subset of room ids that have had new things since the given key. If the key is too old it will just return the given list. """ + assert type(key) is int + if key > self._earliest_known_key: keys = self._cache.keys() i = keys.bisect_right(key) @@ -73,6 +77,8 @@ class RoomStreamChangeCache(object): def room_has_changed(self, room_id, key): """Informs the cache that the room has been changed at the given key. """ + assert type(key) is int + if key > self._earliest_known_key: old_key = self._room_to_key.get(room_id, None) if old_key: From 00cb3eb24b277bb37bd1b7d8449c08a37cb4b014 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 16:37:41 +0000 Subject: [PATCH 192/294] Cache tags and account data --- synapse/storage/account_data.py | 20 ++++- synapse/storage/events.py | 2 +- synapse/storage/receipts.py | 8 +- synapse/storage/stream.py | 8 +- synapse/storage/tags.py | 14 ++++ synapse/util/caches/room_change_cache.py | 92 --------------------- synapse/util/caches/stream_change_cache.py | 95 ++++++++++++++++++++++ 7 files changed, 137 insertions(+), 102 deletions(-) delete mode 100644 synapse/util/caches/room_change_cache.py create mode 100644 synapse/util/caches/stream_change_cache.py diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 9c6597e01..95294c3f6 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -14,6 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore +from synapse.util.caches.stream_change_cache import StreamChangeCache from twisted.internet import defer import ujson as json @@ -23,6 +24,13 @@ logger = logging.getLogger(__name__) class AccountDataStore(SQLBaseStore): + def __init__(self, hs): + super(AccountDataStore, self).__init__(hs) + + self._account_data_stream_cache = StreamChangeCache( + "AccountDataChangeCache", self._account_data_id_gen.get_max_token(None), + max_size=1000, + ) def get_account_data_for_user(self, user_id): """Get all the client account_data for a user. @@ -83,7 +91,7 @@ class AccountDataStore(SQLBaseStore): "get_account_data_for_room", get_account_data_for_room_txn ) - def get_updated_account_data_for_user(self, user_id, stream_id): + def get_updated_account_data_for_user(self, user_id, stream_id, room_ids=None): """Get all the client account_data for a that's changed. Args: @@ -120,6 +128,12 @@ class AccountDataStore(SQLBaseStore): return (global_account_data, account_data_by_room) + changed = self._account_data_stream_cache.get_entity_has_changed( + user_id, int(stream_id) + ) + if not changed: + defer.returnValue(({}, {})) + return self.runInteraction( "get_updated_account_data_for_user", get_updated_account_data_for_user_txn ) @@ -186,6 +200,10 @@ class AccountDataStore(SQLBaseStore): "content": content_json, } ) + txn.call_after( + self._account_data_stream_cache.entity_has_changed, + user_id, next_id, + ) self._update_max_stream_id(txn, next_id) with (yield self._account_data_id_gen.get_next(self)) as next_id: diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2d2270b29..5e8555202 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -212,7 +212,7 @@ class EventsStore(SQLBaseStore): if not backfilled: txn.call_after( - self._events_stream_cache.room_has_changed, + self._events_stream_cache.entity_has_changed, event.room_id, event.internal_metadata.stream_ordering, ) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 5ffbfdec5..8068c7374 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -15,7 +15,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList, cached -from synapse.util.caches.room_change_cache import RoomStreamChangeCache +from synapse.util.caches.stream_change_cache import StreamChangeCache from twisted.internet import defer @@ -30,7 +30,7 @@ class ReceiptsStore(SQLBaseStore): def __init__(self, hs): super(ReceiptsStore, self).__init__(hs) - self._receipts_stream_cache = RoomStreamChangeCache( + self._receipts_stream_cache = StreamChangeCache( "ReceiptsRoomChangeCache", self._receipts_id_gen.get_max_token(None) ) @@ -77,7 +77,7 @@ class ReceiptsStore(SQLBaseStore): room_ids = set(room_ids) if from_key: - room_ids = yield self._receipts_stream_cache.get_rooms_changed( + room_ids = yield self._receipts_stream_cache.get_entities_changed( room_ids, from_key ) @@ -222,7 +222,7 @@ class ReceiptsStore(SQLBaseStore): txn.call_after(self.get_linearized_receipts_for_room.invalidate_all) txn.call_after( - self._receipts_stream_cache.room_has_changed, + self._receipts_stream_cache.entity_has_changed, room_id, stream_id ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 6a724193e..c7d789332 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -37,7 +37,7 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks -from synapse.util.caches.room_change_cache import RoomStreamChangeCache +from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.api.constants import EventTypes from synapse.types import RoomStreamToken from synapse.util.logutils import log_function @@ -81,7 +81,7 @@ class StreamStore(SQLBaseStore): def __init__(self, hs): super(StreamStore, self).__init__(hs) - self._events_stream_cache = RoomStreamChangeCache( + self._events_stream_cache = StreamChangeCache( "EventsRoomStreamChangeCache", self._stream_id_gen.get_max_token(None) ) @@ -168,7 +168,7 @@ class StreamStore(SQLBaseStore): def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0): from_id = RoomStreamToken.parse_stream_token(from_key).stream - room_ids = yield self._events_stream_cache.get_rooms_changed( + room_ids = yield self._events_stream_cache.get_entities_changed( room_ids, from_id ) @@ -200,7 +200,7 @@ class StreamStore(SQLBaseStore): defer.returnValue(([], from_key)) if from_id: - has_changed = yield self._events_stream_cache.get_room_has_changed( + has_changed = yield self._events_stream_cache.get_entity_has_changed( room_id, from_id ) diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index 4c39e07cb..50af89919 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -15,6 +15,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached +from synapse.util.caches.stream_change_cache import StreamChangeCache from twisted.internet import defer import ujson as json @@ -24,6 +25,13 @@ logger = logging.getLogger(__name__) class TagsStore(SQLBaseStore): + def __init__(self, hs): + super(TagsStore, self).__init__(hs) + + self._tags_stream_cache = StreamChangeCache( + "TagsChangeCache", self._account_data_id_gen.get_max_token(None), + max_size=1000, + ) def get_max_account_data_stream_id(self): """Get the current max stream id for the private user data stream @@ -80,6 +88,10 @@ class TagsStore(SQLBaseStore): room_ids = [row[0] for row in txn.fetchall()] return room_ids + changed = self._tags_stream_cache.get_entity_has_changed(user_id, int(stream_id)) + if not changed: + defer.returnValue({}) + room_ids = yield self.runInteraction( "get_updated_tags", get_updated_tags_txn ) @@ -177,6 +189,8 @@ class TagsStore(SQLBaseStore): next_id(int): The the revision to advance to. """ + txn.call_after(self._tags_stream_cache.entity_has_changed, user_id, next_id) + update_max_id_sql = ( "UPDATE account_data_max_stream_id" " SET stream_id = ?" diff --git a/synapse/util/caches/room_change_cache.py b/synapse/util/caches/room_change_cache.py deleted file mode 100644 index e8bfedd72..000000000 --- a/synapse/util/caches/room_change_cache.py +++ /dev/null @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.util.caches import cache_counter, caches_by_name - - -from blist import sorteddict -import logging - - -logger = logging.getLogger(__name__) - - -class RoomStreamChangeCache(object): - """Keeps track of the stream_id of the latest change in rooms. - - Given a list of rooms and stream key, it will give a subset of rooms that - may have changed since that key. If the key is too old then the cache - will simply return all rooms. - """ - def __init__(self, name, current_key, size_of_cache=10000): - self._size_of_cache = size_of_cache - self._room_to_key = {} - self._cache = sorteddict() - self._earliest_known_key = current_key - self.name = name - caches_by_name[self.name] = self._cache - - def get_room_has_changed(self, room_id, key): - assert type(key) is int - - if key <= self._earliest_known_key: - return True - - room_key = self._room_to_key.get(room_id, None) - if room_key is None: - return True - - if key < room_key: - return True - - return False - - def get_rooms_changed(self, room_ids, key): - """Returns subset of room ids that have had new things since the - given key. If the key is too old it will just return the given list. - """ - assert type(key) is int - - if key > self._earliest_known_key: - keys = self._cache.keys() - i = keys.bisect_right(key) - - result = set( - self._cache[k] for k in keys[i:] - ).intersection(room_ids) - - cache_counter.inc_hits(self.name) - else: - result = room_ids - cache_counter.inc_misses(self.name) - - return result - - def room_has_changed(self, room_id, key): - """Informs the cache that the room has been changed at the given key. - """ - assert type(key) is int - - if key > self._earliest_known_key: - old_key = self._room_to_key.get(room_id, None) - if old_key: - key = max(key, old_key) - self._cache.pop(old_key, None) - self._cache[key] = room_id - - while len(self._cache) > self._size_of_cache: - k, r = self._cache.popitem() - self._earliest_key = max(k, self._earliest_key) - self._room_to_key.pop(r, None) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py new file mode 100644 index 000000000..33b37f7f2 --- /dev/null +++ b/synapse/util/caches/stream_change_cache.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.util.caches import cache_counter, caches_by_name + + +from blist import sorteddict +import logging + + +logger = logging.getLogger(__name__) + + +class StreamChangeCache(object): + """Keeps track of the stream positions of the latest change in a set of entities. + + Typically the entity will be a room or user id. + + Given a list of entities and a stream position, it will give a subset of + entities that may have changed since that position. If position key is too + old then the cache will simply return all given entities. + """ + def __init__(self, name, current_stream_pos, max_size=10000): + self._max_size = max_size + self._entity_to_key = {} + self._cache = sorteddict() + self._earliest_known_stream_pos = current_stream_pos + self.name = name + caches_by_name[self.name] = self._cache + + def get_entity_has_changed(self, entity, stream_pos): + assert type(stream_pos) is int + + if stream_pos <= self._earliest_known_stream_pos: + return True + + latest_entity_change_pos = self._entity_to_key.get(entity, None) + if latest_entity_change_pos is None: + return True + + if stream_pos < latest_entity_change_pos: + return True + + return False + + def get_entities_changed(self, entities, stream_pos): + """Returns subset of entities that have had new things since the + given position. If the position is too old it will just return the given list. + """ + assert type(stream_pos) is int + + if stream_pos > self._earliest_known_stream_pos: + keys = self._cache.keys() + i = keys.bisect_right(stream_pos) + + result = set( + self._cache[k] for k in keys[i:] + ).intersection(entities) + + cache_counter.inc_hits(self.name) + else: + result = entities + cache_counter.inc_misses(self.name) + + return result + + def entity_has_changed(self, entitiy, stream_pos): + """Informs the cache that the entitiy has been changed at the given + position. + """ + assert type(stream_pos) is int + + if stream_pos > self._earliest_known_stream_pos: + old_pos = self._entity_to_key.get(entitiy, None) + if old_pos: + stream_pos = max(stream_pos, old_pos) + self._cache.pop(old_pos, None) + self._cache[stream_pos] = entitiy + + while len(self._cache) > self._max_size: + k, r = self._cache.popitem() + self._earliest_known_stream_pos = max(k, self._earliest_known_stream_pos) + self._entity_to_key.pop(r, None) From 45cf827c8fe7163a51f1d0d7c9e2531da9b58c8d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 16:39:18 +0000 Subject: [PATCH 193/294] Change name and doc has_entity_changed --- synapse/storage/account_data.py | 2 +- synapse/storage/stream.py | 2 +- synapse/storage/tags.py | 2 +- synapse/util/caches/stream_change_cache.py | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 95294c3f6..62e49e1c0 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -128,7 +128,7 @@ class AccountDataStore(SQLBaseStore): return (global_account_data, account_data_by_room) - changed = self._account_data_stream_cache.get_entity_has_changed( + changed = self._account_data_stream_cache.has_entity_changed( user_id, int(stream_id) ) if not changed: diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index c7d789332..6e81d46c6 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -200,7 +200,7 @@ class StreamStore(SQLBaseStore): defer.returnValue(([], from_key)) if from_id: - has_changed = yield self._events_stream_cache.get_entity_has_changed( + has_changed = yield self._events_stream_cache.has_entity_changed( room_id, from_id ) diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index 50af89919..75ce04092 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -88,7 +88,7 @@ class TagsStore(SQLBaseStore): room_ids = [row[0] for row in txn.fetchall()] return room_ids - changed = self._tags_stream_cache.get_entity_has_changed(user_id, int(stream_id)) + changed = self._tags_stream_cache.has_entity_changed(user_id, int(stream_id)) if not changed: defer.returnValue({}) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 33b37f7f2..3ca0e5778 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -40,7 +40,9 @@ class StreamChangeCache(object): self.name = name caches_by_name[self.name] = self._cache - def get_entity_has_changed(self, entity, stream_pos): + def has_entity_changed(self, entity, stream_pos): + """Returns True if the entity may have been updated since stream_pos + """ assert type(stream_pos) is int if stream_pos <= self._earliest_known_stream_pos: From fdca8ec4187e1b4ea93cbfe17ece6ca4cbadd519 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 16:41:59 +0000 Subject: [PATCH 194/294] Add events index --- .../schema/delta/28/events_room_stream.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 synapse/storage/schema/delta/28/events_room_stream.sql diff --git a/synapse/storage/schema/delta/28/events_room_stream.sql b/synapse/storage/schema/delta/28/events_room_stream.sql new file mode 100644 index 000000000..200c35e6e --- /dev/null +++ b/synapse/storage/schema/delta/28/events_room_stream.sql @@ -0,0 +1,16 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +CREATE INDEX events_room_stream on events(room_id, stream_ordering); From 8fe8951a8d997a652d16758a7abed2b8afd117e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 17:09:09 +0000 Subject: [PATCH 195/294] Cache filters --- synapse/storage/filtering.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index f8fc9bddd..524873681 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -16,12 +16,13 @@ from twisted.internet import defer from ._base import SQLBaseStore +from synapse.util.caches.descriptors import cachedInlineCallbacks import simplejson as json class FilteringStore(SQLBaseStore): - @defer.inlineCallbacks + @cachedInlineCallbacks(num_args=2) def get_user_filter(self, user_localpart, filter_id): def_json = yield self._simple_select_one_onecol( table="user_filters", From 35981c8b71a2ce675f3b8414ca0e7920a5d1658e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 17:19:51 +0000 Subject: [PATCH 196/294] Fix test --- synapse/api/filtering.py | 5 +++++ tests/api/test_filtering.py | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 6c13ada5d..6eff83e5f 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -15,6 +15,8 @@ from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID +import ujson as json + class Filtering(object): @@ -149,6 +151,9 @@ class FilterCollection(object): "include_leave", False ) + def __repr__(self): + return "" % (json.dumps(self._filter_json),) + def get_filter_json(self): return self._filter_json diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 1a4e439d3..ceb008926 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -382,19 +382,20 @@ class FilteringTestCase(unittest.TestCase): "types": ["m.*"] } } - user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( - user_localpart=user_localpart, + user_localpart=user_localpart + "2", user_filter=user_filter_json, ) event = MockEvent( + event_id="$asdasd:localhost", sender="@foo:bar", type="custom.avatar.3d.crazy", ) events = [event] user_filter = yield self.filtering.get_user_filter( - user_localpart=user_localpart, + user_localpart=user_localpart + "2", filter_id=filter_id, ) From 0663c5bd52d52e095258e312fb62c2a9cb3f200a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 17:27:28 +0000 Subject: [PATCH 197/294] Include cache hits with has_entity_changed --- synapse/util/caches/stream_change_cache.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 3ca0e5778..b48f5cb27 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -46,15 +46,19 @@ class StreamChangeCache(object): assert type(stream_pos) is int if stream_pos <= self._earliest_known_stream_pos: + cache_counter.inc_misses(self.name) return True latest_entity_change_pos = self._entity_to_key.get(entity, None) if latest_entity_change_pos is None: + cache_counter.inc_misses(self.name) return True if stream_pos < latest_entity_change_pos: + cache_counter.inc_misses(self.name) return True + cache_counter.inc_hits(self.name) return False def get_entities_changed(self, entities, stream_pos): From 03b2c2577cbf51c80a42319689e1cb4903b8c4af Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 17:29:24 +0000 Subject: [PATCH 198/294] Don't use defer.returnValue --- synapse/storage/account_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 62e49e1c0..88404059e 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -132,7 +132,7 @@ class AccountDataStore(SQLBaseStore): user_id, int(stream_id) ) if not changed: - defer.returnValue(({}, {})) + return ({}, {}) return self.runInteraction( "get_updated_account_data_for_user", get_updated_account_data_for_user_txn From 82cf3a8043a6ffe1f57d78bf88eba06ad0c53fbe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 17:44:04 +0000 Subject: [PATCH 199/294] Fix inequalities --- synapse/util/caches/stream_change_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index b48f5cb27..483e0cdf9 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -45,7 +45,7 @@ class StreamChangeCache(object): """ assert type(stream_pos) is int - if stream_pos <= self._earliest_known_stream_pos: + if stream_pos < self._earliest_known_stream_pos: cache_counter.inc_misses(self.name) return True @@ -67,7 +67,7 @@ class StreamChangeCache(object): """ assert type(stream_pos) is int - if stream_pos > self._earliest_known_stream_pos: + if stream_pos >= self._earliest_known_stream_pos: keys = self._cache.keys() i = keys.bisect_right(stream_pos) From 40431251cba7ef1b623559db972600ece40818a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 18:05:43 +0000 Subject: [PATCH 200/294] Correctly update _entity_to_key --- synapse/util/caches/stream_change_cache.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 483e0cdf9..22a9f8f46 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -82,18 +82,19 @@ class StreamChangeCache(object): return result - def entity_has_changed(self, entitiy, stream_pos): - """Informs the cache that the entitiy has been changed at the given + def entity_has_changed(self, entity, stream_pos): + """Informs the cache that the entity has been changed at the given position. """ assert type(stream_pos) is int if stream_pos > self._earliest_known_stream_pos: - old_pos = self._entity_to_key.get(entitiy, None) + old_pos = self._entity_to_key.get(entity, None) if old_pos: stream_pos = max(stream_pos, old_pos) self._cache.pop(old_pos, None) - self._cache[stream_pos] = entitiy + self._cache[stream_pos] = entity + self._entity_to_key[entity] = stream_pos while len(self._cache) > self._max_size: k, r = self._cache.popitem() From 3f5dd18bd44ae426d3b1ff062dd64acbad72f8ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 18:11:41 +0000 Subject: [PATCH 201/294] If the same as the earliest key, assume nothing has changed. --- synapse/util/caches/stream_change_cache.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 22a9f8f46..c673b1bdf 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -49,6 +49,11 @@ class StreamChangeCache(object): cache_counter.inc_misses(self.name) return True + if stream_pos == self._earliest_known_stream_pos: + # If the same as the earliest key, assume nothing has changed. + cache_counter.inc_hits(self.name) + return False + latest_entity_change_pos = self._entity_to_key.get(entity, None) if latest_entity_change_pos is None: cache_counter.inc_misses(self.name) From 467c27a1f90b873d6838ad1351399551cfa9cc07 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Jan 2016 18:20:00 +0000 Subject: [PATCH 202/294] Amalgamate tags and account data stream caches --- synapse/storage/account_data.py | 3 ++- synapse/storage/tags.py | 18 +++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 88404059e..822c8bbe0 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -28,7 +28,8 @@ class AccountDataStore(SQLBaseStore): super(AccountDataStore, self).__init__(hs) self._account_data_stream_cache = StreamChangeCache( - "AccountDataChangeCache", self._account_data_id_gen.get_max_token(None), + "AccountDataAndTagsChangeCache", + self._account_data_id_gen.get_max_token(None), max_size=1000, ) diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index 75ce04092..e1a9c0c26 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -15,7 +15,6 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached -from synapse.util.caches.stream_change_cache import StreamChangeCache from twisted.internet import defer import ujson as json @@ -25,14 +24,6 @@ logger = logging.getLogger(__name__) class TagsStore(SQLBaseStore): - def __init__(self, hs): - super(TagsStore, self).__init__(hs) - - self._tags_stream_cache = StreamChangeCache( - "TagsChangeCache", self._account_data_id_gen.get_max_token(None), - max_size=1000, - ) - def get_max_account_data_stream_id(self): """Get the current max stream id for the private user data stream @@ -88,7 +79,9 @@ class TagsStore(SQLBaseStore): room_ids = [row[0] for row in txn.fetchall()] return room_ids - changed = self._tags_stream_cache.has_entity_changed(user_id, int(stream_id)) + changed = self._account_data_stream_cache.has_entity_changed( + user_id, int(stream_id) + ) if not changed: defer.returnValue({}) @@ -189,7 +182,10 @@ class TagsStore(SQLBaseStore): next_id(int): The the revision to advance to. """ - txn.call_after(self._tags_stream_cache.entity_has_changed, user_id, next_id) + txn.call_after( + self._account_data_stream_cache.entity_has_changed, + user_id, next_id + ) update_max_id_sql = ( "UPDATE account_data_max_stream_id" From 50e18938a96583b2dc8cbc69aa7182a35e3435af Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 10:00:45 +0000 Subject: [PATCH 203/294] Reset size on clear --- synapse/util/caches/lrucache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index e6a66dc04..5182cadb9 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -37,6 +37,7 @@ class LruCache(object): """ def __init__(self, max_size, keylen=1, cache_type=dict): cache = cache_type() + self.cache = cache # Used for introspection. self.size = 0 list_root = [] list_root[:] = [list_root, list_root, None, None] @@ -142,6 +143,7 @@ class LruCache(object): list_root[NEXT] = list_root list_root[PREV] = list_root cache.clear() + self.size = 0 @synchronized def cache_len(): From 766526e1142e7ad0ffb43bd075b0ff2d6265e4cb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 10:11:21 +0000 Subject: [PATCH 204/294] Make TreeCache keep track of its own size. --- synapse/util/caches/lrucache.py | 9 +++------ synapse/util/caches/treecache.py | 7 +++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 5182cadb9..ca9ffdf1b 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -38,7 +38,6 @@ class LruCache(object): def __init__(self, max_size, keylen=1, cache_type=dict): cache = cache_type() self.cache = cache # Used for introspection. - self.size = 0 list_root = [] list_root[:] = [list_root, list_root, None, None] @@ -61,7 +60,6 @@ class LruCache(object): prev_node[NEXT] = node next_node[PREV] = node cache[key] = node - self.size += 1 def move_node_to_front(node): prev_node = node[PREV] @@ -80,7 +78,6 @@ class LruCache(object): next_node = node[NEXT] prev_node[NEXT] = next_node next_node[PREV] = prev_node - self.size -= 1 @synchronized def cache_get(key, default=None): @@ -99,7 +96,7 @@ class LruCache(object): node[VALUE] = value else: add_node(key, value) - if self.size > max_size: + if len(cache) > max_size: todelete = list_root[PREV] delete_node(todelete) cache.pop(todelete[KEY], None) @@ -111,7 +108,7 @@ class LruCache(object): return node[VALUE] else: add_node(key, value) - if self.size > max_size: + if len(cache) > max_size: todelete = list_root[PREV] delete_node(todelete) cache.pop(todelete[KEY], None) @@ -147,7 +144,7 @@ class LruCache(object): @synchronized def cache_len(): - return self.size + return len(cache) @synchronized def cache_contains(key): diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index 3b5886091..a29ea8144 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -8,6 +8,7 @@ class TreeCache(object): Keys must be tuples. """ def __init__(self): + self.size = 0 self.root = {} def __setitem__(self, key, value): @@ -21,6 +22,7 @@ class TreeCache(object): for k in key[:-1]: node = node.setdefault(k, {}) node[key[-1]] = value + self.size += 1 def get(self, key, default=None): node = self.root @@ -31,6 +33,7 @@ class TreeCache(object): return node.get(key[-1], default) def clear(self): + self.size = 0 self.root = {} def pop(self, key, default=None): @@ -57,4 +60,8 @@ class TreeCache(object): break node_and_keys[i+1][0].pop(k) + self.size -= 1 return popped + + def __len__(self): + return self.size From a30364c1f99bdd7d5cb0fe82ebdfe52d996defef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 10:44:46 +0000 Subject: [PATCH 205/294] Correctly bookkeep the size of TreeCache --- synapse/util/caches/treecache.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index a29ea8144..3331ea9eb 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -21,7 +21,7 @@ class TreeCache(object): node = self.root for k in key[:-1]: node = node.setdefault(k, {}) - node[key[-1]] = value + node[key[-1]] = _Entry(value) self.size += 1 def get(self, key, default=None): @@ -30,7 +30,7 @@ class TreeCache(object): node = node.get(k, None) if node is None: return default - return node.get(key[-1], default) + return node.get(key[-1], _Entry(default)).value def clear(self): self.size = 0 @@ -60,8 +60,33 @@ class TreeCache(object): break node_and_keys[i+1][0].pop(k) - self.size -= 1 + popped, cnt = _strip_and_count_entires(popped) + self.size -= cnt return popped def __len__(self): return self.size + + +class _Entry(object): + __slots__ = ["value"] + + def __init__(self, value): + object.__setattr__(self, "value", value) + + +def _strip_and_count_entires(d): + """Takes an _Entry or dict with leaves of _Entry's, and either returns the + value or a dictionary with _Entry's replaced by their values. + + Also returns the count of _Entry's + """ + if isinstance(d, dict): + cnt = 0 + for key, value in d.items(): + v, n = _strip_and_count_entires(value) + d[key] = v + cnt += n + return d, cnt + else: + return d.value, 1 From c046630c330b5abc4403f15afa2ffb13b7b8aa08 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 11:17:54 +0000 Subject: [PATCH 206/294] Remove spurious self.size --- synapse/util/caches/lrucache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index ca9ffdf1b..f7423f2fa 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -140,7 +140,6 @@ class LruCache(object): list_root[NEXT] = list_root list_root[PREV] = list_root cache.clear() - self.size = 0 @synchronized def cache_len(): From fb7299800feb0b8bc0f8429fc5b840abea0609d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 11:29:14 +0000 Subject: [PATCH 207/294] Directly set self.value --- synapse/util/caches/treecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index 3331ea9eb..29d02f7e9 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -72,7 +72,7 @@ class _Entry(object): __slots__ = ["value"] def __init__(self, value): - object.__setattr__(self, "value", value) + self.value = value def _strip_and_count_entires(d): From 4fce59f2747d9c73784470e84396a09ae70bbda7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 11:33:11 +0000 Subject: [PATCH 208/294] Add tests --- tests/util/test_lrucache.py | 7 +++++++ tests/util/test_treecache.py | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index 2cd3d2645..bab366fb7 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -19,6 +19,7 @@ from .. import unittest from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache + class LruCacheTestCase(unittest.TestCase): def test_get_set(self): @@ -72,3 +73,9 @@ class LruCacheTestCase(unittest.TestCase): self.assertEquals(cache.get(("vehicles", "car")), "vroom") self.assertEquals(cache.get(("vehicles", "train")), "chuff") # Man from del_multi say "Yes". + + def test_clear(self): + cache = LruCache(1) + cache["key"] = 1 + cache.clear() + self.assertEquals(len(cache), 0) diff --git a/tests/util/test_treecache.py b/tests/util/test_treecache.py index 9946ceb3f..1efbeb6b3 100644 --- a/tests/util/test_treecache.py +++ b/tests/util/test_treecache.py @@ -25,6 +25,7 @@ class TreeCacheTestCase(unittest.TestCase): cache[("b",)] = "B" self.assertEquals(cache.get(("a",)), "A") self.assertEquals(cache.get(("b",)), "B") + self.assertEquals(len(cache), 2) def test_pop_onelevel(self): cache = TreeCache() @@ -33,6 +34,7 @@ class TreeCacheTestCase(unittest.TestCase): self.assertEquals(cache.pop(("a",)), "A") self.assertEquals(cache.pop(("a",)), None) self.assertEquals(cache.get(("b",)), "B") + self.assertEquals(len(cache), 1) def test_get_set_twolevel(self): cache = TreeCache() @@ -42,6 +44,7 @@ class TreeCacheTestCase(unittest.TestCase): self.assertEquals(cache.get(("a", "a")), "AA") self.assertEquals(cache.get(("a", "b")), "AB") self.assertEquals(cache.get(("b", "a")), "BA") + self.assertEquals(len(cache), 3) def test_pop_twolevel(self): cache = TreeCache() @@ -53,6 +56,7 @@ class TreeCacheTestCase(unittest.TestCase): self.assertEquals(cache.get(("a", "b")), "AB") self.assertEquals(cache.pop(("b", "a")), "BA") self.assertEquals(cache.pop(("b", "a")), None) + self.assertEquals(len(cache), 1) def test_pop_mixedlevel(self): cache = TreeCache() @@ -64,3 +68,11 @@ class TreeCacheTestCase(unittest.TestCase): self.assertEquals(cache.get(("a", "a")), None) self.assertEquals(cache.get(("a", "b")), None) self.assertEquals(cache.get(("b", "a")), "BA") + self.assertEquals(len(cache), 1) + + def test_clear(self): + cache = TreeCache() + cache[("a",)] = "A" + cache[("b",)] = "B" + cache.clear() + self.assertEquals(len(cache), 0) From 5687a00e4ec24b78e41b0377bfe524d7f0b13a43 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 29 Jan 2016 13:26:15 +0000 Subject: [PATCH 209/294] Allow three_pid_creds as well as threePidCreds in /account/3pid --- synapse/rest/client/v2_alpha/account.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index d50717270..a614b79d4 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -116,9 +116,10 @@ class ThreepidRestServlet(RestServlet): body = parse_json_dict_from_request(request) - if 'threePidCreds' not in body: + threePidCreds = body.get('threePidCreds') + threePidCreds = body.get('three_pid_creds', threePidCreds) + if threePidCreds is None: raise SynapseError(400, "Missing param", Codes.MISSING_PARAM) - threePidCreds = body['threePidCreds'] requester = yield self.auth.get_user_by_req(request) user_id = requester.user.to_string() From ea320d34641f98c7aa477e2340ff265c1ac63419 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 13:34:48 +0000 Subject: [PATCH 210/294] Don't work out unread_notifs_for_room_id unless needed --- synapse/handlers/sync.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 51ec4702d..075566417 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -697,17 +697,6 @@ class SyncHandler(BaseHandler): if just_joined: state = yield self.get_state_at(room_id, now_token) - notifs = yield self.unread_notifs_for_room_id( - room_id, sync_config, all_ephemeral_by_room - ) - - unread_notifications = {} - if notifs is not None: - unread_notifications["notification_count"] = len(notifs) - unread_notifications["highlight_count"] = len([ - 1 for notif in notifs if _action_has_highlight(notif["actions"]) - ]) - state = { (e.type, e.state_key): e for e in sync_config.filter_collection.filter_room_state(state.values()) @@ -725,6 +714,7 @@ class SyncHandler(BaseHandler): ephemeral_by_room.get(room_id, []) ) + unread_notifications = {} room_sync = JoinedSyncResult( room_id=room_id, timeline=batch, @@ -734,6 +724,17 @@ class SyncHandler(BaseHandler): unread_notifications=unread_notifications, ) + if room_sync: + notifs = yield self.unread_notifs_for_room_id( + room_id, sync_config, all_ephemeral_by_room + ) + + if notifs is not None: + unread_notifications["notification_count"] = len(notifs) + unread_notifications["highlight_count"] = len([ + 1 for notif in notifs if _action_has_highlight(notif["actions"]) + ]) + logger.debug("Room sync: %r", room_sync) defer.returnValue(room_sync) From ebc5f00efed5c7f72601933f55032947077c50a0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 13:37:40 +0000 Subject: [PATCH 211/294] Bump AccountDataAndTagsChangeCache size --- synapse/storage/account_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 822c8bbe0..ed6587429 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -30,7 +30,7 @@ class AccountDataStore(SQLBaseStore): self._account_data_stream_cache = StreamChangeCache( "AccountDataAndTagsChangeCache", self._account_data_id_gen.get_max_token(None), - max_size=1000, + max_size=10000, ) def get_account_data_for_user(self, user_id): From 96bb4bf38ab9fe204c956b3d3e88ccb19acc1728 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 13:56:22 +0000 Subject: [PATCH 212/294] Update CHANGES --- CHANGES.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index dd4863c3e..b9c956fd0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.12.1-rc1 (2016-01-xx) +Changes in synapse v0.12.1-rc1 (2016-01-29) =========================================== Features: @@ -18,8 +18,8 @@ Changes: * Change to require unbanning before other membership changes (PR #501) * Change default push rules to notify for all messages (PR #486) * Change default push rules to not notify on membership changes (PR #514) -* Change default push rules to only notify messages in one to one rooms - (PR #529) +* Change default push rules in one to one rooms to only notify for events that + are messages (PR #529) * Change ``/sync`` to reject requests with a ``from`` query param (PR #512) * Change server manhole to use SSH rather than telnet (PR #473) * Change server to require AS users to be registered before use (PR #487) @@ -39,7 +39,7 @@ Bug fixes: * Fix bug where ``/events`` would always return immediately for guest users (PR #480) * Fix bug where ``/sync`` unexpectedly returned old left rooms (PR #481) -* Fix enabling and disabling push rules (P #498) +* Fix enabling and disabling push rules (PR #498) * Fix bug where ``/register`` returned 500 when given unicode username (PR #513) From 0fcafbece8258105fa4e81be4657ecc36359d258 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 29 Jan 2016 14:12:26 +0000 Subject: [PATCH 213/294] Add config option for setting the trusted id servers, disabling checking the ID server in integration tests --- synapse/config/registration.py | 7 +++++++ synapse/handlers/identity.py | 25 +++++++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index d3f4b9d54..76d2d2d64 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -34,6 +34,7 @@ class RegistrationConfig(Config): self.registration_shared_secret = config.get("registration_shared_secret") self.macaroon_secret_key = config.get("macaroon_secret_key") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) + self.trusted_third_party_id_servers = config["trusted_third_party_id_servers"] self.allow_guest_access = config.get("allow_guest_access", False) def default_config(self, **kwargs): @@ -60,6 +61,12 @@ class RegistrationConfig(Config): # participate in rooms hosted on this server which have been made # accessible to anonymous users. allow_guest_access: False + + # The list of identity servers trusted to verify third party + # identifiers by this server. + trusted_third_party_id_servers: + - matrix.org + - vector.im """ % locals() def add_arguments(self, parser): diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 819ec57c4..77f133be8 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -36,14 +36,15 @@ class IdentityHandler(BaseHandler): self.http_client = hs.get_simple_http_client() + self.trusted_id_servers = set(hs.config.trusted_third_party_id_servers) + self.trust_any_id_server_just_for_testing_do_not_use = ( + hs.config.use_insecure_ssl_client_just_for_testing_do_not_use + ) + @defer.inlineCallbacks def threepid_from_creds(self, creds): yield run_on_reactor() - # XXX: make this configurable! - # trustedIdServers = ['matrix.org', 'localhost:8090'] - trustedIdServers = ['matrix.org', 'vector.im'] - if 'id_server' in creds: id_server = creds['id_server'] elif 'idServer' in creds: @@ -58,10 +59,18 @@ class IdentityHandler(BaseHandler): else: raise SynapseError(400, "No client_secret in creds") - if id_server not in trustedIdServers: - logger.warn('%s is not a trusted ID server: rejecting 3pid ' + - 'credentials', id_server) - defer.returnValue(None) + if id_server not in self.trusted_id_servers: + if self.trust_any_id_server_just_for_testing_do_not_use: + logger.warn( + "Trusting untrustworthy ID server %r even though it isn't" + " in the trusted id list for testing because" + " 'use_insecure_ssl_client_just_for_testing_do_not_use'" + " is set in the config" + ) + else: + logger.warn('%s is not a trusted ID server: rejecting 3pid ' + + 'credentials', id_server) + defer.returnValue(None) data = {} try: From 18579534ea67f2d98c189e2ddeccc4bfecb491eb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 14:37:59 +0000 Subject: [PATCH 214/294] Prefill stream change caches --- synapse/storage/__init__.py | 49 +++++++++++++++++++++- synapse/storage/account_data.py | 9 ---- synapse/storage/stream.py | 8 ---- synapse/util/caches/stream_change_cache.py | 5 ++- 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index eb8884230..95ae97d50 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -45,9 +45,10 @@ from .search import SearchStore from .tags import TagsStore from .account_data import AccountDataStore - from util.id_generators import IdGenerator, StreamIdGenerator +from synapse.util.caches.stream_change_cache import StreamChangeCache + import logging @@ -117,8 +118,54 @@ class DataStore(RoomMemberStore, RoomStore, self._push_rule_id_gen = IdGenerator("push_rules", "id", self) self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) + events_max = self._stream_id_gen.get_max_token(None) + event_cache_prefill = self._get_cache_dict( + db_conn, "events", + entity_column="room_id", + stream_column="stream_ordering", + max_value=events_max, + ) + self._events_stream_cache = StreamChangeCache( + "EventsRoomStreamChangeCache", events_max, + prefilled_cache=event_cache_prefill, + ) + + account_max = self._account_data_id_gen.get_max_token(None) + account_cache_prefill = self._get_cache_dict( + db_conn, "account_data", + entity_column="user_id", + stream_column="stream_id", + max_value=account_max, + ) + self._account_data_stream_cache = StreamChangeCache( + "AccountDataAndTagsChangeCache", account_max, + prefilled_cache=account_cache_prefill, + ) + super(DataStore, self).__init__(hs) + def _get_cache_dict(self, db_conn, table, entity_column, stream_column, max_value): + sql = ( + "SELECT %(entity)s, MAX(%(stream)s) FROM %(table)s" + " WHERE %(stream)s > max(? - 100000, 0)" + " GROUP BY %(entity)s" + " ORDER BY MAX(%(stream)s) DESC" + " LIMIT 10000" + ) % { + "table": table, + "entity": entity_column, + "stream": stream_column, + } + + txn = db_conn.cursor() + txn.execute(sql, (int(max_value),)) + rows = txn.fetchall() + + return { + row[0]: row[1] + for row in rows + } + @defer.inlineCallbacks def insert_client_ip(self, user, access_token, ip, user_agent): now = int(self._clock.time_msec()) diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index ed6587429..625d062eb 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -14,7 +14,6 @@ # limitations under the License. from ._base import SQLBaseStore -from synapse.util.caches.stream_change_cache import StreamChangeCache from twisted.internet import defer import ujson as json @@ -24,14 +23,6 @@ logger = logging.getLogger(__name__) class AccountDataStore(SQLBaseStore): - def __init__(self, hs): - super(AccountDataStore, self).__init__(hs) - - self._account_data_stream_cache = StreamChangeCache( - "AccountDataAndTagsChangeCache", - self._account_data_id_gen.get_max_token(None), - max_size=10000, - ) def get_account_data_for_user(self, user_id): """Get all the client account_data for a user. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 6e81d46c6..e245d2f91 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -37,7 +37,6 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks -from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.api.constants import EventTypes from synapse.types import RoomStreamToken from synapse.util.logutils import log_function @@ -78,13 +77,6 @@ def upper_bound(token): class StreamStore(SQLBaseStore): - def __init__(self, hs): - super(StreamStore, self).__init__(hs) - - self._events_stream_cache = StreamChangeCache( - "EventsRoomStreamChangeCache", self._stream_id_gen.get_max_token(None) - ) - @defer.inlineCallbacks def get_appservice_room_stream(self, service, from_key, to_key, limit=0): # NB this lives here instead of appservice.py so we can reuse the diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index c673b1bdf..891cb619f 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -32,7 +32,7 @@ class StreamChangeCache(object): entities that may have changed since that position. If position key is too old then the cache will simply return all given entities. """ - def __init__(self, name, current_stream_pos, max_size=10000): + def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}): self._max_size = max_size self._entity_to_key = {} self._cache = sorteddict() @@ -40,6 +40,9 @@ class StreamChangeCache(object): self.name = name caches_by_name[self.name] = self._cache + for entity, stream_pos in prefilled_cache.items(): + self.entity_has_changed(entity, stream_pos) + def has_entity_changed(self, entity, stream_pos): """Returns True if the entity may have been updated since stream_pos """ From f67d60496a8a9b2c95fcacb6d4c539a1d4b6a105 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 14:41:16 +0000 Subject: [PATCH 215/294] Convert param style --- synapse/storage/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 95ae97d50..2ed505cb1 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -85,6 +85,7 @@ class DataStore(RoomMemberStore, RoomStore, def __init__(self, db_conn, hs): self.hs = hs + self.database_engine = hs.database_engine cur = db_conn.cursor() try: @@ -157,6 +158,8 @@ class DataStore(RoomMemberStore, RoomStore, "stream": stream_column, } + sql = self.database_engine.convert_param_style(sql) + txn = db_conn.cursor() txn.execute(sql, (int(max_value),)) rows = txn.fetchall() From 45488e0ffae5100c3a82568642736aff203e1602 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 14:42:01 +0000 Subject: [PATCH 216/294] Max is not a function --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2ed505cb1..4d374a8b0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -148,7 +148,7 @@ class DataStore(RoomMemberStore, RoomStore, def _get_cache_dict(self, db_conn, table, entity_column, stream_column, max_value): sql = ( "SELECT %(entity)s, MAX(%(stream)s) FROM %(table)s" - " WHERE %(stream)s > max(? - 100000, 0)" + " WHERE %(stream)s > ? - 100000" " GROUP BY %(entity)s" " ORDER BY MAX(%(stream)s) DESC" " LIMIT 10000" From 3d60686c0ceeb88c4f6269110e92dc0c7bf5a3b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 14:49:11 +0000 Subject: [PATCH 217/294] Actually use cache --- synapse/storage/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4d374a8b0..957fff3c2 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -120,26 +120,26 @@ class DataStore(RoomMemberStore, RoomStore, self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) events_max = self._stream_id_gen.get_max_token(None) - event_cache_prefill = self._get_cache_dict( + event_cache_prefill, min_event_val = self._get_cache_dict( db_conn, "events", entity_column="room_id", stream_column="stream_ordering", max_value=events_max, ) self._events_stream_cache = StreamChangeCache( - "EventsRoomStreamChangeCache", events_max, + "EventsRoomStreamChangeCache", min_event_val, prefilled_cache=event_cache_prefill, ) account_max = self._account_data_id_gen.get_max_token(None) - account_cache_prefill = self._get_cache_dict( + account_cache_prefill, min_acc_val = self._get_cache_dict( db_conn, "account_data", entity_column="user_id", stream_column="stream_id", max_value=account_max, ) self._account_data_stream_cache = StreamChangeCache( - "AccountDataAndTagsChangeCache", account_max, + "AccountDataAndTagsChangeCache", min_acc_val, prefilled_cache=account_cache_prefill, ) @@ -151,7 +151,6 @@ class DataStore(RoomMemberStore, RoomStore, " WHERE %(stream)s > ? - 100000" " GROUP BY %(entity)s" " ORDER BY MAX(%(stream)s) DESC" - " LIMIT 10000" ) % { "table": table, "entity": entity_column, @@ -164,11 +163,18 @@ class DataStore(RoomMemberStore, RoomStore, txn.execute(sql, (int(max_value),)) rows = txn.fetchall() - return { - row[0]: row[1] + cache = { + row[0]: int(row[1]) for row in rows } + if cache: + min_val = min(cache.values()) + else: + min_val = max_value + + return cache, min_val + @defer.inlineCallbacks def insert_client_ip(self, user, access_token, ip, user_agent): now = int(self._clock.time_msec()) From f2d5ff5bf2cb95eb0a6619ae7fb40603175c8a7d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 29 Jan 2016 14:53:14 +0000 Subject: [PATCH 218/294] Fix the mock homserver used in the tests --- tests/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils.py b/tests/utils.py index 43cc2b30c..431252a6f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -49,6 +49,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.disable_registration = False config.macaroon_secret_key = "not even a little secret" config.server_name = "server.under.test" + config.trusted_third_party_id_servers = [] if "clock" not in kargs: kargs["clock"] = MockClock() From b5dbced9389d072d4bd15002c7ddffba9e54340e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 14:53:59 +0000 Subject: [PATCH 219/294] Don't prefill account data --- synapse/storage/__init__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 957fff3c2..a6cb58856 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -132,15 +132,8 @@ class DataStore(RoomMemberStore, RoomStore, ) account_max = self._account_data_id_gen.get_max_token(None) - account_cache_prefill, min_acc_val = self._get_cache_dict( - db_conn, "account_data", - entity_column="user_id", - stream_column="stream_id", - max_value=account_max, - ) self._account_data_stream_cache = StreamChangeCache( - "AccountDataAndTagsChangeCache", min_acc_val, - prefilled_cache=account_cache_prefill, + "AccountDataAndTagsChangeCache", account_max, ) super(DataStore, self).__init__(hs) From 6927d0e091b4b6124bd45a627c7f299f2544c5b4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 29 Jan 2016 15:01:26 +0000 Subject: [PATCH 220/294] Add missing param to the log line --- synapse/handlers/identity.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 77f133be8..656ce124f 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -65,7 +65,8 @@ class IdentityHandler(BaseHandler): "Trusting untrustworthy ID server %r even though it isn't" " in the trusted id list for testing because" " 'use_insecure_ssl_client_just_for_testing_do_not_use'" - " is set in the config" + " is set in the config", + id_server, ) else: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + From 4a6eb5eb45c8c77c02a38b9cd3448174a1cf018a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 15:21:38 +0000 Subject: [PATCH 221/294] Make /events always return a newer token, if one exists --- synapse/notifier.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 6eaa65071..acd4019a9 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -22,6 +22,8 @@ from synapse.util.async import run_on_reactor, ObservableDeferred from synapse.types import StreamToken import synapse.metrics +from collections import namedtuple + import logging @@ -118,6 +120,11 @@ class _NotifierUserStream(object): return _NotificationListener(self.notify_deferred.observe()) +class EventStreamResult(namedtuple("EventStreamResult", ("events", "tokens"))): + def __nonzero__(self): + return bool(self.events) + + class Notifier(object): """ This class is responsible for notifying any listeners when there are new events available for it. @@ -356,7 +363,7 @@ class Notifier(object): @defer.inlineCallbacks def check_for_updates(before_token, after_token): if not after_token.is_after(before_token): - defer.returnValue(None) + defer.returnValue(EventStreamResult([], (before_token, before_token))) events = [] end_token = from_token @@ -372,7 +379,7 @@ class Notifier(object): new_events, new_key = yield source.get_new_events( user=user, from_key=getattr(from_token, keyname), - limit=limit, + limit=max(limit * 2, 10), is_guest=is_peeking, room_ids=room_ids, ) @@ -388,10 +395,7 @@ class Notifier(object): events.extend(new_events) end_token = end_token.copy_and_replace(keyname, new_key) - if events: - defer.returnValue((events, (from_token, end_token))) - else: - defer.returnValue(None) + defer.returnValue(EventStreamResult(events, (from_token, end_token))) user_id_for_stream = user.to_string() if is_peeking: @@ -415,9 +419,6 @@ class Notifier(object): from_token=from_token, ) - if result is None: - result = ([], (from_token, from_token)) - defer.returnValue(result) @defer.inlineCallbacks From 13724569eccbd2431f46868099afd44128af19d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 15:33:44 +0000 Subject: [PATCH 222/294] Deal with None limit --- synapse/notifier.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index acd4019a9..32bd16661 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -376,10 +376,14 @@ class Notifier(object): continue if only_keys and name not in only_keys: continue + if limit: + new_limit = max(limit * 2, 10) + else: + new_limit = 10 new_events, new_key = yield source.get_new_events( user=user, from_key=getattr(from_token, keyname), - limit=max(limit * 2, 10), + limit=new_limit, is_guest=is_peeking, room_ids=room_ids, ) From 8da95b6f1bb1a37597f0b89c4da88b064401b0b8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 15:39:17 +0000 Subject: [PATCH 223/294] Comment. Remove superfluous order by --- synapse/storage/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a6cb58856..ee2153737 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -139,11 +139,13 @@ class DataStore(RoomMemberStore, RoomStore, super(DataStore, self).__init__(hs) def _get_cache_dict(self, db_conn, table, entity_column, stream_column, max_value): + # Fetch a mapping of room_id -> max stream position for "recent" rooms. + # It doesn't really matter how many we get, the StreamChangeCache will + # do the right thing to ensure it respects the max size of cache. sql = ( "SELECT %(entity)s, MAX(%(stream)s) FROM %(table)s" " WHERE %(stream)s > ? - 100000" " GROUP BY %(entity)s" - " ORDER BY MAX(%(stream)s) DESC" ) % { "table": table, "entity": entity_column, From e70165039cd718af0d4aca46df8857e805c39e28 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 16:24:34 +0000 Subject: [PATCH 224/294] If stream pos is greater then earliest known key and entity hasn't changed, then entity hasn't changed --- synapse/util/caches/stream_change_cache.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 891cb619f..b37f1c072 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -52,15 +52,10 @@ class StreamChangeCache(object): cache_counter.inc_misses(self.name) return True - if stream_pos == self._earliest_known_stream_pos: - # If the same as the earliest key, assume nothing has changed. - cache_counter.inc_hits(self.name) - return False - latest_entity_change_pos = self._entity_to_key.get(entity, None) if latest_entity_change_pos is None: - cache_counter.inc_misses(self.name) - return True + cache_counter.inc_hits(self.name) + return False if stream_pos < latest_entity_change_pos: cache_counter.inc_misses(self.name) @@ -98,7 +93,7 @@ class StreamChangeCache(object): if stream_pos > self._earliest_known_stream_pos: old_pos = self._entity_to_key.get(entity, None) - if old_pos: + if old_pos is not None: stream_pos = max(stream_pos, old_pos) self._cache.pop(old_pos, None) self._cache[stream_pos] = entity From cc9c97e0dc0cf399d5d6013f12746063091b619e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 16:41:51 +0000 Subject: [PATCH 225/294] Invalidate _account_data_stream_cache correctly --- synapse/storage/account_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 625d062eb..b8387fc50 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -157,6 +157,10 @@ class AccountDataStore(SQLBaseStore): "content": content_json, } ) + txn.call_after( + self._account_data_stream_cache.entity_has_changed, + user_id, next_id, + ) self._update_max_stream_id(txn, next_id) with (yield self._account_data_id_gen.get_next(self)) as next_id: From 25c311eaf603cef8cbf9e6501aad83d53c304ebb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Jan 2016 16:52:48 +0000 Subject: [PATCH 226/294] Cache get_room_changes_for_user --- synapse/storage/__init__.py | 4 ++++ synapse/storage/roommember.py | 4 ++++ synapse/storage/stream.py | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ee2153737..c91c7a372 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -131,6 +131,10 @@ class DataStore(RoomMemberStore, RoomStore, prefilled_cache=event_cache_prefill, ) + self._membership_stream_cache = StreamChangeCache( + "MembershipStreamChangeCache", events_max, + ) + account_max = self._account_data_id_gen.get_max_token(None) self._account_data_stream_cache = StreamChangeCache( "AccountDataAndTagsChangeCache", account_max, diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 1d3e004c9..3065b0c1a 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -58,6 +58,10 @@ class RoomMemberStore(SQLBaseStore): txn.call_after(self.get_rooms_for_user.invalidate, (event.state_key,)) txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,)) txn.call_after(self.get_users_in_room.invalidate, (event.room_id,)) + txn.call_after( + self._membership_stream_cache.entity_has_changed, + event.state_key, event.internal_metadata.stream_ordering + ) def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e245d2f91..cc9e62360 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -252,6 +252,13 @@ class StreamStore(SQLBaseStore): if from_key == to_key: return defer.succeed([]) + if from_id: + has_changed = self._membership_stream_cache.has_entity_changed( + user_id, int(from_id) + ) + if not has_changed: + return defer.succeed([]) + def f(txn): if from_id is not None: sql = ( From d98a9f25839ede491201714dc7a9d661cd4ff8f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sun, 31 Jan 2016 13:31:15 +0000 Subject: [PATCH 227/294] Don't use before_token. Its wrong. Use actual limit. --- synapse/notifier.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 32bd16661..29965a9ab 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -363,7 +363,7 @@ class Notifier(object): @defer.inlineCallbacks def check_for_updates(before_token, after_token): if not after_token.is_after(before_token): - defer.returnValue(EventStreamResult([], (before_token, before_token))) + defer.returnValue(EventStreamResult([], (from_token, from_token))) events = [] end_token = from_token @@ -376,14 +376,11 @@ class Notifier(object): continue if only_keys and name not in only_keys: continue - if limit: - new_limit = max(limit * 2, 10) - else: - new_limit = 10 + new_events, new_key = yield source.get_new_events( user=user, from_key=getattr(from_token, keyname), - limit=new_limit, + limit=limit, is_guest=is_peeking, room_ids=room_ids, ) From ceb6b8680a8e419c3c132dcdb675517c5e9f69fd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Feb 2016 10:33:52 +0000 Subject: [PATCH 228/294] Only use room_ids if in get_room_events_stream if is_guest --- synapse/storage/stream.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e245d2f91..a60e662f7 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -316,11 +316,6 @@ class StreamStore(SQLBaseStore): " WHERE m.user_id = ? AND m.membership = 'join'" ) current_room_membership_args = [user_id] - if room_ids: - current_room_membership_sql += " AND m.room_id in (%s)" % ( - ",".join(map(lambda _: "?", room_ids)) - ) - current_room_membership_args = [user_id] + room_ids # We also want to get any membership events about that user, e.g. # invites or leave notifications. From fa48020a52dfb924be4f191d54631b4d05b08ba0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Feb 2016 15:59:40 +0000 Subject: [PATCH 229/294] Move state calculations from rest to handler --- synapse/handlers/sync.py | 162 ++++++++++++++++----------- synapse/rest/client/v2_alpha/sync.py | 75 ------------- 2 files changed, 97 insertions(+), 140 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 075566417..3109e3041 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -23,6 +23,7 @@ from twisted.internet import defer import collections import logging +import itertools logger = logging.getLogger(__name__) @@ -672,35 +673,10 @@ class SyncHandler(BaseHandler): account_data_by_room, all_ephemeral_by_room, batch, full_state=False): - if full_state: - state = yield self.get_state_at(room_id, now_token) - - elif batch.limited: - current_state = yield self.get_state_at(room_id, now_token) - - state_at_previous_sync = yield self.get_state_at( - room_id, stream_position=since_token - ) - - state = yield self.compute_state_delta( - since_token=since_token, - previous_state=state_at_previous_sync, - current_state=current_state, - ) - else: - state = { - (event.type, event.state_key): event - for event in batch.events if event.is_state() - } - - just_joined = yield self.check_joined_room(sync_config, state) - if just_joined: - state = yield self.get_state_at(room_id, now_token) - - state = { - (e.type, e.state_key): e - for e in sync_config.filter_collection.filter_room_state(state.values()) - } + state = yield self.compute_state_delta( + room_id, batch, sync_config, since_token, now_token, + full_state=full_state + ) account_data = self.account_data_for_room( room_id, tags_by_room, account_data_by_room @@ -766,30 +742,11 @@ class SyncHandler(BaseHandler): logger.debug("Recents %r", batch) - state_events_at_leave = yield self.store.get_state_for_event( - leave_event_id + state_events_delta = yield self.compute_state_delta( + room_id, batch, sync_config, since_token, leave_token, + full_state=full_state ) - if not full_state: - state_at_previous_sync = yield self.get_state_at( - room_id, stream_position=since_token - ) - - state_events_delta = yield self.compute_state_delta( - since_token=since_token, - previous_state=state_at_previous_sync, - current_state=state_events_at_leave, - ) - else: - state_events_delta = state_events_at_leave - - state_events_delta = { - (e.type, e.state_key): e - for e in sync_config.filter_collection.filter_room_state( - state_events_delta.values() - ) - } - account_data = self.account_data_for_room( room_id, tags_by_room, account_data_by_room ) @@ -843,15 +800,18 @@ class SyncHandler(BaseHandler): state = {} defer.returnValue(state) - def compute_state_delta(self, since_token, previous_state, current_state): - """ Works out the differnce in state between the current state and the - state the client got when it last performed a sync. + @defer.inlineCallbacks + def compute_state_delta(self, room_id, batch, sync_config, since_token, now_token, + full_state): + """ Works out the differnce in state between the start of the timeline + and the previous sync. - :param str since_token: the point we are comparing against - :param dict[(str,str), synapse.events.FrozenEvent] previous_state: the - state to compare to - :param dict[(str,str), synapse.events.FrozenEvent] current_state: the - new state + :param str room_id + :param TimelineBatch batch + :param sync_config + :param str since_token + :param str now_token + :param bool full_state :returns A new event dictionary """ @@ -860,12 +820,50 @@ class SyncHandler(BaseHandler): # updates even if they occured logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - state_delta = {} - for key, event in current_state.iteritems(): - if (key not in previous_state or - previous_state[key].event_id != event.event_id): - state_delta[key] = event - return state_delta + if full_state: + if batch: + state = yield self.store.get_state_for_event(batch.events[0].event_id) + else: + state = yield self.get_state_at( + room_id, stream_position=now_token + ) + + timeline_state = { + (event.type, event.state_key): event + for event in batch.events if event.is_state() + } + + state = _calculate_state( + timeline_contains=timeline_state, + timeline_start=state, + previous={}, + ) + elif batch.limited: + state_at_previous_sync = yield self.get_state_at( + room_id, stream_position=since_token + ) + + state_at_timeline_start = yield self.store.get_state_for_event( + batch.events[0].event_id + ) + + timeline_state = { + (event.type, event.state_key): event + for event in batch.events if event.is_state() + } + + state = _calculate_state( + timeline_contains=timeline_state, + timeline_start=state_at_timeline_start, + previous=state_at_previous_sync, + ) + else: + state = {} + + defer.returnValue({ + (e.type, e.state_key): e + for e in sync_config.filter_collection.filter_room_state(state.values()) + }) def check_joined_room(self, sync_config, state_delta): """ @@ -912,3 +910,37 @@ def _action_has_highlight(actions): pass return False + + +def _calculate_state(timeline_contains, timeline_start, previous): + """Works out what state to include in a sync response. + + Args: + timeline_contains (dict): state in the timeline + timeline_start (dict): state at the start of the timeline + previous (dict): state at the end of the previous sync (or empty dict + if there is an initial sync) + + Returns: + dict + """ + event_id_to_state = { + e.event_id: e + for e in itertools.chain( + timeline_contains.values(), + previous.values(), + timeline_start.values(), + ) + } + + tc_ids = set(e.event_id for e in timeline_contains.values()) + p_ids = set(e.event_id for e in previous.values()) + ts_ids = set(e.event_id for e in timeline_start.values()) + + state_ids = (ts_ids - p_ids) - tc_ids + + evs = (event_id_to_state[e] for e in state_ids) + return { + (e.type, e.state_key): e + for e in evs + } diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 07b5b5dfd..140ce2704 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -20,7 +20,6 @@ from synapse.http.servlet import ( ) from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken -from synapse.events import FrozenEvent from synapse.events.utils import ( serialize_event, format_event_for_client_v2_without_room_id, ) @@ -287,9 +286,6 @@ class SyncRestServlet(RestServlet): state_dict = room.state timeline_events = room.timeline.events - state_dict = SyncRestServlet._rollback_state_for_timeline( - state_dict, timeline_events) - state_events = state_dict.values() serialized_state = [serialize(e) for e in state_events] @@ -314,77 +310,6 @@ class SyncRestServlet(RestServlet): return result - @staticmethod - def _rollback_state_for_timeline(state, timeline): - """ - Wind the state dictionary backwards, so that it represents the - state at the start of the timeline, rather than at the end. - - :param dict[(str, str), synapse.events.EventBase] state: the - state dictionary. Will be updated to the state before the timeline. - :param list[synapse.events.EventBase] timeline: the event timeline - :return: updated state dictionary - """ - - result = state.copy() - - for timeline_event in reversed(timeline): - if not timeline_event.is_state(): - continue - - event_key = (timeline_event.type, timeline_event.state_key) - - logger.debug("Considering %s for removal", event_key) - - state_event = result.get(event_key) - if (state_event is None or - state_event.event_id != timeline_event.event_id): - # the event in the timeline isn't present in the state - # dictionary. - # - # the most likely cause for this is that there was a fork in - # the event graph, and the state is no longer valid. Really, - # the event shouldn't be in the timeline. We're going to ignore - # it for now, however. - logger.debug("Found state event %r in timeline which doesn't " - "match state dictionary", timeline_event) - continue - - prev_event_id = timeline_event.unsigned.get("replaces_state", None) - - prev_content = timeline_event.unsigned.get('prev_content') - prev_sender = timeline_event.unsigned.get('prev_sender') - # Empircally it seems possible for the event to have a - # "replaces_state" key but not a prev_content or prev_sender - # markjh conjectures that it could be due to the server not - # having a copy of that event. - # If this is the case the we ignore the previous event. This will - # cause the displayname calculations on the client to be incorrect - if prev_event_id is None or not prev_content or not prev_sender: - logger.debug( - "Removing %r from the state dict, as it is missing" - " prev_content (prev_event_id=%r)", - timeline_event.event_id, prev_event_id - ) - del result[event_key] - else: - logger.debug( - "Replacing %r with %r in state dict", - timeline_event.event_id, prev_event_id - ) - result[event_key] = FrozenEvent({ - "type": timeline_event.type, - "state_key": timeline_event.state_key, - "content": prev_content, - "sender": prev_sender, - "event_id": prev_event_id, - "room_id": timeline_event.room_id, - }) - - logger.debug("New value: %r", result.get(event_key)) - - return result - def register_servlets(hs, http_server): SyncRestServlet(hs).register(http_server) From 4bf448be254808c83aeb5ae28e601752664bc9e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Feb 2016 16:26:51 +0000 Subject: [PATCH 230/294] Switch over /events to use per room caches --- synapse/handlers/room.py | 25 ++++++++++++++++++++----- synapse/storage/stream.py | 4 ++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 58e2d25f9..aca795e1c 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1008,15 +1008,30 @@ class RoomEventSource(object): limit=limit, ) else: - events, end_key = yield self.store.get_room_events_stream( - user_id=user.to_string(), + room_events = yield self.store.get_room_changes_for_user( + user.to_string(), from_key, to_key + ) + + room_to_events = yield self.store.get_room_events_stream_for_rooms( + room_ids=room_ids, from_key=from_key, to_key=to_key, - limit=limit, - room_ids=room_ids, - is_guest=is_guest, + limit=limit or 10, ) + events = list(room_events) + events.extend(e for evs, _ in room_to_events.values() for e in evs) + + events.sort(key=lambda e: e.internal_metadata.after) + + if limit: + events[:] = events[:limit] + + if events: + end_key = events[-1].internal_metadata.after + else: + end_key = to_key + defer.returnValue((events, end_key)) def get_current_key(self, direction='f'): diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8dc8f5c64..fd84aa899 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -39,7 +39,6 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.api.constants import EventTypes from synapse.types import RoomStreamToken -from synapse.util.logutils import log_function import logging @@ -288,11 +287,12 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(ret, rows, topo_order=False) + return ret return self.runInteraction("get_room_changes_for_user", f) - @log_function def get_room_events_stream( self, user_id, From 89b40b225cda4326081f6735b2a8a9bff5ce3446 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Feb 2016 16:32:46 +0000 Subject: [PATCH 231/294] Order things correctly --- synapse/handlers/room.py | 2 +- synapse/storage/stream.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index aca795e1c..a71cba8ef 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1022,7 +1022,7 @@ class RoomEventSource(object): events = list(room_events) events.extend(e for evs, _ in room_to_events.values() for e in evs) - events.sort(key=lambda e: e.internal_metadata.after) + events.sort(key=lambda e: e.internal_metadata.order) if limit: events[:] = events[:limit] diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index fd84aa899..a03458c2f 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -598,6 +598,10 @@ class StreamStore(SQLBaseStore): internal = event.internal_metadata internal.before = str(RoomStreamToken(topo, stream - 1)) internal.after = str(RoomStreamToken(topo, stream)) + internal.order = ( + int(topo) if topo else 0, + int(stream), + ) @defer.inlineCallbacks def get_events_around(self, room_id, event_id, before_limit, after_limit): From d7ac861d3b34bbc6d0d0b66370903831ef6ee9bf Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 1 Feb 2016 16:33:19 +0000 Subject: [PATCH 232/294] Pull guest access token out of the auth session params, otherwise it will break if you open the email on a different device. --- synapse/rest/client/v2_alpha/register.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index c4d025b46..5d50dd9e3 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -152,6 +152,7 @@ class RegisterRestServlet(RestServlet): desired_username = params.get("username", None) new_password = params.get("password", None) + guest_access_token = params.get("guest_access_token", None) (user_id, token) = yield self.registration_handler.register( localpart=desired_username, From 854ca32f10f6e1c8041160d4866470098c7f6fc1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Feb 2016 16:52:27 +0000 Subject: [PATCH 233/294] Comments --- synapse/handlers/sync.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3109e3041..8d8d10da3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -807,11 +807,12 @@ class SyncHandler(BaseHandler): and the previous sync. :param str room_id - :param TimelineBatch batch + :param TimelineBatch batch: The timeline batch for the room that will + be sent to the user. :param sync_config - :param str since_token - :param str now_token - :param bool full_state + :param str since_token: Token of the end of the previous batch. May be None. + :param str now_token: Token of the end of the current batch. + :param bool full_state: Whether to force returning the full state. :returns A new event dictionary """ @@ -919,7 +920,7 @@ def _calculate_state(timeline_contains, timeline_start, previous): timeline_contains (dict): state in the timeline timeline_start (dict): state at the start of the timeline previous (dict): state at the end of the previous sync (or empty dict - if there is an initial sync) + if this is an initial sync) Returns: dict From b023995538ae2ed13934638009463abcbae08f92 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Feb 2016 14:11:14 +0000 Subject: [PATCH 234/294] WARN if we get a topo token instead of stream. --- synapse/handlers/room.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a71cba8ef..68e2c75a4 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -18,7 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.types import UserID, RoomAlias, RoomID +from synapse.types import UserID, RoomAlias, RoomID, RoomStreamToken from synapse.api.constants import ( EventTypes, Membership, JoinRules, RoomCreationPreset, ) @@ -997,6 +997,11 @@ class RoomEventSource(object): to_key = yield self.get_current_key() + from_token = RoomStreamToken.parse(from_key) + if from_token.topological: + logger.warn("Stream has topological part!!!! %r", from_key) + from_key = "s%s" % (from_token.stream,) + app_service = yield self.store.get_app_service_by_user_id( user.to_string() ) From 69214ea671d3e219d53698cc608c8e4d414cc3f7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 2 Feb 2016 14:42:31 +0000 Subject: [PATCH 235/294] Pass make_guest whne we autogen a user ID --- synapse/handlers/register.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c11b98d0b..abd1a16a4 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -139,7 +139,9 @@ class RegistrationHandler(BaseHandler): yield self.store.register( user_id=user_id, token=token, - password_hash=password_hash) + password_hash=password_hash, + make_guest=make_guest + ) yield registered_user(self.distributor, user) except SynapseError: From 65e92eca4912848b03f71b7b7d29727015be31ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Feb 2016 15:19:34 +0000 Subject: [PATCH 236/294] Change the way we do public room list fetching --- synapse/handlers/room.py | 80 ++++++++++++++----- synapse/storage/room.py | 2 +- .../schema/delta/28/public_roms_index.sql | 16 ++++ 3 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 synapse/storage/schema/delta/28/public_roms_index.sql diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a71cba8ef..1b3f624c6 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -876,31 +876,71 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def get_public_room_list(self): - chunk = yield self.store.get_rooms(is_public=True) + room_ids = yield self.store.get_public_room_ids() - room_members = yield defer.gatherResults( - [ - self.store.get_users_in_room(room["room_id"]) - for room in chunk - ], - consumeErrors=True, - ).addErrback(unwrapFirstError) + @defer.inlineCallbacks + def handle_room(room_id): + aliases = yield self.store.get_aliases_for_room(room_id) + if not aliases: + defer.returnValue(None) - avatar_urls = yield defer.gatherResults( - [ - self.get_room_avatar_url(room["room_id"]) - for room in chunk - ], - consumeErrors=True, - ).addErrback(unwrapFirstError) + state = yield self.state_handler.get_current_state(room_id) - for i, room in enumerate(chunk): - room["num_joined_members"] = len(room_members[i]) - if avatar_urls[i]: - room["avatar_url"] = avatar_urls[i] + result = {"aliases": aliases, "room_id": room_id} + + name_event = state.get((EventTypes.Name, ""), None) + if name_event: + name = name_event.content.get("name", None) + if name: + result["name"] = name + + topic_event = state.get((EventTypes.Topic, ""), None) + if topic_event: + topic = topic_event.content.get("topic", None) + if topic: + result["topic"] = topic + + canonical_event = state.get((EventTypes.CanonicalAlias, ""), None) + if canonical_event: + canonical_alias = canonical_event.content.get("alias", None) + if canonical_alias: + result["canonical_alias"] = canonical_alias + + visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None) + visibility = None + if visibility_event: + visibility = visibility_event.content.get("history_visibility", None) + result["world_readable"] = visibility == "world_readable" + + guest_event = state.get((EventTypes.GuestAccess, ""), None) + guest = None + if guest_event: + guest = guest_event.content.get("guest_access", None) + result["guest_can_join"] = guest == "can_join" + + avatar_event = state.get(("m.room.avatar", ""), None) + if avatar_event: + avatar_url = avatar_event.content.get("url", None) + if avatar_url: + result["avatar_url"] = avatar_url + + result["num_joined_members"] = sum( + 1 for (event_type, _), ev in state.items() + if event_type == EventTypes.Member and ev.membership == Membership.JOIN + ) + + defer.returnValue(result) + + result = [] + for chunk in (room_ids[i:i+10] for i in xrange(0, len(room_ids), 10)): + chunk_result = yield defer.gatherResults([ + handle_room(room_id) + for room_id in chunk + ], consumeErrors=True).addErrback(unwrapFirstError) + result.extend(v for v in chunk_result if v) # FIXME (erikj): START is no longer a valid value - defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) + defer.returnValue({"start": "START", "end": "END", "chunk": result}) @defer.inlineCallbacks def get_room_avatar_url(self, room_id): diff --git a/synapse/storage/room.py b/synapse/storage/room.py index dc09a3aab..1b6311f33 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cachedInlineCallbacks +from synapse.util.caches.descriptors import cachedInlineCallbacks, cached from .engines import PostgresEngine, Sqlite3Engine import collections diff --git a/synapse/storage/schema/delta/28/public_roms_index.sql b/synapse/storage/schema/delta/28/public_roms_index.sql new file mode 100644 index 000000000..ba62a974a --- /dev/null +++ b/synapse/storage/schema/delta/28/public_roms_index.sql @@ -0,0 +1,16 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +CREATE INDEX public_room_index on rooms(is_public); From 477b1ed6cfd130e5a004cda0c0b84509da2aa006 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Feb 2016 15:58:14 +0000 Subject: [PATCH 237/294] Fetch events in a separate transaction. This has a couple of benefits: - It reduces the time of transactions, allowing other database requests to run. - Fetching events is given a dedicated database thread, and so can't starve other database requests. --- synapse/storage/stream.py | 55 +++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index a03458c2f..bcae3d718 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -220,27 +220,29 @@ class StreamStore(SQLBaseStore): rows = self.cursor_to_dict(txn) - ret = self._get_events_txn( - txn, - [r["event_id"] for r in rows], - get_prev_content=True - ) + return rows - self._set_before_and_after(ret, rows, topo_order=False) + rows = yield self.runInteraction("get_room_events_stream_for_room", f) - ret.reverse() + ret = yield self._get_events( + [r["event_id"] for r in rows], + get_prev_content=True + ) - if rows: - key = "s%d" % min(r["stream_ordering"] for r in rows) - else: - # Assume we didn't get anything because there was nothing to - # get. - key = from_key + self._set_before_and_after(ret, rows, topo_order=False) - return ret, key - res = yield self.runInteraction("get_room_events_stream_for_room", f) - defer.returnValue(res) + ret.reverse() + if rows: + key = "s%d" % min(r["stream_ordering"] for r in rows) + else: + # Assume we didn't get anything because there was nothing to + # get. + key = from_key + + defer.returnValue((ret, key)) + + @defer.inlineCallbacks def get_room_changes_for_user(self, user_id, from_key, to_key): if from_key is not None: from_id = RoomStreamToken.parse_stream_token(from_key).stream @@ -249,14 +251,14 @@ class StreamStore(SQLBaseStore): to_id = RoomStreamToken.parse_stream_token(to_key).stream if from_key == to_key: - return defer.succeed([]) + defer.returnValue([]) if from_id: has_changed = self._membership_stream_cache.has_entity_changed( user_id, int(from_id) ) if not has_changed: - return defer.succeed([]) + defer.returnValue([]) def f(txn): if from_id is not None: @@ -281,17 +283,18 @@ class StreamStore(SQLBaseStore): txn.execute(sql, (user_id, to_id,)) rows = self.cursor_to_dict(txn) - ret = self._get_events_txn( - txn, - [r["event_id"] for r in rows], - get_prev_content=True - ) + return rows - self._set_before_and_after(ret, rows, topo_order=False) + rows = yield self.runInteraction("get_room_changes_for_user", f) - return ret + ret = yield self._get_events( + [r["event_id"] for r in rows], + get_prev_content=True + ) - return self.runInteraction("get_room_changes_for_user", f) + self._set_before_and_after(ret, rows, topo_order=False) + + defer.returnValue(ret) def get_room_events_stream( self, From 8a391e33ae21f9a62c57cca8eea47435a14a6247 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Feb 2016 16:12:10 +0000 Subject: [PATCH 238/294] s/get_room_changes_for_user/get_membership_changes_for_user/ --- synapse/handlers/room.py | 2 +- synapse/handlers/sync.py | 2 +- synapse/storage/stream.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 68e2c75a4..799221c19 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1013,7 +1013,7 @@ class RoomEventSource(object): limit=limit, ) else: - room_events = yield self.store.get_room_changes_for_user( + room_events = yield self.store.get_membership_changes_for_user( user.to_string(), from_key, to_key ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 8d8d10da3..dc686db54 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -479,7 +479,7 @@ class SyncHandler(BaseHandler): ) # Get a list of membership change events that have happened. - rooms_changed = yield self.store.get_room_changes_for_user( + rooms_changed = yield self.store.get_membership_changes_for_user( user_id, since_token.room_key, now_token.room_key ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index bcae3d718..338a9d40d 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -243,7 +243,7 @@ class StreamStore(SQLBaseStore): defer.returnValue((ret, key)) @defer.inlineCallbacks - def get_room_changes_for_user(self, user_id, from_key, to_key): + def get_membership_changes_for_user(self, user_id, from_key, to_key): if from_key is not None: from_id = RoomStreamToken.parse_stream_token(from_key).stream else: @@ -285,7 +285,7 @@ class StreamStore(SQLBaseStore): return rows - rows = yield self.runInteraction("get_room_changes_for_user", f) + rows = yield self.runInteraction("get_membership_changes_for_user", f) ret = yield self._get_events( [r["event_id"] for r in rows], From d83d004ccdb7ace1dcb51b8acf7645bc176b10a5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 2 Feb 2016 17:18:50 +0000 Subject: [PATCH 239/294] Fix flake8 warnings for new flake8 --- setup.cfg | 1 + synapse/api/auth.py | 2 +- synapse/app/__init__.py | 19 +++++++++++ synapse/app/homeserver.py | 40 +++++++----------------- synapse/appservice/api.py | 2 +- synapse/federation/federation_client.py | 2 +- synapse/handlers/_base.py | 2 +- synapse/handlers/directory.py | 4 +-- synapse/handlers/events.py | 2 +- synapse/handlers/presence.py | 2 +- synapse/handlers/register.py | 2 +- synapse/handlers/room.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/notifier.py | 2 +- synapse/push/push_rule_evaluator.py | 2 +- synapse/rest/client/v1/login.py | 2 +- synapse/rest/client/v1/pusher.py | 4 +-- synapse/rest/client/v1/register.py | 3 +- synapse/rest/client/v2_alpha/register.py | 3 +- synapse/rest/client/versions.py | 4 +-- synapse/server.py | 2 +- synapse/state.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 7 +++-- synapse/storage/engines/sqlite3.py | 2 +- synapse/storage/event_federation.py | 2 +- synapse/storage/events.py | 6 ++-- synapse/storage/stream.py | 2 +- synapse/util/__init__.py | 2 +- synapse/util/caches/descriptors.py | 4 +-- synapse/util/caches/expiringcache.py | 2 +- synapse/util/caches/treecache.py | 2 +- synapse/util/logutils.py | 2 +- synapse/util/ratelimitutils.py | 2 +- 34 files changed, 74 insertions(+), 67 deletions(-) diff --git a/setup.cfg b/setup.cfg index ba027c7d1..e7fc5ffe7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,3 +16,4 @@ ignore = [flake8] max-line-length = 90 +ignore = W503 diff --git a/synapse/api/auth.py b/synapse/api/auth.py index b5536e856..c5a2865e2 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -574,7 +574,7 @@ class Auth(object): raise AuthError( 403, "Application service has not registered this user" - ) + ) defer.returnValue(user_id) @defer.inlineCallbacks diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index bfebb0f64..1bc427980 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -12,3 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import sys +sys.dont_write_bytecode = True + +from synapse.python_dependencies import ( + check_requirements, MissingRequirementError +) # NOQA + +try: + check_requirements() +except MissingRequirementError as e: + message = "\n".join([ + "Missing Requirement: %s" % (e.message,), + "To install run:", + " pip install --upgrade --force \"%s\"" % (e.dependency,), + "", + ]) + sys.stderr.writelines(message) + sys.exit(1) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e5066c48e..c3066d6a0 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -14,27 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -from synapse.rest import ClientRestResource +import synapse + +import contextlib +import logging +import os +import re +import resource +import subprocess +import sys +import time -sys.dont_write_bytecode = True from synapse.python_dependencies import ( - check_requirements, DEPENDENCY_LINKS, MissingRequirementError + check_requirements, DEPENDENCY_LINKS ) -if __name__ == '__main__': - try: - check_requirements() - except MissingRequirementError as e: - message = "\n".join([ - "Missing Requirement: %s" % (e.message,), - "To install run:", - " pip install --upgrade --force \"%s\"" % (e.dependency,), - "", - ]) - sys.stderr.writelines(message) - sys.exit(1) - +from synapse.rest import ClientRestResource from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import are_all_users_on_domain from synapse.storage.prepare_database import UpgradeDatabaseException @@ -73,17 +68,6 @@ from synapse import events from daemonize import Daemonize -import synapse - -import contextlib -import logging -import os -import re -import resource -import subprocess -import time - - logger = logging.getLogger("synapse.app.homeserver") diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index e1c07028e..bc9060532 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -29,7 +29,7 @@ class ApplicationServiceApi(SimpleHttpClient): pushing. """ - def __init__(self, hs): + def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) self.clock = hs.get_clock() diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index c6259f9dc..e30e2da58 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -57,7 +57,7 @@ class FederationClient(FederationBase): cache_name="get_pdu_cache", clock=self._clock, max_len=1000, - expiry_ms=120*1000, + expiry_ms=120 * 1000, reset_expiry_on_get=False, ) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 744a9ee50..1423df6cf 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -147,7 +147,7 @@ class BaseHandler(object): ) if not allowed: raise LimitExceededError( - retry_after_ms=int(1000*(time_allowed - time_now)), + retry_after_ms=int(1000 * (time_allowed - time_now)), ) @defer.inlineCallbacks diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 691564c65..4efecb1ff 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -175,8 +175,8 @@ class DirectoryHandler(BaseHandler): # If this server is in the list of servers, return it first. if self.server_name in servers: servers = ( - [self.server_name] - + [s for s in servers if s != self.server_name] + [self.server_name] + + [s for s in servers if s != self.server_name] ) else: servers = list(servers) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 254b483da..5ad8f3779 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -130,7 +130,7 @@ class EventStreamHandler(BaseHandler): # Add some randomness to this value to try and mitigate against # thundering herds on restart. - timeout = random.randint(int(timeout*0.9), int(timeout*1.1)) + timeout = random.randint(int(timeout * 0.9), int(timeout * 1.1)) events, tokens = yield self.notifier.get_events_for( auth_user, pagin_config, timeout, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index d36eb3b8d..d0c21ff5c 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -34,7 +34,7 @@ metrics = synapse.metrics.get_metrics_for(__name__) # Don't bother bumping "last active" time if it differs by less than 60 seconds -LAST_ACTIVE_GRANULARITY = 60*1000 +LAST_ACTIVE_GRANULARITY = 60 * 1000 # Keep no more than this number of offline serial revisions MAX_OFFLINE_SERIALS = 1000 diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index abd1a16a4..b8fbcf923 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -213,7 +213,7 @@ class RegistrationHandler(BaseHandler): 400, "User ID must only contain characters which do not" " require URL encoding." - ) + ) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 799221c19..088b76d23 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -927,7 +927,7 @@ class RoomContextHandler(BaseHandler): Returns: dict, or None if the event isn't found """ - before_limit = math.floor(limit/2.) + before_limit = math.floor(limit / 2.) after_limit = limit - before_limit now_token = yield self.hs.get_event_sources().get_current_token() diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index da13e32e7..c3589534f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -152,7 +152,7 @@ class MatrixFederationHttpClient(object): return self.clock.time_bound_deferred( request_deferred, - time_out=timeout/1000. if timeout else 60, + time_out=timeout / 1000. if timeout else 60, ) response = yield preserve_context_over_fn( diff --git a/synapse/notifier.py b/synapse/notifier.py index 29965a9ab..1a90bd55c 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -308,7 +308,7 @@ class Notifier(object): def timed_out(): if listener: listener.deferred.cancel() - timer = self.clock.call_later(timeout/1000., timed_out) + timer = self.clock.call_later(timeout / 1000., timed_out) prev_token = from_token while not result: diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index dca018af9..2a2b4437d 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -304,7 +304,7 @@ def _flatten_dict(d, prefix=[], result={}): if isinstance(value, basestring): result[".".join(prefix + [key])] = value.lower() elif hasattr(value, "items"): - _flatten_dict(value, prefix=(prefix+[key]), result=result) + _flatten_dict(value, prefix=(prefix + [key]), result=result) return result diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 07836709f..7199113da 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -89,7 +89,7 @@ class LoginRestServlet(ClientV1RestServlet): LoginRestServlet.SAML2_TYPE): relay_state = "" if "relay_state" in login_submission: - relay_state = "&RelayState="+urllib.quote( + relay_state = "&RelayState=" + urllib.quote( login_submission["relay_state"]) result = { "uri": "%s%s" % (self.idp_redirect_url, relay_state) diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index e218ed215..5547f1b11 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -52,7 +52,7 @@ class PusherRestServlet(ClientV1RestServlet): if i not in content: missing.append(i) if len(missing): - raise SynapseError(400, "Missing parameters: "+','.join(missing), + raise SynapseError(400, "Missing parameters: " + ','.join(missing), errcode=Codes.MISSING_PARAM) logger.debug("set pushkey %s to kind %s", content['pushkey'], content['kind']) @@ -83,7 +83,7 @@ class PusherRestServlet(ClientV1RestServlet): data=content['data'] ) except PusherConfigException as pce: - raise SynapseError(400, "Config Error: "+pce.message, + raise SynapseError(400, "Config Error: " + pce.message, errcode=Codes.MISSING_PARAM) defer.returnValue((200, {})) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index 5378a9a93..2bfd4d96b 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -38,7 +38,8 @@ logger = logging.getLogger(__name__) if hasattr(hmac, "compare_digest"): compare_digest = hmac.compare_digest else: - compare_digest = lambda a, b: a == b + def compare_digest(a, b): + return a == b class RegisterRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 5d50dd9e3..56a5bbec3 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -34,7 +34,8 @@ from synapse.util.async import run_on_reactor if hasattr(hmac, "compare_digest"): compare_digest = hmac.compare_digest else: - compare_digest = lambda a, b: a == b + def compare_digest(a, b): + return a == b logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 349ef6b39..ca5468c40 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -26,9 +26,7 @@ class VersionsRestServlet(RestServlet): def on_GET(self, request): return (200, { - "versions": [ - "r0.0.1", - ] + "versions": ["r0.0.1"] }) diff --git a/synapse/server.py b/synapse/server.py index 5fee7fe13..368d61557 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -23,7 +23,7 @@ from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.enterprise import adbapi from synapse.federation import initialize_http_replication -from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory +from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers diff --git a/synapse/state.py b/synapse/state.py index 0acf309fe..b9a138752 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -63,7 +63,7 @@ class StateHandler(object): cache_name="state_cache", clock=self.clock, max_len=SIZE_OF_CACHE, - expiry_ms=EVICTION_TIMEOUT_SECONDS*1000, + expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, reset_expiry_on_get=True, ) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c91c7a372..5a9e7720d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -59,7 +59,7 @@ logger = logging.getLogger(__name__) # Number of msec of granularity to store the user IP 'last seen' time. Smaller # times give more inserts into the database even for readonly API hits # 120 seconds == 2 minutes -LAST_SEEN_GRANULARITY = 120*1000 +LAST_SEEN_GRANULARITY = 120 * 1000 class DataStore(RoomMemberStore, RoomStore, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5e7732054..cfb87d932 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -185,7 +185,7 @@ class SQLBaseStore(object): time_then = self._previous_loop_ts self._previous_loop_ts = time_now - ratio = (curr - prev)/(time_now - time_then) + ratio = (curr - prev) / (time_now - time_then) top_three_counters = self._txn_perf_counters.interval( time_now - time_then, limit=3 @@ -643,7 +643,10 @@ class SQLBaseStore(object): if not iterable: defer.returnValue(results) - chunks = [iterable[i:i+batch_size] for i in xrange(0, len(iterable), batch_size)] + chunks = [ + iterable[i:i + batch_size] + for i in xrange(0, len(iterable), batch_size) + ] for chunk in chunks: rows = yield self.runInteraction( desc, diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 400c10103..91fac33b8 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -54,7 +54,7 @@ class Sqlite3Engine(object): def _parse_match_info(buf): bufsize = len(buf) - return [struct.unpack('@I', buf[i:i+4])[0] for i in range(0, bufsize, 4)] + return [struct.unpack('@I', buf[i:i + 4])[0] for i in range(0, bufsize, 4)] def _rank(raw_match_info): diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 5f32eec6f..ce2c79402 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -58,7 +58,7 @@ class EventFederationStore(SQLBaseStore): new_front = set() front_list = list(front) chunks = [ - front_list[x:x+100] + front_list[x:x + 100] for x in xrange(0, len(front), 100) ] for chunk in chunks: diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 5e8555202..4d7cdd00d 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -84,7 +84,7 @@ class EventsStore(SQLBaseStore): event.internal_metadata.stream_ordering = stream chunks = [ - events_and_contexts[x:x+100] + events_and_contexts[x:x + 100] for x in xrange(0, len(events_and_contexts), 100) ] @@ -740,7 +740,7 @@ class EventsStore(SQLBaseStore): rows = [] N = 200 for i in range(1 + len(events) / N): - evs = events[i*N:(i + 1)*N] + evs = events[i * N:(i + 1) * N] if not evs: break @@ -755,7 +755,7 @@ class EventsStore(SQLBaseStore): " LEFT JOIN rejections as rej USING (event_id)" " LEFT JOIN redactions as r ON e.event_id = r.redacts" " WHERE e.event_id IN (%s)" - ) % (",".join(["?"]*len(evs)),) + ) % (",".join(["?"] * len(evs)),) txn.execute(sql, evs) rows.extend(self.cursor_to_dict(txn)) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 338a9d40d..2c49a5e49 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -168,7 +168,7 @@ class StreamStore(SQLBaseStore): results = {} room_ids = list(room_ids) - for rm_ids in (room_ids[i:i+20] for i in xrange(0, len(room_ids), 20)): + for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)): res = yield defer.gatherResults([ self.get_room_events_stream_for_room( room_id, from_key, to_key, limit diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index f1fe963ad..7566d9eb3 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -46,7 +46,7 @@ class Clock(object): def looping_call(self, f, msec): l = task.LoopingCall(f) - l.start(msec/1000.0, now=False) + l.start(msec / 1000.0, now=False) return l def stop_looping_call(self, loop): diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 88e56e330..e27917c63 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -149,7 +149,7 @@ class CacheDescriptor(object): self.lru = lru self.tree = tree - self.arg_names = inspect.getargspec(orig).args[1:num_args+1] + self.arg_names = inspect.getargspec(orig).args[1:num_args + 1] if len(self.arg_names) < self.num_args: raise Exception( @@ -250,7 +250,7 @@ class CacheListDescriptor(object): self.num_args = num_args self.list_name = list_name - self.arg_names = inspect.getargspec(orig).args[1:num_args+1] + self.arg_names = inspect.getargspec(orig).args[1:num_args + 1] self.list_pos = self.arg_names.index(self.list_name) self.cache = cache diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 494226f5e..62cae9964 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -55,7 +55,7 @@ class ExpiringCache(object): def f(): self._prune_cache() - self._clock.looping_call(f, self._expiry_ms/2) + self._clock.looping_call(f, self._expiry_ms / 2) def __setitem__(self, key, value): now = self._clock.time_msec() diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index 29d02f7e9..03bc1401b 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -58,7 +58,7 @@ class TreeCache(object): if n: break - node_and_keys[i+1][0].pop(k) + node_and_keys[i + 1][0].pop(k) popped, cnt = _strip_and_count_entires(popped) self.size -= cnt diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index d5b1a37ef..c37a15778 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -111,7 +111,7 @@ def time_function(f): _log_debug_as_f( f, "[FUNC END] {%s-%d} %f", - (func_name, id, end-start,), + (func_name, id, end - start,), ) return r diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index c37d6f12e..ea321bc6a 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -163,7 +163,7 @@ class _PerHostRatelimiter(object): "Ratelimit [%s]: sleeping req", id(request_id), ) - ret_defer = sleep(self.sleep_msec/1000.0) + ret_defer = sleep(self.sleep_msec / 1000.0) self.sleeping_requests.add(request_id) From a644ac6d2c710d8dbe59d8c2fbf8ee95e1e32bad Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 2 Feb 2016 17:23:12 +0000 Subject: [PATCH 240/294] Explain what W503 is --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e7fc5ffe7..f8cc13c84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,4 +16,4 @@ ignore = [flake8] max-line-length = 90 -ignore = W503 +ignore = W503 ; W503 requires that binary operators be at the end, not start, of lines. Erik doesn't like it. From 2df6114bc449194fa99aae3f7c41b37e1ea0dbcf Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 2 Feb 2016 19:21:49 +0000 Subject: [PATCH 241/294] Log more diagnostics for unrecognised access tokens --- synapse/api/auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index c5a2865e2..5bba9343f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -696,6 +696,7 @@ class Auth(object): def _look_up_user_by_access_token(self, token): ret = yield self.store.get_user_by_access_token(token) if not ret: + logger.warn("Unrecognised access token - not in store: %s" % (token,)) raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.", errcode=Codes.UNKNOWN_TOKEN @@ -713,6 +714,7 @@ class Auth(object): token = request.args["access_token"][0] service = yield self.store.get_app_service_by_token(token) if not service: + logger.warn("Unrecognised appservice access token: %s" % (token,)) raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.", From b32121a5d1c833ab3e2c164e642ef982364069e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 10:30:56 +0000 Subject: [PATCH 242/294] Unused import --- synapse/storage/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 1b6311f33..dc09a3aab 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cachedInlineCallbacks, cached +from synapse.util.caches.descriptors import cachedInlineCallbacks from .engines import PostgresEngine, Sqlite3Engine import collections From 771528ab1323715271b9e968d2d337b88910fb2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 10:50:49 +0000 Subject: [PATCH 243/294] Change event_push_actions_rm_tokens schema --- synapse/handlers/sync.py | 6 +-- synapse/push/__init__.py | 2 +- synapse/storage/event_push_actions.py | 47 +++++++++++++------ synapse/storage/prepare_database.py | 2 +- .../storage/schema/delta/29/push_actions.sql | 31 ++++++++++++ 5 files changed, 67 insertions(+), 21 deletions(-) create mode 100644 synapse/storage/schema/delta/29/push_actions.sql diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dc686db54..0292e0673 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -706,10 +706,8 @@ class SyncHandler(BaseHandler): ) if notifs is not None: - unread_notifications["notification_count"] = len(notifs) - unread_notifications["highlight_count"] = len([ - 1 for notif in notifs if _action_has_highlight(notif["actions"]) - ]) + unread_notifications["notification_count"] = notifs["notify_count"] + unread_notifications["highlight_count"] = notifs["highlight_count"] logger.debug("Room sync: %r", room_sync) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 9bc0b356f..8b9d0f03e 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -316,7 +316,7 @@ class Pusher(object): r.room_id, self.user_id, last_unread_event_id ) ) - badge += len(notifs) + badge += notifs["notify_count"] defer.returnValue(badge) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index a05c4f84c..aca321920 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -37,7 +37,11 @@ class EventPushActionsStore(SQLBaseStore): 'event_id': event.event_id, 'user_id': uid, 'profile_tag': profile_tag, - 'actions': json.dumps(actions) + 'actions': json.dumps(actions), + 'stream_ordering': event.internal_metadata.stream_ordering, + 'topological_ordering': event.depth, + 'notif': 1, + 'highlight': 1 if _action_has_highlight(actions) else 0, }) def f(txn): @@ -74,26 +78,28 @@ class EventPushActionsStore(SQLBaseStore): topological_ordering = results[0][1] sql = ( - "SELECT ea.event_id, ea.actions" - " FROM event_push_actions ea, events e" - " WHERE ea.room_id = e.room_id" - " AND ea.event_id = e.event_id" - " AND ea.user_id = ?" - " AND ea.room_id = ?" + "SELECT sum(notif), sum(highlight)" + " FROM event_push_actions ea" + " WHERE" + " user_id = ?" + " AND room_id = ?" " AND (" - " e.topological_ordering > ?" - " OR (e.topological_ordering = ? AND e.stream_ordering > ?)" + " topological_ordering > ?" + " OR (topological_ordering = ? AND stream_ordering > ?)" ")" ) txn.execute(sql, ( user_id, room_id, topological_ordering, topological_ordering, stream_ordering - ) - ) - return [ - {"event_id": row[0], "actions": json.loads(row[1])} - for row in txn.fetchall() - ] + )) + row = txn.fetchone() + if row: + return { + "notify_count": row[0] or 0, + "highlight_count": row[1] or 0, + } + else: + return {"notify_count": 0, "highlight_count": 0} ret = yield self.runInteraction( "get_unread_event_push_actions_by_room", @@ -117,3 +123,14 @@ class EventPushActionsStore(SQLBaseStore): "remove_push_actions_for_event_id", f ) + + +def _action_has_highlight(actions): + for action in actions: + try: + if action.get("set_tweak", None) == "highlight": + return action.get("value", True) + except AttributeError: + pass + + return False diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index c1f5f9978..d782b8e25 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 28 +SCHEMA_VERSION = 29 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/29/push_actions.sql b/synapse/storage/schema/delta/29/push_actions.sql new file mode 100644 index 000000000..7e7b09820 --- /dev/null +++ b/synapse/storage/schema/delta/29/push_actions.sql @@ -0,0 +1,31 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE event_push_actions ADD COLUMN topological_ordering BIGINT; +ALTER TABLE event_push_actions ADD COLUMN stream_ordering BIGINT; +ALTER TABLE event_push_actions ADD COLUMN notif SMALLINT; +ALTER TABLE event_push_actions ADD COLUMN highlight SMALLINT; + +UPDATE event_push_actions SET stream_ordering = ( + SELECT stream_ordering FROM events WHERE event_id = event_push_actions.event_id +), topological_ordering = ( + SELECT topological_ordering FROM events WHERE event_id = event_push_actions.event_id +); + +UPDATE event_push_actions SET notif = 1, highlight = 0; + +CREATE INDEX event_push_actions_rm_tokens on event_push_actions( + user_id, room_id, topological_ordering, stream_ordering +); From 5f280837a6757c3e9fe226c3fac35409c23d3967 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 3 Feb 2016 11:27:39 +0000 Subject: [PATCH 244/294] Add macaroon inspection script --- scripts-dev/dump_macaroon.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 scripts-dev/dump_macaroon.py diff --git a/scripts-dev/dump_macaroon.py b/scripts-dev/dump_macaroon.py new file mode 100755 index 000000000..6e45be75d --- /dev/null +++ b/scripts-dev/dump_macaroon.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python2 + +import pymacaroons +import sys + +if len(sys.argv) == 1: + sys.stderr.write("usage: %s macaroon [key]\n" % (sys.argv[0],)) + sys.exit(1) + +macaroon_string = sys.argv[1] +key = sys.argv[2] if len(sys.argv) > 2 else None + +macaroon = pymacaroons.Macaroon.deserialize(macaroon_string) +print macaroon.inspect() + +print "" + +verifier = pymacaroons.Verifier() +verifier.satisfy_general(lambda c: True) +try: + verifier.verify(macaroon, key) + print "Signature is correct" +except Exception as e: + print e.message From 772b45c745688745d4c6d6b60047eeafaef773ee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 11:43:26 +0000 Subject: [PATCH 245/294] Remove unused method --- synapse/handlers/room.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0daf2ce28..42581289e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -942,14 +942,6 @@ class RoomListHandler(BaseHandler): # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": result}) - @defer.inlineCallbacks - def get_room_avatar_url(self, room_id): - event = yield self.hs.get_state_handler().get_current_state( - room_id, "m.room.avatar" - ) - if event and "url" in event.content: - defer.returnValue(event.content["url"]) - class RoomContextHandler(BaseHandler): @defer.inlineCallbacks From 9cd80a7b5c62d2687fd4a2e6481a7701b6d782c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 11:52:57 +0000 Subject: [PATCH 246/294] PEP8 --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 42581289e..bfd7e44e9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -932,7 +932,7 @@ class RoomListHandler(BaseHandler): defer.returnValue(result) result = [] - for chunk in (room_ids[i:i+10] for i in xrange(0, len(room_ids), 10)): + for chunk in (room_ids[i:i + 10] for i in xrange(0, len(room_ids), 10)): chunk_result = yield defer.gatherResults([ handle_room(room_id) for room_id in chunk From f8aae79a72e462f4af65a22d0665192867522174 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 13:23:32 +0000 Subject: [PATCH 247/294] Simplify get_rooms --- synapse/app/homeserver.py | 4 +- synapse/storage/room.py | 84 ++++---------------------------------- tests/storage/test_room.py | 26 ------------ 3 files changed, 9 insertions(+), 105 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index c3066d6a0..0a6a19033 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -674,8 +674,8 @@ def run(hs): stats["uptime_seconds"] = uptime stats["total_users"] = yield hs.get_datastore().count_all_users() - all_rooms = yield hs.get_datastore().get_rooms(False) - stats["total_room_count"] = len(all_rooms) + room_count = yield hs.get_datastore().get_room_count() + stats["total_room_count"] = room_count stats["daily_active_users"] = yield hs.get_datastore().count_daily_users() daily_messages = yield hs.get_datastore().count_daily_messages() diff --git a/synapse/storage/room.py b/synapse/storage/room.py index dc09a3aab..46ab38a31 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -87,90 +87,20 @@ class RoomStore(SQLBaseStore): desc="get_public_room_ids", ) - @defer.inlineCallbacks - def get_rooms(self, is_public): - """Retrieve a list of all public rooms. - - Args: - is_public (bool): True if the rooms returned should be public. - Returns: - A list of room dicts containing at least a "room_id" key, a - "topic" key if one is set, and a "name" key if one is set + def get_room_count(self): + """Retrieve a list of all rooms """ def f(txn): - def subquery(table_name, column_name=None): - column_name = column_name or table_name - return ( - "SELECT %(table_name)s.event_id as event_id, " - "%(table_name)s.room_id as room_id, %(column_name)s " - "FROM %(table_name)s " - "INNER JOIN current_state_events as c " - "ON c.event_id = %(table_name)s.event_id " % { - "column_name": column_name, - "table_name": table_name, - } - ) + sql = "SELECT count(*) FROM rooms" + txn.execute(sql) + row = txn.fetchone() + return row[0] or 0 - sql = ( - "SELECT" - " r.room_id," - " max(n.name)," - " max(t.topic)," - " max(v.history_visibility)," - " max(g.guest_access)" - " FROM rooms AS r" - " LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id" - " LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id" - " LEFT JOIN (%(history_visibility)s) AS v ON v.room_id = r.room_id" - " LEFT JOIN (%(guest_access)s) AS g ON g.room_id = r.room_id" - " WHERE r.is_public = ?" - " GROUP BY r.room_id" % { - "topic": subquery("topics", "topic"), - "name": subquery("room_names", "name"), - "history_visibility": subquery("history_visibility"), - "guest_access": subquery("guest_access"), - } - ) - - txn.execute(sql, (is_public,)) - - rows = txn.fetchall() - - for i, row in enumerate(rows): - room_id = row[0] - aliases = self._simple_select_onecol_txn( - txn, - table="room_aliases", - keyvalues={ - "room_id": room_id - }, - retcol="room_alias", - ) - - rows[i] = list(row) + [aliases] - - return rows - - rows = yield self.runInteraction( + return self.runInteraction( "get_rooms", f ) - ret = [ - { - "room_id": r[0], - "name": r[1], - "topic": r[2], - "world_readable": r[3] == "world_readable", - "guest_can_join": r[4] == "can_join", - "aliases": r[5], - } - for r in rows - if r[5] # We only return rooms that have at least one alias. - ] - - defer.returnValue(ret) - def _store_room_topic_txn(self, txn, event): if hasattr(event, "content") and "topic" in event.content: self._simple_insert_txn( diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index 7fdbfc60f..0baaf3df2 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -51,32 +51,6 @@ class RoomStoreTestCase(unittest.TestCase): (yield self.store.get_room(self.room.to_string())) ) - @defer.inlineCallbacks - def test_get_rooms(self): - # get_rooms does an INNER JOIN on the room_aliases table :( - - rooms = yield self.store.get_rooms(is_public=True) - # Should be empty before we add the alias - self.assertEquals([], rooms) - - yield self.store.create_room_alias_association( - room_alias=self.alias, - room_id=self.room.to_string(), - servers=["test"] - ) - - rooms = yield self.store.get_rooms(is_public=True) - - self.assertEquals(1, len(rooms)) - self.assertEquals({ - "name": None, - "room_id": self.room.to_string(), - "topic": None, - "aliases": [self.alias.to_string()], - "world_readable": False, - "guest_can_join": False, - }, rooms[0]) - class RoomEventsStoreTestCase(unittest.TestCase): From d4f72a5bfb95d07d5af3f49c736823840659101a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 13:51:25 +0000 Subject: [PATCH 248/294] Allowing tagging log contexts --- synapse/handlers/sync.py | 10 ++++++++++ synapse/http/server.py | 41 ++++++++++++++++++++++++-------------- synapse/util/logcontext.py | 7 ++++++- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dc686db54..72ccaf1e3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,6 +18,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes from synapse.util import unwrapFirstError +from synapse.util.logcontext import LoggingContext from twisted.internet import defer @@ -140,6 +141,15 @@ class SyncHandler(BaseHandler): A Deferred SyncResult. """ + context = LoggingContext.current_context() + if context: + if since_token is None: + context.tag = "initial_sync" + elif full_state: + context.tag = "full_state_sync" + else: + context.tag = "incremental_sync" + if timeout == 0 or since_token is None or full_state: # we are going to return immediately, so don't bother calling # notifier.wait_for_events. diff --git a/synapse/http/server.py b/synapse/http/server.py index 10d1fcd3f..c250a4604 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -41,7 +41,7 @@ metrics = synapse.metrics.get_metrics_for(__name__) incoming_requests_counter = metrics.register_counter( "requests", - labels=["method", "servlet"], + labels=["method", "servlet", "tag"], ) outgoing_responses_counter = metrics.register_counter( "responses", @@ -50,23 +50,23 @@ outgoing_responses_counter = metrics.register_counter( response_timer = metrics.register_distribution( "response_time", - labels=["method", "servlet"] + labels=["method", "servlet", "tag"] ) response_ru_utime = metrics.register_distribution( - "response_ru_utime", labels=["method", "servlet"] + "response_ru_utime", labels=["method", "servlet", "tag"] ) response_ru_stime = metrics.register_distribution( - "response_ru_stime", labels=["method", "servlet"] + "response_ru_stime", labels=["method", "servlet", "tag"] ) response_db_txn_count = metrics.register_distribution( - "response_db_txn_count", labels=["method", "servlet"] + "response_db_txn_count", labels=["method", "servlet", "tag"] ) response_db_txn_duration = metrics.register_distribution( - "response_db_txn_duration", labels=["method", "servlet"] + "response_db_txn_duration", labels=["method", "servlet", "tag"] ) @@ -226,7 +226,6 @@ class JsonResource(HttpServer, resource.Resource): servlet_classname = servlet_instance.__class__.__name__ else: servlet_classname = "%r" % callback - incoming_requests_counter.inc(request.method, servlet_classname) args = [ urllib.unquote(u).decode("UTF-8") if u else u for u in m.groups() @@ -237,21 +236,33 @@ class JsonResource(HttpServer, resource.Resource): code, response = callback_return self._send_response(request, code, response) - response_timer.inc_by( - self.clock.time_msec() - start, request.method, servlet_classname - ) - try: context = LoggingContext.current_context() + + tag = "" + if context: + tag = context.tag + + incoming_requests_counter.inc(request.method, servlet_classname, tag) + + response_timer.inc_by( + self.clock.time_msec() - start, request.method, + servlet_classname, tag + ) + ru_utime, ru_stime = context.get_resource_usage() - response_ru_utime.inc_by(ru_utime, request.method, servlet_classname) - response_ru_stime.inc_by(ru_stime, request.method, servlet_classname) + response_ru_utime.inc_by( + ru_utime, request.method, servlet_classname, tag + ) + response_ru_stime.inc_by( + ru_stime, request.method, servlet_classname, tag + ) response_db_txn_count.inc_by( - context.db_txn_count, request.method, servlet_classname + context.db_txn_count, request.method, servlet_classname, tag ) response_db_txn_duration.inc_by( - context.db_txn_duration, request.method, servlet_classname + context.db_txn_duration, request.method, servlet_classname, tag ) except: pass diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 0595c0fa4..e701092cd 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -47,7 +47,8 @@ class LoggingContext(object): """ __slots__ = [ - "parent_context", "name", "usage_start", "usage_end", "main_thread", "__dict__" + "parent_context", "name", "usage_start", "usage_end", "main_thread", + "__dict__", "tag", ] thread_local = threading.local() @@ -72,6 +73,9 @@ class LoggingContext(object): def add_database_transaction(self, duration_ms): pass + def __nonzero__(self): + return False + sentinel = Sentinel() def __init__(self, name=None): @@ -83,6 +87,7 @@ class LoggingContext(object): self.db_txn_duration = 0. self.usage_start = None self.main_thread = threading.current_thread() + self.tag = "" def __str__(self): return "%s@%x" % (self.name, id(self)) From 5054806ec1f64fd784d9e74d73a678643d539c3f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 3 Feb 2016 14:42:01 +0000 Subject: [PATCH 249/294] Rename config field to reflect yaml name --- synapse/config/registration.py | 6 +++--- synapse/rest/client/v1/register.py | 4 ++-- synapse/rest/client/v2_alpha/register.py | 2 +- tests/rest/client/v1/test_events.py | 2 +- tests/rest/client/v2_alpha/test_register.py | 4 ++-- tests/utils.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 76d2d2d64..90ea19bd4 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -23,11 +23,11 @@ from distutils.util import strtobool class RegistrationConfig(Config): def read_config(self, config): - self.disable_registration = not bool( + self.enable_registration = bool( strtobool(str(config["enable_registration"])) ) if "disable_registration" in config: - self.disable_registration = bool( + self.enable_registration = not bool( strtobool(str(config["disable_registration"])) ) @@ -78,6 +78,6 @@ class RegistrationConfig(Config): def read_arguments(self, args): if args.enable_registration is not None: - self.disable_registration = not bool( + self.enable_registration = bool( strtobool(str(args.enable_registration)) ) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index 2bfd4d96b..6d6d03c34 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -59,7 +59,7 @@ class RegisterRestServlet(ClientV1RestServlet): # } # TODO: persistent storage self.sessions = {} - self.disable_registration = hs.config.disable_registration + self.enable_registration = hs.config.enable_registration def on_GET(self, request): if self.hs.config.enable_registration_captcha: @@ -113,7 +113,7 @@ class RegisterRestServlet(ClientV1RestServlet): is_using_shared_secret = login_type == LoginType.SHARED_SECRET can_register = ( - not self.disable_registration + self.enable_registration or is_application_server or is_using_shared_secret ) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 56a5bbec3..ec5c21fa1 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -117,7 +117,7 @@ class RegisterRestServlet(RestServlet): return # == Normal User Registration == (everyone else) - if self.hs.config.disable_registration: + if not self.hs.config.enable_registration: raise SynapseError(403, "Registration has been disabled") guest_access_token = body.get("guest_access_token", None) diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index b260e269a..e9698bfdc 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -122,7 +122,7 @@ class EventStreamPermissionsTestCase(RestTestCase): self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) hs.config.enable_registration_captcha = False - hs.config.disable_registration = False + hs.config.enable_registration = True hs.get_handlers().federation_handler = Mock() diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index f9a2b2248..df0841b0b 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -41,7 +41,7 @@ class RegisterRestServletTestCase(unittest.TestCase): self.hs.hostname = "superbig~testing~thing.com" self.hs.get_auth = Mock(return_value=self.auth) self.hs.get_handlers = Mock(return_value=self.handlers) - self.hs.config.disable_registration = False + self.hs.config.enable_registration = True # init the thing we're testing self.servlet = RegisterRestServlet(self.hs) @@ -120,7 +120,7 @@ class RegisterRestServletTestCase(unittest.TestCase): })) def test_POST_disabled_registration(self): - self.hs.config.disable_registration = True + self.hs.config.enable_registration = False self.request_data = json.dumps({ "username": "kermit", "password": "monkey" diff --git a/tests/utils.py b/tests/utils.py index 431252a6f..3b1eb50d8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,7 +46,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config = Mock() config.signing_key = [MockKey()] config.event_cache_size = 1 - config.disable_registration = False + config.enable_registration = True config.macaroon_secret_key = "not even a little secret" config.server_name = "server.under.test" config.trusted_third_party_id_servers = [] From 24277fbb97540499daf4984b3ac1a1b81a0a19de Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 3 Feb 2016 14:59:19 +0000 Subject: [PATCH 250/294] Don't return null if profile display name / avatar url isn't set: omit them instead --- synapse/rest/client/v1/profile.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index b15defdd0..f294f3d7e 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -33,7 +33,11 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): user, ) - defer.returnValue((200, {"displayname": displayname})) + ret = {} + if displayname is not None: + ret["displayname"] = displayname + + defer.returnValue((200, {ret})) @defer.inlineCallbacks def on_PUT(self, request, user_id): @@ -66,7 +70,11 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): user, ) - defer.returnValue((200, {"avatar_url": avatar_url})) + ret = {} + if avatar_url is not None: + ret["avatar_url"] = avatar_url + + defer.returnValue((200, ret)) @defer.inlineCallbacks def on_PUT(self, request, user_id): @@ -102,10 +110,13 @@ class ProfileRestServlet(ClientV1RestServlet): user, ) - defer.returnValue((200, { - "displayname": displayname, - "avatar_url": avatar_url - })) + ret = {} + if displayname is not None: + ret["displayname"] = displayname + if avatar_url is not None: + ret["avatar_url"] = avatar_url + + defer.returnValue((200, ret)) def register_servlets(hs, http_server): From 156cea5b45acac88a030f5b469c1921a2f3573cc Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 3 Feb 2016 15:04:51 +0000 Subject: [PATCH 251/294] No braces here --- synapse/rest/client/v1/profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index f294f3d7e..3c5a21292 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): if displayname is not None: ret["displayname"] = displayname - defer.returnValue((200, {ret})) + defer.returnValue((200, ret)) @defer.inlineCallbacks def on_PUT(self, request, user_id): From 33c71c3a4b06852285d8e446cbe2f57293ae0975 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 16:17:18 +0000 Subject: [PATCH 252/294] Preserve log context over when deferring to thread pool in media repo --- synapse/rest/media/v1/base_resource.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index bdc65f019..58d56ec7a 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -28,6 +28,7 @@ from twisted.protocols.basic import FileSender from synapse.util.async import ObservableDeferred from synapse.util.stringutils import is_ascii +from synapse.util.logcontext import preserve_context_over_fn import os @@ -276,7 +277,8 @@ class BaseMediaResource(Resource): ) self._makedirs(t_path) - t_len = yield threads.deferToThread( + t_len = yield preserve_context_over_fn( + threads.deferToThread, self._generate_thumbnail, input_path, t_path, t_width, t_height, t_method, t_type ) @@ -298,7 +300,8 @@ class BaseMediaResource(Resource): ) self._makedirs(t_path) - t_len = yield threads.deferToThread( + t_len = yield preserve_context_over_fn( + threads.deferToThread, self._generate_thumbnail, input_path, t_path, t_width, t_height, t_method, t_type ) @@ -372,7 +375,7 @@ class BaseMediaResource(Resource): media_id, t_width, t_height, t_type, t_method, t_len )) - yield threads.deferToThread(generate_thumbnails) + yield preserve_context_over_fn(threads.deferToThread, generate_thumbnails) for l in local_thumbnails: yield self.store.store_local_thumbnail(*l) @@ -445,7 +448,7 @@ class BaseMediaResource(Resource): t_width, t_height, t_type, t_method, t_len ]) - yield threads.deferToThread(generate_thumbnails) + yield preserve_context_over_fn(threads.deferToThread, generate_thumbnails) for r in remote_thumbnails: yield self.store.store_remote_media_thumbnail(*r) From b84d59c5f01914fe53d2673c5c7e372f5c61d088 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 16:22:35 +0000 Subject: [PATCH 253/294] Add descriptions --- synapse/storage/appservice.py | 3 ++- synapse/storage/keys.py | 1 + synapse/storage/registration.py | 1 + synapse/storage/stream.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index b5aa55c0a..1100c6771 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -276,7 +276,8 @@ class ApplicationServiceTransactionStore(SQLBaseStore): "application_services_state", dict(as_id=service.id), ["state"], - allow_none=True + allow_none=True, + desc="get_appservice_state", ) if result: defer.returnValue(result.get("state")) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 8022b8cfc..fd05bfe54 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -39,6 +39,7 @@ class KeyStore(SQLBaseStore): table="server_tls_certificates", keyvalues={"server_name": server_name}, retcols=("tls_certificate",), + desc="get_server_certificate", ) tls_certificate = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_ASN1, tls_certificate_bytes, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 70cde0d04..bd35e19be 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -134,6 +134,7 @@ class RegistrationStore(SQLBaseStore): }, retcols=["name", "password_hash", "is_guest"], allow_none=True, + desc="get_user_by_id", ) def get_users_by_id_case_insensitive(self, user_id): diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2c49a5e49..50436cb2d 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -564,6 +564,7 @@ class StreamStore(SQLBaseStore): table="events", keyvalues={"event_id": event_id}, retcols=("stream_ordering", "topological_ordering"), + desc="get_topological_token_for_event", ).addCallback(lambda row: "t%d-%d" % ( row["topological_ordering"], row["stream_ordering"],) ) From aa4af94c69b8b1c263dacfce0358aaef97d3e323 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 16:29:32 +0000 Subject: [PATCH 254/294] We return dicts now. --- synapse/storage/event_push_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index aca321920..2742e0c00 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -72,7 +72,7 @@ class EventPushActionsStore(SQLBaseStore): ) results = txn.fetchall() if len(results) == 0: - return [] + return {} stream_ordering = results[0][0] topological_ordering = results[0][1] From 709e09e1c3dfc85f585dbf782de9701c5db0a02d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 16:32:20 +0000 Subject: [PATCH 255/294] Remove old log line --- synapse/rest/client/v1/room.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c7ea15c62..81bfe377b 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -429,8 +429,6 @@ class RoomEventContext(ClientV1RestServlet): serialize_event(event, time_now) for event in results["state"] ] - logger.info("Responding with %r", results) - defer.returnValue((200, results)) From 4d36e732307ad35eb070af384058f227d7d85dd0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 16:35:00 +0000 Subject: [PATCH 256/294] Actually return something sensible --- synapse/storage/event_push_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 2742e0c00..d0a969f50 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -72,7 +72,7 @@ class EventPushActionsStore(SQLBaseStore): ) results = txn.fetchall() if len(results) == 0: - return {} + return {"notify_count": 0, "highlight_count": 0} stream_ordering = results[0][0] topological_ordering = results[0][1] From 6a9f1209dfe5b3c43726aff24000129856bdc084 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 5 Feb 2016 01:58:23 +0000 Subject: [PATCH 257/294] Error if macaroon key is missing from config Currently we store all access tokens in the DB, and fall back to that check if we can't validate the macaroon, so our fallback works here, but for guests, their macaroons don't get persisted, so we don't get to find them in the database. Each restart, we generate a new ephemeral key, so guests lose access after each server restart. I tried to fix up the config stuff to be less insane, but gave up, so instead I bolt on yet another piece of custom one-off insanity. Also, add some basic tests for config generation and loading. --- synapse/app/homeserver.py | 20 ++++++--- synapse/config/__main__.py | 7 +++- synapse/config/_base.py | 35 ++++++++++------ synapse/config/registration.py | 18 ++++++-- tests/config/__init__.py | 14 +++++++ tests/config/test_generate.py | 50 ++++++++++++++++++++++ tests/config/test_load.py | 77 ++++++++++++++++++++++++++++++++++ 7 files changed, 198 insertions(+), 23 deletions(-) create mode 100644 tests/config/__init__.py create mode 100644 tests/config/test_generate.py create mode 100644 tests/config/test_load.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0a6a19033..89238cb7e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -24,6 +24,7 @@ import resource import subprocess import sys import time +from synapse.config._base import ConfigError from synapse.python_dependencies import ( check_requirements, DEPENDENCY_LINKS @@ -350,11 +351,20 @@ def setup(config_options): Returns: HomeServer """ - config = HomeServerConfig.load_config( - "Synapse Homeserver", - config_options, - generate_section="Homeserver" - ) + try: + config = HomeServerConfig.load_config( + "Synapse Homeserver", + config_options, + generate_section="Homeserver" + ) + except ConfigError as e: + sys.stderr.write("\n" + e.message + "\n") + sys.exit(1) + + if not config: + # If a config isn't returned, and an exception isn't raised, we're just + # generating config files and shouldn't try to continue. + sys.exit(0) config.setup_logging() diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py index ea9e7907a..0a3b70e11 100644 --- a/synapse/config/__main__.py +++ b/synapse/config/__main__.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from synapse.config._base import ConfigError if __name__ == "__main__": import sys @@ -21,7 +22,11 @@ if __name__ == "__main__": if action == "read": key = sys.argv[2] - config = HomeServerConfig.load_config("", sys.argv[3:]) + try: + config = HomeServerConfig.load_config("", sys.argv[3:]) + except ConfigError as e: + sys.stderr.write("\n" + e.message + "\n") + sys.exit(1) print getattr(config, key) sys.exit(0) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index a9304a11b..15d78ff33 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -17,7 +17,6 @@ import argparse import errno import os import yaml -import sys from textwrap import dedent @@ -136,13 +135,20 @@ class Config(object): results.append(getattr(cls, name)(self, *args, **kargs)) return results - def generate_config(self, config_dir_path, server_name, report_stats=None): + def generate_config( + self, + config_dir_path, + server_name, + is_generating_file, + report_stats=None, + ): default_config = "# vim:ft=yaml\n" default_config += "\n\n".join(dedent(conf) for conf in self.invoke_all( "default_config", config_dir_path=config_dir_path, server_name=server_name, + is_generating_file=is_generating_file, report_stats=report_stats, )) @@ -244,8 +250,10 @@ class Config(object): server_name = config_args.server_name if not server_name: - print "Must specify a server_name to a generate config for." - sys.exit(1) + raise ConfigError( + "Must specify a server_name to a generate config for." + " Pass -H server.name." + ) if not os.path.exists(config_dir_path): os.makedirs(config_dir_path) with open(config_path, "wb") as config_file: @@ -253,6 +261,7 @@ class Config(object): config_dir_path=config_dir_path, server_name=server_name, report_stats=(config_args.report_stats == "yes"), + is_generating_file=True ) obj.invoke_all("generate_files", config) config_file.write(config_bytes) @@ -266,7 +275,7 @@ class Config(object): "If this server name is incorrect, you will need to" " regenerate the SSL certificates" ) - sys.exit(0) + return else: print ( "Config file %r already exists. Generating any missing key" @@ -302,25 +311,25 @@ class Config(object): specified_config.update(yaml_config) if "server_name" not in specified_config: - sys.stderr.write("\n" + MISSING_SERVER_NAME + "\n") - sys.exit(1) + raise ConfigError(MISSING_SERVER_NAME) server_name = specified_config["server_name"] _, config = obj.generate_config( config_dir_path=config_dir_path, - server_name=server_name + server_name=server_name, + is_generating_file=False, ) config.pop("log_config") config.update(specified_config) if "report_stats" not in config: - sys.stderr.write( - "\n" + MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS + "\n" + - MISSING_REPORT_STATS_SPIEL + "\n") - sys.exit(1) + raise ConfigError( + MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS + "\n" + + MISSING_REPORT_STATS_SPIEL + ) if generate_keys: obj.invoke_all("generate_files", config) - sys.exit(0) + return obj.invoke_all("read_config", config) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 90ea19bd4..9b6dacc5b 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -33,12 +33,24 @@ class RegistrationConfig(Config): self.registration_shared_secret = config.get("registration_shared_secret") self.macaroon_secret_key = config.get("macaroon_secret_key") + if self.macaroon_secret_key is None: + raise Exception( + "Config is missing missing macaroon_secret_key - please set it" + " in your config file." + ) self.bcrypt_rounds = config.get("bcrypt_rounds", 12) self.trusted_third_party_id_servers = config["trusted_third_party_id_servers"] self.allow_guest_access = config.get("allow_guest_access", False) - def default_config(self, **kwargs): + def default_config(self, is_generating_file=False, **kwargs): registration_shared_secret = random_string_with_symbols(50) + + macaroon_line = "" + if is_generating_file: + macaroon_line += '\n macaroon_secret_key: "%s"\n' % ( + random_string_with_symbols(50), + ) + macaroon_secret_key = random_string_with_symbols(50) return """\ ## Registration ## @@ -49,9 +61,7 @@ class RegistrationConfig(Config): # If set, allows registration by anyone who also has the shared # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" - - macaroon_secret_key: "%(macaroon_secret_key)s" - +%(macaroon_line)s # Set the number of bcrypt rounds used to generate password hash. # Larger numbers increase the work factor needed to generate the hash. # The default number of rounds is 12. diff --git a/tests/config/__init__.py b/tests/config/__init__.py new file mode 100644 index 000000000..b7df13c9e --- /dev/null +++ b/tests/config/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/config/test_generate.py b/tests/config/test_generate.py new file mode 100644 index 000000000..4329d7397 --- /dev/null +++ b/tests/config/test_generate.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os.path +import shutil +import tempfile +from synapse.config.homeserver import HomeServerConfig +from tests import unittest + + +class ConfigGenerationTestCase(unittest.TestCase): + + def setUp(self): + self.dir = tempfile.mkdtemp() + print self.dir + self.file = os.path.join(self.dir, "homeserver.yaml") + + def tearDown(self): + shutil.rmtree(self.dir) + + def test_generate_config_generates_files(self): + HomeServerConfig.load_config("", [ + "--generate-config", + "-c", self.file, + "--report-stats=yes", + "-H", "lemurs.win" + ]) + + self.assertSetEqual( + set([ + "homeserver.yaml", + "lemurs.win.log.config", + "lemurs.win.signing.key", + "lemurs.win.tls.crt", + "lemurs.win.tls.dh", + "lemurs.win.tls.key", + ]), + set(os.listdir(self.dir)) + ) diff --git a/tests/config/test_load.py b/tests/config/test_load.py new file mode 100644 index 000000000..7f4127971 --- /dev/null +++ b/tests/config/test_load.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os.path +import shutil +import tempfile +import yaml +from synapse.config.homeserver import HomeServerConfig +from tests import unittest + + +class ConfigLoadingTestCase(unittest.TestCase): + + def setUp(self): + self.dir = tempfile.mkdtemp() + print self.dir + self.file = os.path.join(self.dir, "homeserver.yaml") + + def tearDown(self): + shutil.rmtree(self.dir) + + def test_load_fails_if_server_name_missing(self): + self.generate_config_and_remove_lines_containing("server_name") + with self.assertRaises(Exception): + HomeServerConfig.load_config("", ["-c", self.file]) + + def test_generates_and_loads_macaroon_secret_key(self): + self.generate_config() + + with open(self.file, + "r") as f: + raw = yaml.load(f) + self.assertIn("macaroon_secret_key", raw) + + config = HomeServerConfig.load_config("", ["-c", self.file]) + self.assertTrue( + hasattr(config, "macaroon_secret_key"), + "Want config to have attr macaroon_secret_key" + ) + if len(config.macaroon_secret_key) < 5: + self.fail( + "Want macaroon secret key to be string of at least length 5," + "was: %r" % (config.macaroon_secret_key,) + ) + + def test_load_fails_if_macaroon_secret_key_missing(self): + self.generate_config_and_remove_lines_containing("macaroon") + with self.assertRaises(Exception): + HomeServerConfig.load_config("", ["-c", self.file]) + + def generate_config(self): + HomeServerConfig.load_config("", [ + "--generate-config", + "-c", self.file, + "--report-stats=yes", + "-H", "lemurs.win" + ]) + + def generate_config_and_remove_lines_containing(self, needle): + self.generate_config() + + with open(self.file, "r") as f: + contents = f.readlines() + contents = [l for l in contents if needle not in l] + with open(self.file, "w") as f: + f.write("".join(contents)) From 737c4223ef25fd2856d0ff6cc111d14b19f1adec Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 5 Feb 2016 10:47:46 +0000 Subject: [PATCH 258/294] Host /media/r0 as well as /media/v1 --- synapse/api/urls.py | 3 ++- synapse/app/homeserver.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 19824f9a0..0fd9b7f24 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -23,5 +23,6 @@ WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" SERVER_KEY_V2_PREFIX = "/_matrix/key/v2" -MEDIA_PREFIX = "/_matrix/media/v1" +MEDIA_PREFIX = "/_matrix/media/r0" +LEGACY_MEDIA_PREFIX = "/_matrix/media/v1" APP_SERVICE_PREFIX = "/_matrix/appservice/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 89238cb7e..e5c7e39cf 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -56,7 +56,7 @@ from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource from synapse.api.urls import ( FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, STATIC_PREFIX, + SERVER_KEY_PREFIX, LEGACY_MEDIA_PREFIX, MEDIA_PREFIX, STATIC_PREFIX, SERVER_KEY_V2_PREFIX, ) from synapse.config.homeserver import HomeServerConfig @@ -148,8 +148,10 @@ class SynapseHomeServer(HomeServer): }) if name in ["media", "federation", "client"]: + media_repo = MediaRepositoryResource(self) resources.update({ - MEDIA_PREFIX: MediaRepositoryResource(self), + MEDIA_PREFIX: media_repo, + LEGACY_MEDIA_PREFIX: media_repo, CONTENT_REPO_PREFIX: ContentRepoResource( self, self.config.uploads_path, self.auth, self.content_addr ), From 79a1c0574b33955d28bfb12697ccd5a7be779b36 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 5 Feb 2016 11:22:30 +0000 Subject: [PATCH 259/294] Allocate guest user IDs numericcally The current random IDs are ugly and confusing when presented in UIs. This makes them prettier and easier to read. Also, disable non-automated registration of numeric IDs so that we don't need to worry so much about people carving out our automated address space and us needing to keep retrying ID registration. --- synapse/handlers/register.py | 55 +++++++++++++++++++++------------ synapse/storage/registration.py | 36 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b8fbcf923..2660fd21a 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -21,7 +21,6 @@ from synapse.api.errors import ( AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError ) from ._base import BaseHandler -import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor from synapse.http.client import CaptchaServerHttpClient @@ -45,6 +44,8 @@ class RegistrationHandler(BaseHandler): self.distributor.declare("registered_user") self.captcha_client = CaptchaServerHttpClient(hs) + self._next_generated_user_id = None + @defer.inlineCallbacks def check_username(self, localpart, guest_access_token=None): yield run_on_reactor() @@ -91,7 +92,7 @@ class RegistrationHandler(BaseHandler): Args: localpart : The local part of the user ID to register. If None, - one will be randomly generated. + one will be generated. password (str) : The password to assign to this user so they can login again. This can be None which means they cannot login again via a password (e.g. the user is an application service user). @@ -108,6 +109,18 @@ class RegistrationHandler(BaseHandler): if localpart: yield self.check_username(localpart, guest_access_token=guest_access_token) + was_guest = guest_access_token is not None + + if not was_guest: + try: + int(localpart) + raise RegistrationError( + 400, + "Numeric user IDs are reserved for guest users." + ) + except ValueError: + pass + user = UserID(localpart, self.hs.hostname) user_id = user.to_string() @@ -118,40 +131,36 @@ class RegistrationHandler(BaseHandler): user_id=user_id, token=token, password_hash=password_hash, - was_guest=guest_access_token is not None, + was_guest=was_guest, make_guest=make_guest, ) yield registered_user(self.distributor, user) else: - # autogen a random user ID + # autogen a sequential user ID attempts = 0 - user_id = None token = None - while not user_id: + user = None + while not user: + localpart = yield self._generate_user_id(attempts > 0) + user = UserID(localpart, self.hs.hostname) + user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) + if generate_token: + token = self.auth_handler().generate_access_token(user_id) try: - localpart = self._generate_user_id() - user = UserID(localpart, self.hs.hostname) - user_id = user.to_string() - yield self.check_user_id_is_valid(user_id) - if generate_token: - token = self.auth_handler().generate_access_token(user_id) yield self.store.register( user_id=user_id, token=token, password_hash=password_hash, make_guest=make_guest ) - - yield registered_user(self.distributor, user) except SynapseError: # if user id is taken, just generate another user_id = None token = None attempts += 1 - if attempts > 5: - raise RegistrationError( - 500, "Cannot generate user ID.") + yield registered_user(self.distributor, user) # We used to generate default identicons here, but nowadays # we want clients to generate their own as part of their branding @@ -283,8 +292,16 @@ class RegistrationHandler(BaseHandler): errcode=Codes.EXCLUSIVE ) - def _generate_user_id(self): - return "-" + stringutils.random_string(18) + @defer.inlineCallbacks + def _generate_user_id(self, reseed=False): + if reseed or self._next_generated_user_id is None: + self._next_generated_user_id = ( + yield self.store.find_next_generated_user_id_localpart() + ) + + id = self._next_generated_user_id + self._next_generated_user_id += 1 + defer.returnValue(str(id)) @defer.inlineCallbacks def _validate_captcha(self, ip_addr, private_key, challenge, response): diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index bd35e19be..967c732bd 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + from twisted.internet import defer from synapse.api.errors import StoreError, Codes @@ -351,3 +353,37 @@ class RegistrationStore(SQLBaseStore): ret = yield self.runInteraction("count_users", _count_users) defer.returnValue(ret) + + @defer.inlineCallbacks + def find_next_generated_user_id_localpart(self): + """ + Gets the localpart of the next generated user ID. + + Generated user IDs are integers, and we aim for them to be as small as + we can. Unfortunately, it's possible some of them are already taken by + existing users, and there may be gaps in the already taken range. This + function returns the start of the first allocatable gap. This is to + avoid the case of ID 10000000 being pre-allocated, so us wasting the + first (and shortest) many generated user IDs. + """ + def _find_next_generated_user_id(txn): + txn.execute("SELECT name FROM users") + rows = self.cursor_to_dict(txn) + + regex = re.compile("^@(\d+):") + + found = set() + + for r in rows: + user_id = r["name"] + match = regex.search(user_id) + if match: + found.add(int(match.group(1))) + for i in xrange(len(found) + 1): + if i not in found: + return i + + defer.returnValue((yield self.runInteraction( + "find_next_generated_user_id", + _find_next_generated_user_id + ))) From b052621f679103052426baf086fd48074199e81b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 5 Feb 2016 15:17:47 +0000 Subject: [PATCH 260/294] List the URL patterns in synapse --- scripts-dev/list_url_patterns.py | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 scripts-dev/list_url_patterns.py diff --git a/scripts-dev/list_url_patterns.py b/scripts-dev/list_url_patterns.py new file mode 100755 index 000000000..23bcc6b03 --- /dev/null +++ b/scripts-dev/list_url_patterns.py @@ -0,0 +1,54 @@ +#! /usr/bin/python + +import ast +import argparse +import os +import sys +import yaml + +PATTERNS = [] + + +class CallVisitor(ast.NodeVisitor): + def visit_Call(self, node): + if isinstance(node.func, ast.Name): + name = node.func.id + else: + return + + if name == "client_path_patterns": + PATTERNS.append(node.args[0].s) + elif name == "client_v2_patterns": + PATTERNS.append(node.args[0].s) + + +def find_patterns_in_code(input_code): + input_ast = ast.parse(input_code) + visitor = CallVisitor() + visitor.visit(input_ast) + + +def find_patterns_in_file(filepath): + with open(filepath) as f: + find_patterns_in_code(f.read()) + + +parser = argparse.ArgumentParser(description='Find url patterns.') + +parser.add_argument( + "directories", nargs='+', metavar="DIR", + help="Directories to search for definitions" +) + +args = parser.parse_args() + + +for directory in args.directories: + for root, dirs, files in os.walk(directory): + for filename in files: + if filename.endswith(".py"): + filepath = os.path.join(root, filename) + find_patterns_in_file(filepath) + + +yaml.dump(sorted(PATTERNS), sys.stdout, default_flow_style=False) From 77c7ed0e93bff2ef7bb1981bf1b9a996aaca6f19 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 5 Feb 2016 15:43:27 +0000 Subject: [PATCH 261/294] Report the v1 and v2 patterns separately --- scripts-dev/list_url_patterns.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts-dev/list_url_patterns.py b/scripts-dev/list_url_patterns.py index 23bcc6b03..58d40c4ff 100755 --- a/scripts-dev/list_url_patterns.py +++ b/scripts-dev/list_url_patterns.py @@ -6,8 +6,13 @@ import os import sys import yaml -PATTERNS = [] +PATTERNS_V1 = [] +PATTERNS_V2 = [] +RESULT = { + "v1": PATTERNS_V1, + "v2": PATTERNS_V2, +} class CallVisitor(ast.NodeVisitor): def visit_Call(self, node): @@ -16,10 +21,11 @@ class CallVisitor(ast.NodeVisitor): else: return + if name == "client_path_patterns": - PATTERNS.append(node.args[0].s) + PATTERNS_V1.append(node.args[0].s) elif name == "client_v2_patterns": - PATTERNS.append(node.args[0].s) + PATTERNS_V2.append(node.args[0].s) def find_patterns_in_code(input_code): @@ -50,5 +56,7 @@ for directory in args.directories: filepath = os.path.join(root, filename) find_patterns_in_file(filepath) +PATTERNS_V1.sort() +PATTERNS_V2.sort() -yaml.dump(sorted(PATTERNS), sys.stdout, default_flow_style=False) +yaml.dump(RESULT, sys.stdout, default_flow_style=False) From 1d19a5ec0fff73af9cee8c21118020b31be47379 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 8 Feb 2016 10:50:55 +0000 Subject: [PATCH 262/294] Reject additional path segments --- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/presence.py | 4 ++-- synapse/rest/client/v1/profile.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index e2f5eb7b2..5ec52707e 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class WhoisRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/admin/whois/(?P[^/]*)") + PATTERNS = client_path_patterns("/admin/whois/(?P[^/]*)$") @defer.inlineCallbacks def on_GET(self, request, user_id): diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index a6f8754e3..9410ac527 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) class PresenceStatusRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/presence/(?P[^/]*)/status") + PATTERNS = client_path_patterns("/presence/(?P[^/]*)/status$") @defer.inlineCallbacks def on_GET(self, request, user_id): @@ -73,7 +73,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): class PresenceListRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/presence/list/(?P[^/]*)") + PATTERNS = client_path_patterns("/presence/list/(?P[^/]*)$") @defer.inlineCallbacks def on_GET(self, request, user_id): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 3c5a21292..aeda7bfa3 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -23,7 +23,7 @@ import simplejson as json class ProfileDisplaynameRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/profile/(?P[^/]*)/displayname") + PATTERNS = client_path_patterns("/profile/(?P[^/]*)/displayname$") @defer.inlineCallbacks def on_GET(self, request, user_id): @@ -60,7 +60,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): class ProfileAvatarURLRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/profile/(?P[^/]*)/avatar_url") + PATTERNS = client_path_patterns("/profile/(?P[^/]*)/avatar_url$") @defer.inlineCallbacks def on_GET(self, request, user_id): @@ -97,7 +97,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): class ProfileRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/profile/(?P[^/]*)") + PATTERNS = client_path_patterns("/profile/(?P[^/]*)$") @defer.inlineCallbacks def on_GET(self, request, user_id): From 13e6262659fd544155875e18c0a3351c12d7651d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Feb 2016 10:15:56 +0000 Subject: [PATCH 263/294] Add metrics to pushers --- synapse/http/server.py | 10 +++++ synapse/push/__init__.py | 84 ++++++++++++++++++++++++--------------- synapse/util/metrics.py | 86 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 31 deletions(-) create mode 100644 synapse/util/metrics.py diff --git a/synapse/http/server.py b/synapse/http/server.py index c250a4604..06935783c 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -208,6 +208,9 @@ class JsonResource(HttpServer, resource.Resource): if request.method == "OPTIONS": self._send_response(request, 200, {}) return + + start_context = LoggingContext.current_context() + # Loop through all the registered callbacks to check if the method # and path regex match for path_entry in self.path_regexs.get(request.method, []): @@ -243,6 +246,13 @@ class JsonResource(HttpServer, resource.Resource): if context: tag = context.tag + if context != start_context: + logger.warn( + "Context have unexpectedly changed %r, %r", + context, self.start_context + ) + return + incoming_requests_counter.inc(request.method, servlet_classname, tag) response_timer.inc_by( diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8b9d0f03e..64e581b8b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -17,6 +17,8 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig from synapse.types import StreamToken +from synapse.util.logcontext import LoggingContext +from synapse.util.metrics import Measure import synapse.util.async import push_rule_evaluator as push_rule_evaluator @@ -27,6 +29,16 @@ import random logger = logging.getLogger(__name__) +_NEXT_ID = 1 + + +def _get_next_id(): + global _NEXT_ID + _id = _NEXT_ID + _NEXT_ID += 1 + return _id + + # Pushers could now be moved to pull out of the event_push_actions table instead # of listening on the event stream: this would avoid them having to run the # rules again. @@ -57,6 +69,8 @@ class Pusher(object): self.alive = True self.badge = None + self.name = "Pusher-%d" % (_get_next_id(),) + # The last value of last_active_time that we saw self.last_last_active_time = 0 self.has_unread = True @@ -86,38 +100,46 @@ class Pusher(object): @defer.inlineCallbacks def start(self): - if not self.last_token: - # First-time setup: get a token to start from (we can't - # just start from no token, ie. 'now' - # because we need the result to be reproduceable in case - # we fail to dispatch the push) - config = PaginationConfig(from_token=None, limit='1') - chunk = yield self.evStreamHandler.get_stream( - self.user_id, config, timeout=0, affect_presence=False - ) - self.last_token = chunk['end'] - self.store.update_pusher_last_token( - self.app_id, self.pushkey, self.user_id, self.last_token - ) - logger.info("Pusher %s for user %s starting from token %s", - self.pushkey, self.user_id, self.last_token) - - wait = 0 - while self.alive: - try: - if wait > 0: - yield synapse.util.async.sleep(wait) - yield self.get_and_dispatch() - wait = 0 - except: - if wait == 0: - wait = 1 - else: - wait = min(wait * 2, 1800) - logger.exception( - "Exception in pusher loop for pushkey %s. Pausing for %ds", - self.pushkey, wait + with LoggingContext(self.name): + if not self.last_token: + # First-time setup: get a token to start from (we can't + # just start from no token, ie. 'now' + # because we need the result to be reproduceable in case + # we fail to dispatch the push) + config = PaginationConfig(from_token=None, limit='1') + chunk = yield self.evStreamHandler.get_stream( + self.user_id, config, timeout=0, affect_presence=False ) + self.last_token = chunk['end'] + self.store.update_pusher_last_token( + self.app_id, self.pushkey, self.user_id, self.last_token + ) + logger.info("New pusher %s for user %s starting from token %s", + self.pushkey, self.user_id, self.last_token) + + else: + logger.info( + "Old pusher %s for user %s starting", + self.pushkey, self.user_id, + ) + + wait = 0 + while self.alive: + try: + if wait > 0: + yield synapse.util.async.sleep(wait) + with Measure(self.clock, "push"): + yield self.get_and_dispatch() + wait = 0 + except: + if wait == 0: + wait = 1 + else: + wait = min(wait * 2, 1800) + logger.exception( + "Exception in pusher loop for pushkey %s. Pausing for %ds", + self.pushkey, wait + ) @defer.inlineCallbacks def get_and_dispatch(self): diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py new file mode 100644 index 000000000..daf6087fe --- /dev/null +++ b/synapse/util/metrics.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from synapse.util.logcontext import LoggingContext +import synapse.metrics + +import logging + + +logger = logging.getLogger(__name__) + + +metrics = synapse.metrics.get_metrics_for(__name__) + +block_timer = metrics.register_distribution( + "block_timer", + labels=["block_name"] +) + +block_ru_utime = metrics.register_distribution( + "block_ru_utime", labels=["block_name"] +) + +block_ru_stime = metrics.register_distribution( + "block_ru_stime", labels=["block_name"] +) + +block_db_txn_count = metrics.register_distribution( + "block_db_txn_count", labels=["block_name"] +) + +block_db_txn_duration = metrics.register_distribution( + "block_db_txn_duration", labels=["block_name"] +) + + +class Measure(object): + __slots__ = ["clock", "name", "start_context", "start"] + + def __init__(self, clock, name): + self.clock = clock + self.name = name + self.start_context = None + self.start = None + + def __enter__(self): + self.start = self.clock.time_msec() + self.start_context = LoggingContext.current_context() + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + return + + duration = self.clock.time_msec() - self.start + block_timer.inc_by(duration, self.name) + + context = LoggingContext.current_context() + if not context: + return + + if context != self.start_context: + logger.warn( + "Context have unexpectedly changed %r, %r", + context, self.start_context + ) + return + + ru_utime, ru_stime = context.get_resource_usage() + + block_ru_utime.inc_by(ru_utime, self.name) + block_ru_stime.inc_by(ru_stime, self.name) + block_db_txn_count.inc_by(context.db_txn_count, self.name) + block_db_txn_duration.inc_by(context.db_txn_duration, self.name) From 2c1fbea5319db2c64fa486adb32b5e66680b6daf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Feb 2016 10:22:44 +0000 Subject: [PATCH 264/294] Fix up logcontexts --- synapse/api/auth.py | 4 +- synapse/app/homeserver.py | 2 + synapse/crypto/keyring.py | 83 +++++++++-------- synapse/federation/federation_server.py | 4 +- synapse/federation/transaction_queue.py | 3 - synapse/handlers/_base.py | 10 +- synapse/handlers/events.py | 11 ++- synapse/handlers/federation.py | 50 +--------- synapse/handlers/presence.py | 20 ++-- synapse/handlers/register.py | 2 +- synapse/handlers/room.py | 11 ++- synapse/handlers/sync.py | 40 ++++---- synapse/http/server.py | 5 +- synapse/notifier.py | 58 ++++++------ synapse/push/__init__.py | 2 +- synapse/push/pusherpool.py | 9 +- synapse/rest/client/v2_alpha/account_data.py | 4 +- synapse/rest/client/v2_alpha/tags.py | 4 +- synapse/storage/_base.py | 18 ++-- synapse/storage/events.py | 34 ++++--- synapse/storage/presence.py | 5 +- synapse/storage/stream.py | 9 +- synapse/util/__init__.py | 6 +- synapse/util/async.py | 11 ++- synapse/util/caches/descriptors.py | 16 +++- synapse/util/caches/snapshot_cache.py | 3 +- synapse/util/distributor.py | 15 +-- synapse/util/logcontext.py | 98 +++++++++++++++++++- synapse/util/logutils.py | 35 +++++++ synapse/util/metrics.py | 10 +- synapse/util/ratelimitutils.py | 3 +- 31 files changed, 356 insertions(+), 229 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 5bba9343f..e2f84c4d5 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -24,6 +24,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError, EventSizeError from synapse.types import Requester, RoomID, UserID, EventID from synapse.util.logutils import log_function +from synapse.util.logcontext import preserve_context_over_fn from unpaddedbase64 import decode_base64 import logging @@ -529,7 +530,8 @@ class Auth(object): default=[""] )[0] if user and access_token and ip_addr: - self.store.insert_client_ip( + preserve_context_over_fn( + self.store.insert_client_ip, user=user, access_token=access_token, ip=ip_addr, diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e5c7e39cf..2b4be7bdd 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -709,6 +709,8 @@ def run(hs): phone_home_task.start(60 * 60 * 24, now=False) def in_thread(): + # Uncomment to enable tracing of log context changes. + # sys.settrace(logcontext_tracer) with LoggingContext("run"): change_resource_limit(hs.config.soft_file_limit) reactor.run() diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index cddec0b2b..d08ee0aa9 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -18,6 +18,10 @@ from synapse.api.errors import SynapseError, Codes from synapse.util.retryutils import get_retry_limiter from synapse.util import unwrapFirstError from synapse.util.async import ObservableDeferred +from synapse.util.logcontext import ( + preserve_context_over_deferred, preserve_context_over_fn, PreserveLoggingContext, + preserve_fn +) from twisted.internet import defer @@ -142,40 +146,43 @@ class Keyring(object): for server_name, _ in server_and_json } - # We want to wait for any previous lookups to complete before - # proceeding. - wait_on_deferred = self.wait_for_previous_lookups( - [server_name for server_name, _ in server_and_json], - server_to_deferred, - ) + with PreserveLoggingContext(): - # Actually start fetching keys. - wait_on_deferred.addBoth( - lambda _: self.get_server_verify_keys(group_id_to_group, deferreds) - ) + # We want to wait for any previous lookups to complete before + # proceeding. + wait_on_deferred = self.wait_for_previous_lookups( + [server_name for server_name, _ in server_and_json], + server_to_deferred, + ) - # When we've finished fetching all the keys for a given server_name, - # resolve the deferred passed to `wait_for_previous_lookups` so that - # any lookups waiting will proceed. - server_to_gids = {} + # Actually start fetching keys. + wait_on_deferred.addBoth( + lambda _: self.get_server_verify_keys(group_id_to_group, deferreds) + ) - def remove_deferreds(res, server_name, group_id): - server_to_gids[server_name].discard(group_id) - if not server_to_gids[server_name]: - d = server_to_deferred.pop(server_name, None) - if d: - d.callback(None) - return res + # When we've finished fetching all the keys for a given server_name, + # resolve the deferred passed to `wait_for_previous_lookups` so that + # any lookups waiting will proceed. + server_to_gids = {} - for g_id, deferred in deferreds.items(): - server_name = group_id_to_group[g_id].server_name - server_to_gids.setdefault(server_name, set()).add(g_id) - deferred.addBoth(remove_deferreds, server_name, g_id) + def remove_deferreds(res, server_name, group_id): + server_to_gids[server_name].discard(group_id) + if not server_to_gids[server_name]: + d = server_to_deferred.pop(server_name, None) + if d: + d.callback(None) + return res + + for g_id, deferred in deferreds.items(): + server_name = group_id_to_group[g_id].server_name + server_to_gids.setdefault(server_name, set()).add(g_id) + deferred.addBoth(remove_deferreds, server_name, g_id) # Pass those keys to handle_key_deferred so that the json object # signatures can be verified return [ - handle_key_deferred( + preserve_context_over_fn( + handle_key_deferred, group_id_to_group[g_id], deferreds[g_id], ) @@ -198,12 +205,13 @@ class Keyring(object): if server_name in self.key_downloads ] if wait_on: - yield defer.DeferredList(wait_on) + with PreserveLoggingContext(): + yield defer.DeferredList(wait_on) else: break for server_name, deferred in server_to_deferred.items(): - d = ObservableDeferred(deferred) + d = ObservableDeferred(preserve_context_over_deferred(deferred)) self.key_downloads[server_name] = d def rm(r, server_name): @@ -244,12 +252,13 @@ class Keyring(object): for group in group_id_to_group.values(): for key_id in group.key_ids: if key_id in merged_results[group.server_name]: - group_id_to_deferred[group.group_id].callback(( - group.group_id, - group.server_name, - key_id, - merged_results[group.server_name][key_id], - )) + with PreserveLoggingContext(): + group_id_to_deferred[group.group_id].callback(( + group.group_id, + group.server_name, + key_id, + merged_results[group.server_name][key_id], + )) break else: missing_groups.setdefault( @@ -504,7 +513,7 @@ class Keyring(object): yield defer.gatherResults( [ - self.store_keys( + preserve_fn(self.store_keys)( server_name=key_server_name, from_server=server_name, verify_keys=verify_keys, @@ -573,7 +582,7 @@ class Keyring(object): yield defer.gatherResults( [ - self.store.store_server_keys_json( + preserve_fn(self.store.store_server_keys_json)( server_name=server_name, key_id=key_id, from_server=server_name, @@ -675,7 +684,7 @@ class Keyring(object): # TODO(markjh): Store whether the keys have expired. yield defer.gatherResults( [ - self.store.store_server_verify_key( + preserve_fn(self.store.store_server_verify_key)( server_name, server_name, key.time_added, key ) for key_id, key in verify_keys.items() diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index a97aa0c94..90718192d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -126,10 +126,8 @@ class FederationServer(FederationBase): results = [] for pdu in pdu_list: - d = self._handle_new_pdu(transaction.origin, pdu) - try: - yield d + yield self._handle_new_pdu(transaction.origin, pdu) results.append({}) except FederationError as e: self.send_failure(e, transaction.origin) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 622adad3a..1928da03b 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -103,7 +103,6 @@ class TransactionQueue(object): else: return not destination.startswith("localhost") - @defer.inlineCallbacks def enqueue_pdu(self, pdu, destinations, order): # We loop through all destinations to see whether we already have # a transaction in progress. If we do, stick it in the pending_pdus @@ -141,8 +140,6 @@ class TransactionQueue(object): deferreds.append(deferred) - yield defer.DeferredList(deferreds, consumeErrors=True) - # NO inlineCallbacks def enqueue_edu(self, edu): destination = edu.destination diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 1423df6cf..fa83d3e46 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -293,19 +293,11 @@ class BaseHandler(object): with PreserveLoggingContext(): # Don't block waiting on waking up all the listeners. - notify_d = self.notifier.on_new_room_event( + self.notifier.on_new_room_event( event, event_stream_id, max_stream_id, extra_users=extra_users ) - def log_failure(f): - logger.warn( - "Failed to notify about %s: %s", - event.event_id, f.value - ) - - notify_d.addErrback(log_failure) - # If invite, remove room_state from unsigned before sending. event.unsigned.pop("invite_room_state", None) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 5ad8f3779..4933c31c1 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.types import UserID from synapse.events.utils import serialize_event +from synapse.util.logcontext import preserve_context_over_fn from ._base import BaseHandler @@ -29,11 +30,17 @@ logger = logging.getLogger(__name__) def started_user_eventstream(distributor, user): - return distributor.fire("started_user_eventstream", user) + return preserve_context_over_fn( + distributor.fire, + "started_user_eventstream", user + ) def stopped_user_eventstream(distributor, user): - return distributor.fire("stopped_user_eventstream", user) + return preserve_context_over_fn( + distributor.fire, + "stopped_user_eventstream", user + ) class EventStreamHandler(BaseHandler): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2ce1e9d6c..b78b0502d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -221,19 +221,11 @@ class FederationHandler(BaseHandler): extra_users.append(target_user) with PreserveLoggingContext(): - d = self.notifier.on_new_room_event( + self.notifier.on_new_room_event( event, event_stream_id, max_stream_id, extra_users=extra_users ) - def log_failure(f): - logger.warn( - "Failed to notify about %s: %s", - event.event_id, f.value - ) - - d.addErrback(log_failure) - if event.type == EventTypes.Member: if event.membership == Membership.JOIN: prev_state = context.current_state.get((event.type, event.state_key)) @@ -643,19 +635,11 @@ class FederationHandler(BaseHandler): ) with PreserveLoggingContext(): - d = self.notifier.on_new_room_event( + self.notifier.on_new_room_event( event, event_stream_id, max_stream_id, extra_users=[joinee] ) - def log_failure(f): - logger.warn( - "Failed to notify about %s: %s", - event.event_id, f.value - ) - - d.addErrback(log_failure) - logger.debug("Finished joining %s to %s", joinee, room_id) finally: room_queue = self.room_queues[room_id] @@ -730,18 +714,10 @@ class FederationHandler(BaseHandler): extra_users.append(target_user) with PreserveLoggingContext(): - d = self.notifier.on_new_room_event( + self.notifier.on_new_room_event( event, event_stream_id, max_stream_id, extra_users=extra_users ) - def log_failure(f): - logger.warn( - "Failed to notify about %s: %s", - event.event_id, f.value - ) - - d.addErrback(log_failure) - if event.type == EventTypes.Member: if event.content["membership"] == Membership.JOIN: user = UserID.from_string(event.state_key) @@ -811,19 +787,11 @@ class FederationHandler(BaseHandler): target_user = UserID.from_string(event.state_key) with PreserveLoggingContext(): - d = self.notifier.on_new_room_event( + self.notifier.on_new_room_event( event, event_stream_id, max_stream_id, extra_users=[target_user], ) - def log_failure(f): - logger.warn( - "Failed to notify about %s: %s", - event.event_id, f.value - ) - - d.addErrback(log_failure) - defer.returnValue(event) @defer.inlineCallbacks @@ -948,18 +916,10 @@ class FederationHandler(BaseHandler): extra_users.append(target_user) with PreserveLoggingContext(): - d = self.notifier.on_new_room_event( + self.notifier.on_new_room_event( event, event_stream_id, max_stream_id, extra_users=extra_users ) - def log_failure(f): - logger.warn( - "Failed to notify about %s: %s", - event.event_id, f.value - ) - - d.addErrback(log_failure) - new_pdu = event destinations = set() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index d0c21ff5c..b61394f2b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -378,9 +378,9 @@ class PresenceHandler(BaseHandler): was_polling = target_user in self._user_cachemap if now_online and not was_polling: - self.start_polling_presence(target_user, state=state) + yield self.start_polling_presence(target_user, state=state) elif not now_online and was_polling: - self.stop_polling_presence(target_user) + yield self.stop_polling_presence(target_user) # TODO(paul): perform a presence push as part of start/stop poll so # we don't have to do this all the time @@ -394,7 +394,8 @@ class PresenceHandler(BaseHandler): if now - prev_state.state.get("last_active", 0) < LAST_ACTIVE_GRANULARITY: return - self.changed_presencelike_data(user, {"last_active": now}) + with PreserveLoggingContext(): + self.changed_presencelike_data(user, {"last_active": now}) def get_joined_rooms_for_user(self, user): """Get the list of rooms a user is joined to. @@ -466,11 +467,12 @@ class PresenceHandler(BaseHandler): local_user, room_ids=[room_id], add_to_cache=False ) - self.push_update_to_local_and_remote( - observed_user=local_user, - users_to_push=[user], - statuscache=statuscache, - ) + with PreserveLoggingContext(): + self.push_update_to_local_and_remote( + observed_user=local_user, + users_to_push=[user], + statuscache=statuscache, + ) @defer.inlineCallbacks def send_presence_invite(self, observer_user, observed_user): @@ -556,7 +558,7 @@ class PresenceHandler(BaseHandler): observer_user.localpart, observed_user.to_string() ) - self.start_polling_presence( + yield self.start_polling_presence( observer_user, target_user=observed_user ) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 2660fd21a..24c850ae9 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -186,7 +186,7 @@ class RegistrationHandler(BaseHandler): token=token, password_hash="" ) - registered_user(self.distributor, user) + yield registered_user(self.distributor, user) defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index bfd7e44e9..a8e3a9029 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -25,6 +25,7 @@ from synapse.api.constants import ( from synapse.api.errors import AuthError, StoreError, SynapseError, Codes from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor +from synapse.util.logcontext import preserve_context_over_fn from signedjson.sign import verify_signed_json from signedjson.key import decode_verify_key_bytes @@ -46,11 +47,17 @@ def collect_presencelike_data(distributor, user, content): def user_left_room(distributor, user, room_id): - return distributor.fire("user_left_room", user=user, room_id=room_id) + return preserve_context_over_fn( + distributor.fire, + "user_left_room", user=user, room_id=room_id + ) def user_joined_room(distributor, user, room_id): - return distributor.fire("user_joined_room", user=user, room_id=room_id) + return preserve_context_over_fn( + distributor.fire, + "user_joined_room", user=user, room_id=room_id + ) class RoomCreationHandler(BaseHandler): diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 72271f262..3f1cda5b0 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,7 +18,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes from synapse.util import unwrapFirstError -from synapse.util.logcontext import LoggingContext +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from twisted.internet import defer @@ -241,15 +241,16 @@ class SyncHandler(BaseHandler): deferreds = [] for event in room_list: if event.membership == Membership.JOIN: - room_sync_deferred = self.full_state_sync_for_joined_room( - room_id=event.room_id, - sync_config=sync_config, - now_token=now_token, - timeline_since_token=timeline_since_token, - ephemeral_by_room=ephemeral_by_room, - tags_by_room=tags_by_room, - account_data_by_room=account_data_by_room, - ) + with PreserveLoggingContext(LoggingContext.current_context()): + room_sync_deferred = self.full_state_sync_for_joined_room( + room_id=event.room_id, + sync_config=sync_config, + now_token=now_token, + timeline_since_token=timeline_since_token, + ephemeral_by_room=ephemeral_by_room, + tags_by_room=tags_by_room, + account_data_by_room=account_data_by_room, + ) room_sync_deferred.addCallback(joined.append) deferreds.append(room_sync_deferred) elif event.membership == Membership.INVITE: @@ -262,15 +263,16 @@ class SyncHandler(BaseHandler): leave_token = now_token.copy_and_replace( "room_key", "s%d" % (event.stream_ordering,) ) - room_sync_deferred = self.full_state_sync_for_archived_room( - sync_config=sync_config, - room_id=event.room_id, - leave_event_id=event.event_id, - leave_token=leave_token, - timeline_since_token=timeline_since_token, - tags_by_room=tags_by_room, - account_data_by_room=account_data_by_room, - ) + with PreserveLoggingContext(LoggingContext.current_context()): + room_sync_deferred = self.full_state_sync_for_archived_room( + sync_config=sync_config, + room_id=event.room_id, + leave_event_id=event.event_id, + leave_token=leave_token, + timeline_since_token=timeline_since_token, + tags_by_room=tags_by_room, + account_data_by_room=account_data_by_room, + ) room_sync_deferred.addCallback(archived.append) deferreds.append(room_sync_deferred) diff --git a/synapse/http/server.py b/synapse/http/server.py index 06935783c..a90e2e112 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -99,9 +99,8 @@ def request_handler(request_handler): request_context.request = request_id with request.processing(): try: - d = request_handler(self, request) - with PreserveLoggingContext(): - yield d + with PreserveLoggingContext(request_context): + yield request_handler(self, request) except CodeMessageException as e: code = e.code if isinstance(e, SynapseError): diff --git a/synapse/notifier.py b/synapse/notifier.py index 1a90bd55c..560866b26 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -18,7 +18,8 @@ from synapse.api.constants import EventTypes from synapse.api.errors import AuthError from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor, ObservableDeferred +from synapse.util.async import ObservableDeferred +from synapse.util.logcontext import PreserveLoggingContext from synapse.types import StreamToken import synapse.metrics @@ -73,7 +74,8 @@ class _NotifierUserStream(object): self.current_token = current_token self.last_notified_ms = time_now_ms - self.notify_deferred = ObservableDeferred(defer.Deferred()) + with PreserveLoggingContext(): + self.notify_deferred = ObservableDeferred(defer.Deferred()) def notify(self, stream_key, stream_id, time_now_ms): """Notify any listeners for this user of a new event from an @@ -88,8 +90,10 @@ class _NotifierUserStream(object): ) self.last_notified_ms = time_now_ms noify_deferred = self.notify_deferred - self.notify_deferred = ObservableDeferred(defer.Deferred()) - noify_deferred.callback(self.current_token) + + with PreserveLoggingContext(): + self.notify_deferred = ObservableDeferred(defer.Deferred()) + noify_deferred.callback(self.current_token) def remove(self, notifier): """ Remove this listener from all the indexes in the Notifier @@ -184,8 +188,6 @@ class Notifier(object): lambda: count(bool, self.appservice_to_user_streams.values()), ) - @log_function - @defer.inlineCallbacks def on_new_room_event(self, event, room_stream_id, max_room_stream_id, extra_users=[]): """ Used by handlers to inform the notifier something has happened @@ -199,12 +201,11 @@ class Notifier(object): until all previous events have been persisted before notifying the client streams. """ - yield run_on_reactor() - - self.pending_new_room_events.append(( - room_stream_id, event, extra_users - )) - self._notify_pending_new_room_events(max_room_stream_id) + with PreserveLoggingContext(): + self.pending_new_room_events.append(( + room_stream_id, event, extra_users + )) + self._notify_pending_new_room_events(max_room_stream_id) def _notify_pending_new_room_events(self, max_room_stream_id): """Notify for the room events that were queued waiting for a previous @@ -251,31 +252,29 @@ class Notifier(object): extra_streams=app_streams, ) - @defer.inlineCallbacks - @log_function def on_new_event(self, stream_key, new_token, users=[], rooms=[], extra_streams=set()): """ Used to inform listeners that something has happend event wise. Will wake up all listeners for the given users and rooms. """ - yield run_on_reactor() - user_streams = set() + with PreserveLoggingContext(): + user_streams = set() - for user in users: - user_stream = self.user_to_user_stream.get(str(user)) - if user_stream is not None: - user_streams.add(user_stream) + for user in users: + user_stream = self.user_to_user_stream.get(str(user)) + if user_stream is not None: + user_streams.add(user_stream) - for room in rooms: - user_streams |= self.room_to_user_streams.get(room, set()) + for room in rooms: + user_streams |= self.room_to_user_streams.get(room, set()) - time_now_ms = self.clock.time_msec() - for user_stream in user_streams: - try: - user_stream.notify(stream_key, new_token, time_now_ms) - except: - logger.exception("Failed to notify listener") + time_now_ms = self.clock.time_msec() + for user_stream in user_streams: + try: + user_stream.notify(stream_key, new_token, time_now_ms) + except: + logger.exception("Failed to notify listener") @defer.inlineCallbacks def wait_for_events(self, user_id, timeout, callback, room_ids=None, @@ -325,7 +324,8 @@ class Notifier(object): # that we don't miss any current_token updates. prev_token = current_token listener = user_stream.new_listener(prev_token) - yield listener.deferred + with PreserveLoggingContext(): + yield listener.deferred except defer.CancelledError: break diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 64e581b8b..8da2d8716 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -111,7 +111,7 @@ class Pusher(object): self.user_id, config, timeout=0, affect_presence=False ) self.last_token = chunk['end'] - self.store.update_pusher_last_token( + yield self.store.update_pusher_last_token( self.app_id, self.pushkey, self.user_id, self.last_token ) logger.info("New pusher %s for user %s starting from token %s", diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index d1b7c0802..d7dcb2de4 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,6 +18,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from synapse.util.logcontext import preserve_fn import logging @@ -76,7 +77,7 @@ class PusherPool: "Removing pusher for app id %s, pushkey %s, user %s", app_id, pushkey, p['user_name'] ) - self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + yield self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) @defer.inlineCallbacks def remove_pushers_by_user(self, user_id): @@ -91,7 +92,7 @@ class PusherPool: "Removing pusher for app id %s, pushkey %s, user %s", p['app_id'], p['pushkey'], p['user_name'] ) - self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + yield self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) @defer.inlineCallbacks def _add_pusher_to_store(self, user_id, access_token, profile_tag, kind, @@ -110,7 +111,7 @@ class PusherPool: lang=lang, data=data, ) - self._refresh_pusher(app_id, pushkey, user_id) + yield self._refresh_pusher(app_id, pushkey, user_id) def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': @@ -166,7 +167,7 @@ class PusherPool: if fullid in self.pushers: self.pushers[fullid].stop() self.pushers[fullid] = p - p.start() + preserve_fn(p.start)() logger.info("Started pushers") diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py index 985efe2a6..1456881c1 100644 --- a/synapse/rest/client/v2_alpha/account_data.py +++ b/synapse/rest/client/v2_alpha/account_data.py @@ -57,7 +57,7 @@ class AccountDataServlet(RestServlet): user_id, account_data_type, body ) - yield self.notifier.on_new_event( + self.notifier.on_new_event( "account_data_key", max_id, users=[user_id] ) @@ -99,7 +99,7 @@ class RoomAccountDataServlet(RestServlet): user_id, room_id, account_data_type, body ) - yield self.notifier.on_new_event( + self.notifier.on_new_event( "account_data_key", max_id, users=[user_id] ) diff --git a/synapse/rest/client/v2_alpha/tags.py b/synapse/rest/client/v2_alpha/tags.py index 42f2203f3..79c436a8c 100644 --- a/synapse/rest/client/v2_alpha/tags.py +++ b/synapse/rest/client/v2_alpha/tags.py @@ -80,7 +80,7 @@ class TagServlet(RestServlet): max_id = yield self.store.add_tag_to_room(user_id, room_id, tag, body) - yield self.notifier.on_new_event( + self.notifier.on_new_event( "account_data_key", max_id, users=[user_id] ) @@ -94,7 +94,7 @@ class TagServlet(RestServlet): max_id = yield self.store.remove_tag_from_room(user_id, room_id, tag) - yield self.notifier.on_new_event( + self.notifier.on_new_event( "account_data_key", max_id, users=[user_id] ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index cfb87d932..2e97ac84a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -15,7 +15,7 @@ import logging from synapse.api.errors import StoreError -from synapse.util.logcontext import preserve_context_over_fn, LoggingContext +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.caches.dictionary_cache import DictionaryCache from synapse.util.caches.descriptors import Cache import synapse.metrics @@ -298,10 +298,10 @@ class SQLBaseStore(object): func, *args, **kwargs ) - result = yield preserve_context_over_fn( - self._db_pool.runWithConnection, - inner_func, *args, **kwargs - ) + with PreserveLoggingContext(): + result = yield self._db_pool.runWithConnection( + inner_func, *args, **kwargs + ) for after_callback, after_args in after_callbacks: after_callback(*after_args) @@ -326,10 +326,10 @@ class SQLBaseStore(object): return func(conn, *args, **kwargs) - result = yield preserve_context_over_fn( - self._db_pool.runWithConnection, - inner_func, *args, **kwargs - ) + with PreserveLoggingContext(): + result = yield self._db_pool.runWithConnection( + inner_func, *args, **kwargs + ) defer.returnValue(result) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 4d7cdd00d..c6ed54721 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -19,7 +19,7 @@ from twisted.internet import defer, reactor from synapse.events import FrozenEvent, USE_FROZEN_DICTS from synapse.events.utils import prune_event -from synapse.util.logcontext import preserve_context_over_deferred +from synapse.util.logcontext import preserve_fn, PreserveLoggingContext from synapse.util.logutils import log_function from synapse.api.constants import EventTypes @@ -664,14 +664,16 @@ class EventsStore(SQLBaseStore): for ids, d in lst: if not d.called: try: - d.callback([ - res[i] - for i in ids - if i in res - ]) + with PreserveLoggingContext(): + d.callback([ + res[i] + for i in ids + if i in res + ]) except: logger.exception("Failed to callback") - reactor.callFromThread(fire, event_list, row_dict) + with PreserveLoggingContext(): + reactor.callFromThread(fire, event_list, row_dict) except Exception as e: logger.exception("do_fetch") @@ -679,10 +681,12 @@ class EventsStore(SQLBaseStore): def fire(evs): for _, d in evs: if not d.called: - d.errback(e) + with PreserveLoggingContext(): + d.errback(e) if event_list: - reactor.callFromThread(fire, event_list) + with PreserveLoggingContext(): + reactor.callFromThread(fire, event_list) @defer.inlineCallbacks def _enqueue_events(self, events, check_redacted=True, @@ -709,18 +713,20 @@ class EventsStore(SQLBaseStore): should_start = False if should_start: - self.runWithConnection( - self._do_fetch - ) + with PreserveLoggingContext(): + self.runWithConnection( + self._do_fetch + ) - rows = yield preserve_context_over_deferred(events_d) + with PreserveLoggingContext(): + rows = yield events_d if not allow_rejected: rows[:] = [r for r in rows if not r["rejects"]] res = yield defer.gatherResults( [ - self._get_event_from_row( + preserve_fn(self._get_event_from_row)( row["internal_metadata"], row["json"], row["redacts"], check_redacted=check_redacted, get_prev_content=get_prev_content, diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 9b3aecaf8..ef525f34c 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -68,8 +68,9 @@ class PresenceStore(SQLBaseStore): for row in rows }) + @defer.inlineCallbacks def set_presence_state(self, user_localpart, new_state): - res = self._simple_update_one( + res = yield self._simple_update_one( table="presence", keyvalues={"user_id": user_localpart}, updatevalues={"state": new_state["state"], @@ -79,7 +80,7 @@ class PresenceStore(SQLBaseStore): ) self.get_presence_state.invalidate((user_localpart,)) - return res + defer.returnValue(res) def allow_presence_visible(self, observed_localpart, observer_userid): return self._simple_insert( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 50436cb2d..367ffc954 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -39,6 +39,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.api.constants import EventTypes from synapse.types import RoomStreamToken +from synapse.util.logcontext import preserve_fn import logging @@ -170,12 +171,12 @@ class StreamStore(SQLBaseStore): room_ids = list(room_ids) for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)): res = yield defer.gatherResults([ - self.get_room_events_stream_for_room( - room_id, from_key, to_key, limit - ).addCallback(lambda r, rm: (rm, r), room_id) + preserve_fn(self.get_room_events_stream_for_room)( + room_id, from_key, to_key, limit, + ) for room_id in room_ids ]) - results.update(dict(res)) + results.update(dict(zip(rm_ids, res))) defer.returnValue(results) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 7566d9eb3..133671e23 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.util.logcontext import LoggingContext, PreserveLoggingContext +from synapse.util.logcontext import PreserveLoggingContext from twisted.internet import defer, reactor, task @@ -61,10 +61,8 @@ class Clock(object): *args: Postional arguments to pass to function. **kwargs: Key arguments to pass to function. """ - current_context = LoggingContext.current_context() - def wrapped_callback(*args, **kwargs): - with PreserveLoggingContext(current_context): + with PreserveLoggingContext(): callback(*args, **kwargs) with PreserveLoggingContext(): diff --git a/synapse/util/async.py b/synapse/util/async.py index 200edd404..640fae389 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -16,13 +16,16 @@ from twisted.internet import defer, reactor -from .logcontext import preserve_context_over_deferred +from .logcontext import PreserveLoggingContext +@defer.inlineCallbacks def sleep(seconds): d = defer.Deferred() - reactor.callLater(seconds, d.callback, seconds) - return preserve_context_over_deferred(d) + with PreserveLoggingContext(): + reactor.callLater(seconds, d.callback, seconds) + res = yield d + defer.returnValue(res) def run_on_reactor(): @@ -54,6 +57,7 @@ class ObservableDeferred(object): object.__setattr__(self, "_result", (True, r)) while self._observers: try: + # TODO: Handle errors here. self._observers.pop().callback(r) except: pass @@ -63,6 +67,7 @@ class ObservableDeferred(object): object.__setattr__(self, "_result", (False, f)) while self._observers: try: + # TODO: Handle errors here. self._observers.pop().errback(f) except: pass diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index e27917c63..277854ccb 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -18,6 +18,9 @@ from synapse.util.async import ObservableDeferred from synapse.util import unwrapFirstError from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache +from synapse.util.logcontext import ( + PreserveLoggingContext, preserve_context_over_deferred, preserve_context_over_fn +) from . import caches_by_name, DEBUG_CACHES, cache_counter @@ -190,7 +193,7 @@ class CacheDescriptor(object): defer.returnValue(cached_result) observer.addCallback(check_result) - return observer + return preserve_context_over_deferred(observer) except KeyError: # Get the sequence number of the cache before reading from the # database so that we can tell if the cache is invalidated @@ -198,6 +201,7 @@ class CacheDescriptor(object): sequence = self.cache.sequence ret = defer.maybeDeferred( + preserve_context_over_fn, self.function_to_call, obj, *args, **kwargs ) @@ -211,7 +215,7 @@ class CacheDescriptor(object): ret = ObservableDeferred(ret, consumeErrors=True) self.cache.update(sequence, cache_key, ret) - return ret.observe() + return preserve_context_over_deferred(ret.observe()) wrapped.invalidate = self.cache.invalidate wrapped.invalidate_all = self.cache.invalidate_all @@ -299,6 +303,7 @@ class CacheListDescriptor(object): args_to_call[self.list_name] = missing ret_d = defer.maybeDeferred( + preserve_context_over_fn, self.function_to_call, **args_to_call ) @@ -308,7 +313,8 @@ class CacheListDescriptor(object): # We need to create deferreds for each arg in the list so that # we can insert the new deferred into the cache. for arg in missing: - observer = ret_d.observe() + with PreserveLoggingContext(): + observer = ret_d.observe() observer.addCallback(lambda r, arg: r.get(arg, None), arg) observer = ObservableDeferred(observer) @@ -327,10 +333,10 @@ class CacheListDescriptor(object): cached[arg] = res - return defer.gatherResults( + return preserve_context_over_deferred(defer.gatherResults( cached.values(), consumeErrors=True, - ).addErrback(unwrapFirstError).addCallback(lambda res: dict(res)) + ).addErrback(unwrapFirstError).addCallback(lambda res: dict(res))) obj.__dict__[self.orig.__name__] = wrapped diff --git a/synapse/util/caches/snapshot_cache.py b/synapse/util/caches/snapshot_cache.py index b1e40417f..d03678b8c 100644 --- a/synapse/util/caches/snapshot_cache.py +++ b/synapse/util/caches/snapshot_cache.py @@ -87,7 +87,8 @@ class SnapshotCache(object): # expire from the rotation of that cache. self.next_result_cache[key] = result self.pending_result_cache.pop(key, None) + return r - result.observe().addBoth(shuffle_along) + result.addBoth(shuffle_along) return result.observe() diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 4ebfebf70..8875813de 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -15,9 +15,7 @@ from twisted.internet import defer -from synapse.util.logcontext import ( - PreserveLoggingContext, preserve_context_over_deferred, -) +from synapse.util.logcontext import PreserveLoggingContext from synapse.util import unwrapFirstError @@ -97,6 +95,7 @@ class Signal(object): Each observer callable may return a Deferred.""" self.observers.append(observer) + @defer.inlineCallbacks def fire(self, *args, **kwargs): """Invokes every callable in the observer list, passing in the args and kwargs. Exceptions thrown by observers are logged but ignored. It is @@ -116,6 +115,7 @@ class Signal(object): failure.getTracebackObject())) if not self.suppress_failures: return failure + return defer.maybeDeferred(observer, *args, **kwargs).addErrback(eb) with PreserveLoggingContext(): @@ -124,8 +124,11 @@ class Signal(object): for observer in self.observers ] - d = defer.gatherResults(deferreds, consumeErrors=True) + res = yield defer.gatherResults( + deferreds, consumeErrors=True + ).addErrback(unwrapFirstError) - d.addErrback(unwrapFirstError) + defer.returnValue(res) - return preserve_context_over_deferred(d) + def __repr__(self): + return "" % (self.name,) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index e701092cd..9134e6790 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -48,7 +48,7 @@ class LoggingContext(object): __slots__ = [ "parent_context", "name", "usage_start", "usage_end", "main_thread", - "__dict__", "tag", + "__dict__", "tag", "alive", ] thread_local = threading.local() @@ -88,6 +88,7 @@ class LoggingContext(object): self.usage_start = None self.main_thread = threading.current_thread() self.tag = "" + self.alive = True def __str__(self): return "%s@%x" % (self.name, id(self)) @@ -106,6 +107,7 @@ class LoggingContext(object): The context that was previously active """ current = cls.current_context() + if current is not context: current.stop() cls.thread_local.current_context = context @@ -117,6 +119,7 @@ class LoggingContext(object): if self.parent_context is not None: raise Exception("Attempt to enter logging context multiple times") self.parent_context = self.set_current_context(self) + self.alive = True return self def __exit__(self, type, value, traceback): @@ -136,6 +139,7 @@ class LoggingContext(object): self ) self.parent_context = None + self.alive = False def __getattr__(self, name): """Delegate member lookup to parent context""" @@ -213,7 +217,7 @@ class PreserveLoggingContext(object): exited. Used to restore the context after a function using @defer.inlineCallbacks is resumed by a callback from the reactor.""" - __slots__ = ["current_context", "new_context"] + __slots__ = ["current_context", "new_context", "has_parent"] def __init__(self, new_context=LoggingContext.sentinel): self.new_context = new_context @@ -224,11 +228,26 @@ class PreserveLoggingContext(object): self.new_context ) + if self.current_context: + self.has_parent = self.current_context.parent_context is not None + if not self.current_context.alive: + logger.warn( + "Entering dead context: %s", + self.current_context, + ) + def __exit__(self, type, value, traceback): """Restores the current logging context""" - LoggingContext.set_current_context(self.current_context) + context = LoggingContext.set_current_context(self.current_context) + + if context != self.new_context: + logger.warn( + "Unexpected logging context: %s is not %s", + context, self.new_context, + ) + if self.current_context is not LoggingContext.sentinel: - if self.current_context.parent_context is None: + if not self.current_context.alive: logger.warn( "Restoring dead context: %s", self.current_context, @@ -289,3 +308,74 @@ def preserve_context_over_deferred(deferred): d = _PreservingContextDeferred(current_context) deferred.chainDeferred(d) return d + + +def preserve_fn(f): + """Ensures that function is called with correct context and that context is + restored after return. Useful for wrapping functions that return a deferred + which you don't yield on. + """ + current = LoggingContext.current_context() + + def g(*args, **kwargs): + with PreserveLoggingContext(current): + return f(*args, **kwargs) + + return g + + +# modules to ignore in `logcontext_tracer` +_to_ignore = [ + "synapse.util.logcontext", + "synapse.http.server", + "synapse.storage._base", + "synapse.util.async", +] + + +def logcontext_tracer(frame, event, arg): + """A tracer that logs whenever a logcontext "unexpectedly" changes within + a function. Probably inaccurate. + + Use by calling `sys.settrace(logcontext_tracer)` in the main thread. + """ + if event == 'call': + name = frame.f_globals["__name__"] + if name.startswith("synapse"): + if name == "synapse.util.logcontext": + if frame.f_code.co_name in ["__enter__", "__exit__"]: + tracer = frame.f_back.f_trace + if tracer: + tracer.just_changed = True + + tracer = frame.f_trace + if tracer: + return tracer + + if not any(name.startswith(ig) for ig in _to_ignore): + return LineTracer() + + +class LineTracer(object): + __slots__ = ["context", "just_changed"] + + def __init__(self): + self.context = LoggingContext.current_context() + self.just_changed = False + + def __call__(self, frame, event, arg): + if event in 'line': + if self.just_changed: + self.context = LoggingContext.current_context() + self.just_changed = False + else: + c = LoggingContext.current_context() + if c != self.context: + logger.info( + "Context changed! %s -> %s, %s, %s", + self.context, c, + frame.f_code.co_filename, frame.f_lineno + ) + self.context = c + + return self diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index c37a15778..3a83828d2 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -168,3 +168,38 @@ def trace_function(f): wrapped.__name__ = func_name return wrapped + + +def get_previous_frames(): + s = inspect.currentframe().f_back.f_back + to_return = [] + while s: + if s.f_globals["__name__"].startswith("synapse"): + filename, lineno, function, _, _ = inspect.getframeinfo(s) + args_string = inspect.formatargvalues(*inspect.getargvalues(s)) + + to_return.append("{{ %s:%d %s - Args: %s }}" % ( + filename, lineno, function, args_string + )) + + s = s.f_back + + return ", ". join(to_return) + + +def get_previous_frame(ignore=[]): + s = inspect.currentframe().f_back.f_back + + while s: + if s.f_globals["__name__"].startswith("synapse"): + if not any(s.f_globals["__name__"].startswith(ig) for ig in ignore): + filename, lineno, function, _, _ = inspect.getframeinfo(s) + args_string = inspect.formatargvalues(*inspect.getargvalues(s)) + + return "{{ %s:%d %s - Args: %s }}" % ( + filename, lineno, function, args_string + ) + + s = s.f_back + + return None diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index daf6087fe..ca4800721 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -68,16 +68,18 @@ class Measure(object): block_timer.inc_by(duration, self.name) context = LoggingContext.current_context() - if not context: - return if context != self.start_context: logger.warn( - "Context have unexpectedly changed %r, %r", - context, self.start_context + "Context have unexpectedly changed from '%s' to '%s'. (%r)", + context, self.start_context, self.name ) return + if not context: + logger.warn("Expected context. (%r)", self.name) + return + ru_utime, ru_stime = context.get_resource_usage() block_ru_utime.inc_by(ru_utime, self.name) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index ea321bc6a..4076eed26 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError from synapse.util.async import sleep +from synapse.util.logcontext import preserve_fn import collections import contextlib @@ -163,7 +164,7 @@ class _PerHostRatelimiter(object): "Ratelimit [%s]: sleeping req", id(request_id), ) - ret_defer = sleep(self.sleep_msec / 1000.0) + ret_defer = preserve_fn(sleep)(self.sleep_msec / 1000.0) self.sleeping_requests.add(request_id) From f078ecbc8fa47075a155765ea07a2211dbbad86d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Feb 2016 16:35:44 +0000 Subject: [PATCH 265/294] Derive macaroon_secret_key from signing key. Unfortunately, there are people that are running synapse without a `macaroon_sercret_key` set. Mandating they set one is a good solution, except that breaking auto upgrades is annoying. --- synapse/config/key.py | 28 +++++++++++++++++++++++++++- synapse/config/registration.py | 18 +++--------------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/synapse/config/key.py b/synapse/config/key.py index ac90cd3fc..a072aec71 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -22,8 +22,14 @@ from signedjson.key import ( read_signing_keys, write_signing_keys, NACL_ED25519 ) from unpaddedbase64 import decode_base64 +from synapse.util.stringutils import random_string_with_symbols import os +import hashlib +import logging + + +logger = logging.getLogger(__name__) class KeyConfig(Config): @@ -40,9 +46,29 @@ class KeyConfig(Config): config["perspectives"] ) - def default_config(self, config_dir_path, server_name, **kwargs): + self.macaroon_secret_key = config.get( + "macaroon_secret_key", self.registration_shared_secret + ) + + if not self.macaroon_secret_key: + # Unfortunately, there are people out there that don't have this + # set. Lets just be "nice" and derive one from their secret key. + logger.warn("Config is missing missing macaroon_secret_key") + seed = self.signing_key[0].seed + self.macaroon_secret_key = hashlib.sha256(seed) + + def default_config(self, config_dir_path, server_name, is_generating_file=False, + **kwargs): base_key_name = os.path.join(config_dir_path, server_name) + + if is_generating_file: + macaroon_secret_key = random_string_with_symbols(50) + else: + macaroon_secret_key = None + return """\ + macaroon_secret_key: "%(macaroon_secret_key)s" + ## Signing Keys ## # Path to the signing key to sign messages with diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 9b6dacc5b..ab062d528 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -32,26 +32,14 @@ class RegistrationConfig(Config): ) self.registration_shared_secret = config.get("registration_shared_secret") - self.macaroon_secret_key = config.get("macaroon_secret_key") - if self.macaroon_secret_key is None: - raise Exception( - "Config is missing missing macaroon_secret_key - please set it" - " in your config file." - ) + self.bcrypt_rounds = config.get("bcrypt_rounds", 12) self.trusted_third_party_id_servers = config["trusted_third_party_id_servers"] self.allow_guest_access = config.get("allow_guest_access", False) - def default_config(self, is_generating_file=False, **kwargs): + def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) - macaroon_line = "" - if is_generating_file: - macaroon_line += '\n macaroon_secret_key: "%s"\n' % ( - random_string_with_symbols(50), - ) - - macaroon_secret_key = random_string_with_symbols(50) return """\ ## Registration ## @@ -61,7 +49,7 @@ class RegistrationConfig(Config): # If set, allows registration by anyone who also has the shared # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" -%(macaroon_line)s + # Set the number of bcrypt rounds used to generate password hash. # Larger numbers increase the work factor needed to generate the hash. # The default number of rounds is 12. From c486b7b41cafeb120815a53992318f911b0d30f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 09:20:06 +0000 Subject: [PATCH 266/294] Change logcontext warns to debug --- synapse/util/logcontext.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 9134e6790..b22a36336 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -231,7 +231,7 @@ class PreserveLoggingContext(object): if self.current_context: self.has_parent = self.current_context.parent_context is not None if not self.current_context.alive: - logger.warn( + logger.debug( "Entering dead context: %s", self.current_context, ) @@ -241,14 +241,14 @@ class PreserveLoggingContext(object): context = LoggingContext.set_current_context(self.current_context) if context != self.new_context: - logger.warn( + logger.debug( "Unexpected logging context: %s is not %s", context, self.new_context, ) if self.current_context is not LoggingContext.sentinel: if not self.current_context.alive: - logger.warn( + logger.debug( "Restoring dead context: %s", self.current_context, ) From 549698b1e025263b2c4d8aa3f9b20f1c8761dc59 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 09:37:09 +0000 Subject: [PATCH 267/294] Don't measure across event stream call, as it lasts for a long time. --- synapse/push/__init__.py | 244 +++++++++++++++++++-------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8da2d8716..b9522a805 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -128,8 +128,7 @@ class Pusher(object): try: if wait > 0: yield synapse.util.async.sleep(wait) - with Measure(self.clock, "push"): - yield self.get_and_dispatch() + yield self.get_and_dispatch() wait = 0 except: if wait == 0: @@ -151,115 +150,27 @@ class Pusher(object): only_keys=("room", "receipt",), ) - # limiting to 1 may get 1 event plus 1 presence event, so - # pick out the actual event - single_event = None - read_receipt = None - for c in chunk['chunk']: - if 'event_id' in c: # Hmmm... - single_event = c - elif c['type'] == 'm.receipt': - read_receipt = c + with Measure(self.clock, "push"): + # limiting to 1 may get 1 event plus 1 presence event, so + # pick out the actual event + single_event = None + read_receipt = None + for c in chunk['chunk']: + if 'event_id' in c: # Hmmm... + single_event = c + elif c['type'] == 'm.receipt': + read_receipt = c - have_updated_badge = False - if read_receipt: - for receipt_part in read_receipt['content'].values(): - if 'm.read' in receipt_part: - if self.user_id in receipt_part['m.read'].keys(): - have_updated_badge = True + have_updated_badge = False + if read_receipt: + for receipt_part in read_receipt['content'].values(): + if 'm.read' in receipt_part: + if self.user_id in receipt_part['m.read'].keys(): + have_updated_badge = True - if not single_event: - if have_updated_badge: - yield self.update_badge() - self.last_token = chunk['end'] - yield self.store.update_pusher_last_token( - self.app_id, - self.pushkey, - self.user_id, - self.last_token - ) - return - - if not self.alive: - return - - processed = False - - rule_evaluator = yield \ - push_rule_evaluator.evaluator_for_user_id_and_profile_tag( - self.user_id, self.profile_tag, single_event['room_id'], self.store - ) - - actions = yield rule_evaluator.actions_for_event(single_event) - tweaks = rule_evaluator.tweaks_for_actions(actions) - - if 'notify' in actions: - self.badge = yield self._get_badge_count() - rejected = yield self.dispatch_push(single_event, tweaks, self.badge) - self.has_unread = True - if isinstance(rejected, list) or isinstance(rejected, tuple): - processed = True - for pk in rejected: - if pk != self.pushkey: - # for sanity, we only remove the pushkey if it - # was the one we actually sent... - logger.warn( - ("Ignoring rejected pushkey %s because we" - " didn't send it"), pk - ) - else: - logger.info( - "Pushkey %s was rejected: removing", - pk - ) - yield self.hs.get_pusherpool().remove_pusher( - self.app_id, pk, self.user_id - ) - else: - if have_updated_badge: - yield self.update_badge() - processed = True - - if not self.alive: - return - - if processed: - self.backoff_delay = Pusher.INITIAL_BACKOFF - self.last_token = chunk['end'] - yield self.store.update_pusher_last_token_and_success( - self.app_id, - self.pushkey, - self.user_id, - self.last_token, - self.clock.time_msec() - ) - if self.failing_since: - self.failing_since = None - yield self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_id, - self.failing_since) - else: - if not self.failing_since: - self.failing_since = self.clock.time_msec() - yield self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_id, - self.failing_since - ) - - if (self.failing_since and - self.failing_since < - self.clock.time_msec() - Pusher.GIVE_UP_AFTER): - # we really only give up so that if the URL gets - # fixed, we don't suddenly deliver a load - # of old notifications. - logger.warn("Giving up on a notification to user %s, " - "pushkey %s", - self.user_id, self.pushkey) - self.backoff_delay = Pusher.INITIAL_BACKOFF + if not single_event: + if have_updated_badge: + yield self.update_badge() self.last_token = chunk['end'] yield self.store.update_pusher_last_token( self.app_id, @@ -267,25 +178,114 @@ class Pusher(object): self.user_id, self.last_token ) + return - self.failing_since = None - yield self.store.update_pusher_failing_since( + if not self.alive: + return + + processed = False + + rule_evaluator = yield \ + push_rule_evaluator.evaluator_for_user_id_and_profile_tag( + self.user_id, self.profile_tag, single_event['room_id'], self.store + ) + + actions = yield rule_evaluator.actions_for_event(single_event) + tweaks = rule_evaluator.tweaks_for_actions(actions) + + if 'notify' in actions: + self.badge = yield self._get_badge_count() + rejected = yield self.dispatch_push(single_event, tweaks, self.badge) + self.has_unread = True + if isinstance(rejected, list) or isinstance(rejected, tuple): + processed = True + for pk in rejected: + if pk != self.pushkey: + # for sanity, we only remove the pushkey if it + # was the one we actually sent... + logger.warn( + ("Ignoring rejected pushkey %s because we" + " didn't send it"), pk + ) + else: + logger.info( + "Pushkey %s was rejected: removing", + pk + ) + yield self.hs.get_pusherpool().remove_pusher( + self.app_id, pk, self.user_id + ) + else: + if have_updated_badge: + yield self.update_badge() + processed = True + + if not self.alive: + return + + if processed: + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + yield self.store.update_pusher_last_token_and_success( self.app_id, self.pushkey, self.user_id, - self.failing_since + self.last_token, + self.clock.time_msec() ) + if self.failing_since: + self.failing_since = None + yield self.store.update_pusher_failing_since( + self.app_id, + self.pushkey, + self.user_id, + self.failing_since) else: - logger.warn("Failed to dispatch push for user %s " - "(failing for %dms)." - "Trying again in %dms", - self.user_id, - self.clock.time_msec() - self.failing_since, - self.backoff_delay) - yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *= 2 - if self.backoff_delay > Pusher.MAX_BACKOFF: - self.backoff_delay = Pusher.MAX_BACKOFF + if not self.failing_since: + self.failing_since = self.clock.time_msec() + yield self.store.update_pusher_failing_since( + self.app_id, + self.pushkey, + self.user_id, + self.failing_since + ) + + if (self.failing_since and + self.failing_since < + self.clock.time_msec() - Pusher.GIVE_UP_AFTER): + # we really only give up so that if the URL gets + # fixed, we don't suddenly deliver a load + # of old notifications. + logger.warn("Giving up on a notification to user %s, " + "pushkey %s", + self.user_id, self.pushkey) + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + yield self.store.update_pusher_last_token( + self.app_id, + self.pushkey, + self.user_id, + self.last_token + ) + + self.failing_since = None + yield self.store.update_pusher_failing_since( + self.app_id, + self.pushkey, + self.user_id, + self.failing_since + ) + else: + logger.warn("Failed to dispatch push for user %s " + "(failing for %dms)." + "Trying again in %dms", + self.user_id, + self.clock.time_msec() - self.failing_since, + self.backoff_delay) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *= 2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF def stop(self): self.alive = False From 97294ef2fd5737a58ff0a5d551a17e77dcf1baaf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 10:10:20 +0000 Subject: [PATCH 268/294] Create new context when measuring --- synapse/util/metrics.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index ca4800721..f70d85512 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -48,26 +48,29 @@ block_db_txn_duration = metrics.register_distribution( class Measure(object): - __slots__ = ["clock", "name", "start_context", "start"] + __slots__ = ["clock", "name", "start_context", "start", "new_context"] def __init__(self, clock, name): self.clock = clock self.name = name self.start_context = None self.start = None + self.new_context = LoggingContext(self.name) def __enter__(self): self.start = self.clock.time_msec() self.start_context = LoggingContext.current_context() + self.new_context.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): + self.new_context.__exit__(exc_type, exc_val, exc_tb) if exc_type is not None: return duration = self.clock.time_msec() - self.start block_timer.inc_by(duration, self.name) - context = LoggingContext.current_context() + context = self.new_context if context != self.start_context: logger.warn( From 241b71852e549de8a3042f8a7c22ba56af482657 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 10:16:50 +0000 Subject: [PATCH 269/294] Fix bug in util.metrics.Measure --- synapse/util/metrics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index f70d85512..7c6899aa1 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -63,6 +63,8 @@ class Measure(object): self.new_context.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): + current_context = LoggingContext.current_context() + self.new_context.__exit__(exc_type, exc_val, exc_tb) if exc_type is not None: return @@ -72,7 +74,7 @@ class Measure(object): context = self.new_context - if context != self.start_context: + if context != current_context: logger.warn( "Context have unexpectedly changed from '%s' to '%s'. (%r)", context, self.start_context, self.name From 3e2fcd67b29784f01ed44cb53f152ebd2d98ed6b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 10:50:31 +0000 Subject: [PATCH 270/294] Don't bother copying records on parent context --- synapse/util/logcontext.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index b22a36336..301b30d58 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -116,8 +116,6 @@ class LoggingContext(object): def __enter__(self): """Enters this logging context into thread local storage""" - if self.parent_context is not None: - raise Exception("Attempt to enter logging context multiple times") self.parent_context = self.set_current_context(self) self.alive = True return self @@ -141,14 +139,8 @@ class LoggingContext(object): self.parent_context = None self.alive = False - def __getattr__(self, name): - """Delegate member lookup to parent context""" - return getattr(self.parent_context, name) - def copy_to(self, record): """Copy fields from this context and its parents to the record""" - if self.parent_context is not None: - self.parent_context.copy_to(record) for key, value in self.__dict__.items(): setattr(record, key, value) From 9daa4e2a8547b954171d88a7b141e863cef1fdf9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 11:06:19 +0000 Subject: [PATCH 271/294] Don't create new logging context --- synapse/util/metrics.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 7c6899aa1..57ca3b475 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -48,33 +48,34 @@ block_db_txn_duration = metrics.register_distribution( class Measure(object): - __slots__ = ["clock", "name", "start_context", "start", "new_context"] + __slots__ = [ + "clock", "name", "start_context", "start", "new_context", "ru_utime", + "ru_stime", "db_txn_count", "db_txn_duration" + ] def __init__(self, clock, name): self.clock = clock self.name = name self.start_context = None self.start = None - self.new_context = LoggingContext(self.name) def __enter__(self): self.start = self.clock.time_msec() self.start_context = LoggingContext.current_context() - self.new_context.__enter__() + self.ru_utime, self.ru_stime = self.start_context.get_resource_usage() + self.db_txn_count = self.start_context.db_txn_count + self.db_txn_duration = self.start_context.db_txn_duration def __exit__(self, exc_type, exc_val, exc_tb): - current_context = LoggingContext.current_context() - - self.new_context.__exit__(exc_type, exc_val, exc_tb) if exc_type is not None: return duration = self.clock.time_msec() - self.start block_timer.inc_by(duration, self.name) - context = self.new_context + context = LoggingContext.current_context() - if context != current_context: + if context != self.start_context: logger.warn( "Context have unexpectedly changed from '%s' to '%s'. (%r)", context, self.start_context, self.name @@ -87,7 +88,9 @@ class Measure(object): ru_utime, ru_stime = context.get_resource_usage() - block_ru_utime.inc_by(ru_utime, self.name) - block_ru_stime.inc_by(ru_stime, self.name) - block_db_txn_count.inc_by(context.db_txn_count, self.name) - block_db_txn_duration.inc_by(context.db_txn_duration, self.name) + block_ru_utime.inc_by(ru_utime - self.ru_utime, self.name) + block_ru_stime.inc_by(ru_stime - self.ru_stime, self.name) + block_db_txn_count.inc_by(context.db_txn_count - self.db_txn_count, self.name) + block_db_txn_duration.inc_by( + context.db_txn_duration - self.db_txn_duration, self.name + ) From 31a2b892d8935dcb49cbc3c71a0ccf8aa34eeb97 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 11:25:09 +0000 Subject: [PATCH 272/294] Revert to putting it around the entire block --- synapse/push/__init__.py | 244 +++++++++++++++++++-------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b9522a805..8da2d8716 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -128,7 +128,8 @@ class Pusher(object): try: if wait > 0: yield synapse.util.async.sleep(wait) - yield self.get_and_dispatch() + with Measure(self.clock, "push"): + yield self.get_and_dispatch() wait = 0 except: if wait == 0: @@ -150,27 +151,115 @@ class Pusher(object): only_keys=("room", "receipt",), ) - with Measure(self.clock, "push"): - # limiting to 1 may get 1 event plus 1 presence event, so - # pick out the actual event - single_event = None - read_receipt = None - for c in chunk['chunk']: - if 'event_id' in c: # Hmmm... - single_event = c - elif c['type'] == 'm.receipt': - read_receipt = c + # limiting to 1 may get 1 event plus 1 presence event, so + # pick out the actual event + single_event = None + read_receipt = None + for c in chunk['chunk']: + if 'event_id' in c: # Hmmm... + single_event = c + elif c['type'] == 'm.receipt': + read_receipt = c - have_updated_badge = False - if read_receipt: - for receipt_part in read_receipt['content'].values(): - if 'm.read' in receipt_part: - if self.user_id in receipt_part['m.read'].keys(): - have_updated_badge = True + have_updated_badge = False + if read_receipt: + for receipt_part in read_receipt['content'].values(): + if 'm.read' in receipt_part: + if self.user_id in receipt_part['m.read'].keys(): + have_updated_badge = True - if not single_event: - if have_updated_badge: - yield self.update_badge() + if not single_event: + if have_updated_badge: + yield self.update_badge() + self.last_token = chunk['end'] + yield self.store.update_pusher_last_token( + self.app_id, + self.pushkey, + self.user_id, + self.last_token + ) + return + + if not self.alive: + return + + processed = False + + rule_evaluator = yield \ + push_rule_evaluator.evaluator_for_user_id_and_profile_tag( + self.user_id, self.profile_tag, single_event['room_id'], self.store + ) + + actions = yield rule_evaluator.actions_for_event(single_event) + tweaks = rule_evaluator.tweaks_for_actions(actions) + + if 'notify' in actions: + self.badge = yield self._get_badge_count() + rejected = yield self.dispatch_push(single_event, tweaks, self.badge) + self.has_unread = True + if isinstance(rejected, list) or isinstance(rejected, tuple): + processed = True + for pk in rejected: + if pk != self.pushkey: + # for sanity, we only remove the pushkey if it + # was the one we actually sent... + logger.warn( + ("Ignoring rejected pushkey %s because we" + " didn't send it"), pk + ) + else: + logger.info( + "Pushkey %s was rejected: removing", + pk + ) + yield self.hs.get_pusherpool().remove_pusher( + self.app_id, pk, self.user_id + ) + else: + if have_updated_badge: + yield self.update_badge() + processed = True + + if not self.alive: + return + + if processed: + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + yield self.store.update_pusher_last_token_and_success( + self.app_id, + self.pushkey, + self.user_id, + self.last_token, + self.clock.time_msec() + ) + if self.failing_since: + self.failing_since = None + yield self.store.update_pusher_failing_since( + self.app_id, + self.pushkey, + self.user_id, + self.failing_since) + else: + if not self.failing_since: + self.failing_since = self.clock.time_msec() + yield self.store.update_pusher_failing_since( + self.app_id, + self.pushkey, + self.user_id, + self.failing_since + ) + + if (self.failing_since and + self.failing_since < + self.clock.time_msec() - Pusher.GIVE_UP_AFTER): + # we really only give up so that if the URL gets + # fixed, we don't suddenly deliver a load + # of old notifications. + logger.warn("Giving up on a notification to user %s, " + "pushkey %s", + self.user_id, self.pushkey) + self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] yield self.store.update_pusher_last_token( self.app_id, @@ -178,114 +267,25 @@ class Pusher(object): self.user_id, self.last_token ) - return - if not self.alive: - return - - processed = False - - rule_evaluator = yield \ - push_rule_evaluator.evaluator_for_user_id_and_profile_tag( - self.user_id, self.profile_tag, single_event['room_id'], self.store - ) - - actions = yield rule_evaluator.actions_for_event(single_event) - tweaks = rule_evaluator.tweaks_for_actions(actions) - - if 'notify' in actions: - self.badge = yield self._get_badge_count() - rejected = yield self.dispatch_push(single_event, tweaks, self.badge) - self.has_unread = True - if isinstance(rejected, list) or isinstance(rejected, tuple): - processed = True - for pk in rejected: - if pk != self.pushkey: - # for sanity, we only remove the pushkey if it - # was the one we actually sent... - logger.warn( - ("Ignoring rejected pushkey %s because we" - " didn't send it"), pk - ) - else: - logger.info( - "Pushkey %s was rejected: removing", - pk - ) - yield self.hs.get_pusherpool().remove_pusher( - self.app_id, pk, self.user_id - ) - else: - if have_updated_badge: - yield self.update_badge() - processed = True - - if not self.alive: - return - - if processed: - self.backoff_delay = Pusher.INITIAL_BACKOFF - self.last_token = chunk['end'] - yield self.store.update_pusher_last_token_and_success( + self.failing_since = None + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_id, - self.last_token, - self.clock.time_msec() + self.failing_since ) - if self.failing_since: - self.failing_since = None - yield self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_id, - self.failing_since) else: - if not self.failing_since: - self.failing_since = self.clock.time_msec() - yield self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_id, - self.failing_since - ) - - if (self.failing_since and - self.failing_since < - self.clock.time_msec() - Pusher.GIVE_UP_AFTER): - # we really only give up so that if the URL gets - # fixed, we don't suddenly deliver a load - # of old notifications. - logger.warn("Giving up on a notification to user %s, " - "pushkey %s", - self.user_id, self.pushkey) - self.backoff_delay = Pusher.INITIAL_BACKOFF - self.last_token = chunk['end'] - yield self.store.update_pusher_last_token( - self.app_id, - self.pushkey, - self.user_id, - self.last_token - ) - - self.failing_since = None - yield self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_id, - self.failing_since - ) - else: - logger.warn("Failed to dispatch push for user %s " - "(failing for %dms)." - "Trying again in %dms", - self.user_id, - self.clock.time_msec() - self.failing_since, - self.backoff_delay) - yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *= 2 - if self.backoff_delay > Pusher.MAX_BACKOFF: - self.backoff_delay = Pusher.MAX_BACKOFF + logger.warn("Failed to dispatch push for user %s " + "(failing for %dms)." + "Trying again in %dms", + self.user_id, + self.clock.time_msec() - self.failing_since, + self.backoff_delay) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *= 2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF def stop(self): self.alive = False From 6c558ee8bc1376bc678dd20c9fb462c76bc913bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 11:31:04 +0000 Subject: [PATCH 273/294] Measure some /sync related things --- synapse/handlers/sync.py | 283 +++++++++++++++++++------------------ synapse/handlers/typing.py | 23 +-- 2 files changed, 159 insertions(+), 147 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3f1cda5b0..3e5f27046 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -19,6 +19,7 @@ from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes from synapse.util import unwrapFirstError from synapse.util.logcontext import LoggingContext, PreserveLoggingContext +from synapse.util.metrics import Measure from twisted.internet import defer @@ -368,50 +369,51 @@ class SyncHandler(BaseHandler): typing events for that room. """ - typing_key = since_token.typing_key if since_token else "0" + with Measure(self.clock, "ephemeral_by_room"): + typing_key = since_token.typing_key if since_token else "0" - rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string()) - room_ids = [room.room_id for room in rooms] + rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string()) + room_ids = [room.room_id for room in rooms] - typing_source = self.event_sources.sources["typing"] - typing, typing_key = yield typing_source.get_new_events( - user=sync_config.user, - from_key=typing_key, - limit=sync_config.filter_collection.ephemeral_limit(), - room_ids=room_ids, - is_guest=sync_config.is_guest, - ) - now_token = now_token.copy_and_replace("typing_key", typing_key) + typing_source = self.event_sources.sources["typing"] + typing, typing_key = yield typing_source.get_new_events( + user=sync_config.user, + from_key=typing_key, + limit=sync_config.filter_collection.ephemeral_limit(), + room_ids=room_ids, + is_guest=sync_config.is_guest, + ) + now_token = now_token.copy_and_replace("typing_key", typing_key) - ephemeral_by_room = {} + ephemeral_by_room = {} - for event in typing: - # we want to exclude the room_id from the event, but modifying the - # result returned by the event source is poor form (it might cache - # the object) - room_id = event["room_id"] - event_copy = {k: v for (k, v) in event.iteritems() - if k != "room_id"} - ephemeral_by_room.setdefault(room_id, []).append(event_copy) + for event in typing: + # we want to exclude the room_id from the event, but modifying the + # result returned by the event source is poor form (it might cache + # the object) + room_id = event["room_id"] + event_copy = {k: v for (k, v) in event.iteritems() + if k != "room_id"} + ephemeral_by_room.setdefault(room_id, []).append(event_copy) - receipt_key = since_token.receipt_key if since_token else "0" + receipt_key = since_token.receipt_key if since_token else "0" - receipt_source = self.event_sources.sources["receipt"] - receipts, receipt_key = yield receipt_source.get_new_events( - user=sync_config.user, - from_key=receipt_key, - limit=sync_config.filter_collection.ephemeral_limit(), - room_ids=room_ids, - is_guest=sync_config.is_guest, - ) - now_token = now_token.copy_and_replace("receipt_key", receipt_key) + receipt_source = self.event_sources.sources["receipt"] + receipts, receipt_key = yield receipt_source.get_new_events( + user=sync_config.user, + from_key=receipt_key, + limit=sync_config.filter_collection.ephemeral_limit(), + room_ids=room_ids, + is_guest=sync_config.is_guest, + ) + now_token = now_token.copy_and_replace("receipt_key", receipt_key) - for event in receipts: - room_id = event["room_id"] - # exclude room id, as above - event_copy = {k: v for (k, v) in event.iteritems() - if k != "room_id"} - ephemeral_by_room.setdefault(room_id, []).append(event_copy) + for event in receipts: + room_id = event["room_id"] + # exclude room id, as above + event_copy = {k: v for (k, v) in event.iteritems() + if k != "room_id"} + ephemeral_by_room.setdefault(room_id, []).append(event_copy) defer.returnValue((now_token, ephemeral_by_room)) @@ -619,58 +621,61 @@ class SyncHandler(BaseHandler): """ :returns a Deferred TimelineBatch """ - filtering_factor = 2 - timeline_limit = sync_config.filter_collection.timeline_limit() - load_limit = max(timeline_limit * filtering_factor, 10) - max_repeat = 5 # Only try a few times per room, otherwise - room_key = now_token.room_key - end_key = room_key + with Measure(self.clock, "load_filtered_recents"): + filtering_factor = 2 + timeline_limit = sync_config.filter_collection.timeline_limit() + load_limit = max(timeline_limit * filtering_factor, 10) + max_repeat = 5 # Only try a few times per room, otherwise + room_key = now_token.room_key + end_key = room_key - limited = recents is None or newly_joined_room or timeline_limit < len(recents) + limited = recents is None or newly_joined_room or timeline_limit < len(recents) - if recents is not None: - recents = sync_config.filter_collection.filter_room_timeline(recents) - recents = yield self._filter_events_for_client( - sync_config.user.to_string(), - recents, - is_peeking=sync_config.is_guest, + if recents is not None: + recents = sync_config.filter_collection.filter_room_timeline(recents) + recents = yield self._filter_events_for_client( + sync_config.user.to_string(), + recents, + is_peeking=sync_config.is_guest, + ) + else: + recents = [] + + since_key = None + if since_token and not newly_joined_room: + since_key = since_token.room_key + + while limited and len(recents) < timeline_limit and max_repeat: + events, end_key = yield self.store.get_room_events_stream_for_room( + room_id, + limit=load_limit + 1, + from_key=since_key, + to_key=end_key, + ) + loaded_recents = sync_config.filter_collection.filter_room_timeline( + events + ) + loaded_recents = yield self._filter_events_for_client( + sync_config.user.to_string(), + loaded_recents, + is_peeking=sync_config.is_guest, + ) + loaded_recents.extend(recents) + recents = loaded_recents + + if len(events) <= load_limit: + limited = False + break + max_repeat -= 1 + + if len(recents) > timeline_limit: + limited = True + recents = recents[-timeline_limit:] + room_key = recents[0].internal_metadata.before + + prev_batch_token = now_token.copy_and_replace( + "room_key", room_key ) - else: - recents = [] - - since_key = None - if since_token and not newly_joined_room: - since_key = since_token.room_key - - while limited and len(recents) < timeline_limit and max_repeat: - events, end_key = yield self.store.get_room_events_stream_for_room( - room_id, - limit=load_limit + 1, - from_key=since_key, - to_key=end_key, - ) - loaded_recents = sync_config.filter_collection.filter_room_timeline(events) - loaded_recents = yield self._filter_events_for_client( - sync_config.user.to_string(), - loaded_recents, - is_peeking=sync_config.is_guest, - ) - loaded_recents.extend(recents) - recents = loaded_recents - - if len(events) <= load_limit: - limited = False - break - max_repeat -= 1 - - if len(recents) > timeline_limit: - limited = True - recents = recents[-timeline_limit:] - room_key = recents[0].internal_metadata.before - - prev_batch_token = now_token.copy_and_replace( - "room_key", room_key - ) defer.returnValue(TimelineBatch( events=recents, @@ -831,50 +836,53 @@ class SyncHandler(BaseHandler): # updates even if they occured logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - if full_state: - if batch: - state = yield self.store.get_state_for_event(batch.events[0].event_id) - else: - state = yield self.get_state_at( - room_id, stream_position=now_token + with Measure(self.clock, "compute_state_delta"): + if full_state: + if batch: + state = yield self.store.get_state_for_event( + batch.events[0].event_id + ) + else: + state = yield self.get_state_at( + room_id, stream_position=now_token + ) + + timeline_state = { + (event.type, event.state_key): event + for event in batch.events if event.is_state() + } + + state = _calculate_state( + timeline_contains=timeline_state, + timeline_start=state, + previous={}, + ) + elif batch.limited: + state_at_previous_sync = yield self.get_state_at( + room_id, stream_position=since_token ) - timeline_state = { - (event.type, event.state_key): event - for event in batch.events if event.is_state() - } + state_at_timeline_start = yield self.store.get_state_for_event( + batch.events[0].event_id + ) - state = _calculate_state( - timeline_contains=timeline_state, - timeline_start=state, - previous={}, - ) - elif batch.limited: - state_at_previous_sync = yield self.get_state_at( - room_id, stream_position=since_token - ) + timeline_state = { + (event.type, event.state_key): event + for event in batch.events if event.is_state() + } - state_at_timeline_start = yield self.store.get_state_for_event( - batch.events[0].event_id - ) + state = _calculate_state( + timeline_contains=timeline_state, + timeline_start=state_at_timeline_start, + previous=state_at_previous_sync, + ) + else: + state = {} - timeline_state = { - (event.type, event.state_key): event - for event in batch.events if event.is_state() - } - - state = _calculate_state( - timeline_contains=timeline_state, - timeline_start=state_at_timeline_start, - previous=state_at_previous_sync, - ) - else: - state = {} - - defer.returnValue({ - (e.type, e.state_key): e - for e in sync_config.filter_collection.filter_room_state(state.values()) - }) + defer.returnValue({ + (e.type, e.state_key): e + for e in sync_config.filter_collection.filter_room_state(state.values()) + }) def check_joined_room(self, sync_config, state_delta): """ @@ -896,20 +904,21 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): - last_unread_event_id = self.last_read_event_id_for_room_and_user( - room_id, sync_config.user.to_string(), ephemeral_by_room - ) - - notifs = [] - if last_unread_event_id: - notifs = yield self.store.get_unread_event_push_actions_by_room_for_user( - room_id, sync_config.user.to_string(), last_unread_event_id + with Measure(self.clock, "unread_notifs_for_room_id"): + last_unread_event_id = self.last_read_event_id_for_room_and_user( + room_id, sync_config.user.to_string(), ephemeral_by_room ) - defer.returnValue(notifs) - # There is no new information in this period, so your notification - # count is whatever it was last time. - defer.returnValue(None) + notifs = [] + if last_unread_event_id: + notifs = yield self.store.get_unread_event_push_actions_by_room_for_user( + room_id, sync_config.user.to_string(), last_unread_event_id + ) + defer.returnValue(notifs) + + # There is no new information in this period, so your notification + # count is whatever it was last time. + defer.returnValue(None) def _action_has_highlight(actions): diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 43bf60091..b16d0017d 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import SynapseError, AuthError from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.metrics import Measure from synapse.types import UserID import logging @@ -222,6 +223,7 @@ class TypingNotificationHandler(BaseHandler): class TypingNotificationEventSource(object): def __init__(self, hs): self.hs = hs + self.clock = hs.get_clock() self._handler = None self._room_member_handler = None @@ -247,19 +249,20 @@ class TypingNotificationEventSource(object): } def get_new_events(self, from_key, room_ids, **kwargs): - from_key = int(from_key) - handler = self.handler() + with Measure(self.clock, "typing.get_new_events"): + from_key = int(from_key) + handler = self.handler() - events = [] - for room_id in room_ids: - if room_id not in handler._room_serials: - continue - if handler._room_serials[room_id] <= from_key: - continue + events = [] + for room_id in room_ids: + if room_id not in handler._room_serials: + continue + if handler._room_serials[room_id] <= from_key: + continue - events.append(self._make_event_for(room_id)) + events.append(self._make_event_for(room_id)) - return events, handler._latest_room_serial + return events, handler._latest_room_serial def get_current_key(self): return self.handler()._latest_room_serial From ebaa999f92677d7e1e6b633ea1829f80a69e83c7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 9 Feb 2016 12:46:42 +0000 Subject: [PATCH 274/294] Revert "Reject additional path segments" This reverts commit 1d19a5ec0fff73af9cee8c21118020b31be47379. iOS Console is apparently relying on these paths. --- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/presence.py | 4 ++-- synapse/rest/client/v1/profile.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 5ec52707e..e2f5eb7b2 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class WhoisRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/admin/whois/(?P[^/]*)$") + PATTERNS = client_path_patterns("/admin/whois/(?P[^/]*)") @defer.inlineCallbacks def on_GET(self, request, user_id): diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 9410ac527..a6f8754e3 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) class PresenceStatusRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/presence/(?P[^/]*)/status$") + PATTERNS = client_path_patterns("/presence/(?P[^/]*)/status") @defer.inlineCallbacks def on_GET(self, request, user_id): @@ -73,7 +73,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): class PresenceListRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/presence/list/(?P[^/]*)$") + PATTERNS = client_path_patterns("/presence/list/(?P[^/]*)") @defer.inlineCallbacks def on_GET(self, request, user_id): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index aeda7bfa3..3c5a21292 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -23,7 +23,7 @@ import simplejson as json class ProfileDisplaynameRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/profile/(?P[^/]*)/displayname$") + PATTERNS = client_path_patterns("/profile/(?P[^/]*)/displayname") @defer.inlineCallbacks def on_GET(self, request, user_id): @@ -60,7 +60,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): class ProfileAvatarURLRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/profile/(?P[^/]*)/avatar_url$") + PATTERNS = client_path_patterns("/profile/(?P[^/]*)/avatar_url") @defer.inlineCallbacks def on_GET(self, request, user_id): @@ -97,7 +97,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): class ProfileRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/profile/(?P[^/]*)$") + PATTERNS = client_path_patterns("/profile/(?P[^/]*)") @defer.inlineCallbacks def on_GET(self, request, user_id): From 5b75b637b838479528dcc87955cdd7a2a4210b58 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 9 Feb 2016 13:44:04 +0000 Subject: [PATCH 275/294] Remove pyc files before running tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index bd313a4f3..6f01599af 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = setenv = PYTHONDONTWRITEBYTECODE = no_byte_code commands = - /bin/bash -c "coverage run {env:COVERAGE_OPTS:} --source={toxinidir}/synapse \ + /bin/bash -c "find {toxinidir} -name '*.pyc' -delete ; coverage run {env:COVERAGE_OPTS:} --source={toxinidir}/synapse \ {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}" {env:DUMP_COVERAGE_COMMAND:coverage report -m} From 82631c5f948cfb3cb133379e0e5ad10811d706a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 13:50:29 +0000 Subject: [PATCH 276/294] Fix unit tests --- synapse/handlers/sync.py | 5 ++++- synapse/util/metrics.py | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3e5f27046..446f8bbe9 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -629,7 +629,10 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key end_key = room_key - limited = recents is None or newly_joined_room or timeline_limit < len(recents) + if recents is None or newly_joined_room or timeline_limit < len(recents): + limited = True + else: + limited = False if recents is not None: recents = sync_config.filter_collection.filter_room_timeline(recents) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 57ca3b475..c51b64112 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -62,12 +62,13 @@ class Measure(object): def __enter__(self): self.start = self.clock.time_msec() self.start_context = LoggingContext.current_context() - self.ru_utime, self.ru_stime = self.start_context.get_resource_usage() - self.db_txn_count = self.start_context.db_txn_count - self.db_txn_duration = self.start_context.db_txn_duration + if self.start_context: + self.ru_utime, self.ru_stime = self.start_context.get_resource_usage() + self.db_txn_count = self.start_context.db_txn_count + self.db_txn_duration = self.start_context.db_txn_duration def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type is not None: + if exc_type is not None or not self.start_context: return duration = self.clock.time_msec() - self.start From eff12e838ce10588ca8103c9131dcfe2f2e7950e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 13:55:59 +0000 Subject: [PATCH 277/294] Don't load all ephemeral state for a room on every sync --- synapse/handlers/sync.py | 20 ++++++-------------- synapse/storage/receipts.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 446f8bbe9..6a5868f87 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -319,7 +319,6 @@ class SyncHandler(BaseHandler): ephemeral_by_room=ephemeral_by_room, tags_by_room=tags_by_room, account_data_by_room=account_data_by_room, - all_ephemeral_by_room=ephemeral_by_room, batch=batch, full_state=True, ) @@ -453,13 +452,6 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("presence_key", presence_key) - # We now fetch all ephemeral events for this room in order to get - # this users current read receipt. This could almost certainly be - # optimised. - _, all_ephemeral_by_room = yield self.ephemeral_by_room( - sync_config, now_token - ) - now_token, ephemeral_by_room = yield self.ephemeral_by_room( sync_config, now_token, since_token ) @@ -591,7 +583,6 @@ class SyncHandler(BaseHandler): ephemeral_by_room=ephemeral_by_room, tags_by_room=tags_by_room, account_data_by_room=account_data_by_room, - all_ephemeral_by_room=all_ephemeral_by_room, batch=batch, full_state=full_state, ) @@ -691,7 +682,6 @@ class SyncHandler(BaseHandler): since_token, now_token, ephemeral_by_room, tags_by_room, account_data_by_room, - all_ephemeral_by_room, batch, full_state=False): state = yield self.compute_state_delta( room_id, batch, sync_config, since_token, now_token, @@ -722,7 +712,7 @@ class SyncHandler(BaseHandler): if room_sync: notifs = yield self.unread_notifs_for_room_id( - room_id, sync_config, all_ephemeral_by_room + room_id, sync_config ) if notifs is not None: @@ -906,10 +896,12 @@ class SyncHandler(BaseHandler): return False @defer.inlineCallbacks - def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): + def unread_notifs_for_room_id(self, room_id, sync_config): with Measure(self.clock, "unread_notifs_for_room_id"): - last_unread_event_id = self.last_read_event_id_for_room_and_user( - room_id, sync_config.user.to_string(), ephemeral_by_room + last_unread_event_id = yield self.store.get_last_receipt_event_id_for_user( + user_id=sync_config.user.to_string(), + room_id=room_id, + receipt_type="m.read" ) notifs = [] diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 8068c7374..1aff9f070 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -46,6 +46,20 @@ class ReceiptsStore(SQLBaseStore): desc="get_receipts_for_room", ) + @cached(num_args=3) + def get_last_receipt_event_id_for_user(self, user_id, room_id, receipt_type): + return self._simple_select_one_onecol( + table="receipts_linearized", + keyvalues={ + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id + }, + retcol="event_id", + desc="get_own_receipt_for_user", + allow_none=True, + ) + @cachedInlineCallbacks(num_args=2) def get_receipts_for_user(self, user_id, receipt_type): def f(txn): From 70a8608749e0c1ec7a993a9effc424303af24738 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 14:27:29 +0000 Subject: [PATCH 278/294] Invalidate get_last_receipt_event_id_for_user cache --- synapse/storage/receipts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 1aff9f070..4202a6b3d 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -240,6 +240,11 @@ class ReceiptsStore(SQLBaseStore): room_id, stream_id ) + txn.call_after( + self.get_last_receipt_event_id_for_user.invalidate, + (user_id, room_id, receipt_type) + ) + # We don't want to clobber receipts for more recent events, so we # have to compare orderings of existing receipts sql = ( From feb294d552ac58b867142118c3d19f0b11b2f384 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 14:32:17 +0000 Subject: [PATCH 279/294] Remove dead code --- synapse/handlers/sync.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6a5868f87..ddeed2796 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -179,18 +179,6 @@ class SyncHandler(BaseHandler): else: return self.incremental_sync_with_gap(sync_config, since_token) - def last_read_event_id_for_room_and_user(self, room_id, user_id, ephemeral_by_room): - if room_id not in ephemeral_by_room: - return None - for e in ephemeral_by_room[room_id]: - if e['type'] != 'm.receipt': - continue - for receipt_event_id, val in e['content'].items(): - if 'm.read' in val: - if user_id in val['m.read']: - return receipt_event_id - return None - @defer.inlineCallbacks def full_state_sync(self, sync_config, timeline_since_token): """Get a sync for a client which is starting without any state. From 78d6c1b5bec800671d3ff66acecb2f8bbdf41aa1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 14:44:12 +0000 Subject: [PATCH 280/294] Change a log from debug to info --- synapse/storage/prepare_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index d782b8e25..850736c85 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -211,7 +211,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, logger.debug("applied_delta_files: %s", applied_delta_files) for v in range(start_ver, SCHEMA_VERSION + 1): - logger.debug("Upgrading schema to v%d", v) + logger.info("Upgrading schema to v%d", v) delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) From 13ba8d878ce30dbc16123886a78a0905fc9ad4a5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 14:55:21 +0000 Subject: [PATCH 281/294] Fix test --- tests/config/test_load.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/config/test_load.py b/tests/config/test_load.py index 7f4127971..528e87853 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -54,10 +54,11 @@ class ConfigLoadingTestCase(unittest.TestCase): "was: %r" % (config.macaroon_secret_key,) ) - def test_load_fails_if_macaroon_secret_key_missing(self): + def test_load_suceeds_if_macaroon_secret_key_missing(self): self.generate_config_and_remove_lines_containing("macaroon") - with self.assertRaises(Exception): - HomeServerConfig.load_config("", ["-c", self.file]) + config1 = HomeServerConfig.load_config("", ["-c", self.file]) + config2 = HomeServerConfig.load_config("", ["-c", self.file]) + self.assertEqual(config1.macaroon_secret_key, config2.macaroon_secret_key) def generate_config(self): HomeServerConfig.load_config("", [ From e664e9737ca8ff04043f747a9375ff50440352c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 14:57:43 +0000 Subject: [PATCH 282/294] Fix test --- tests/util/test_log_context.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/util/test_log_context.py b/tests/util/test_log_context.py index efa0f28ba..65a330a0e 100644 --- a/tests/util/test_log_context.py +++ b/tests/util/test_log_context.py @@ -5,6 +5,7 @@ from .. import unittest from synapse.util.async import sleep from synapse.util.logcontext import LoggingContext + class LoggingContextTestCase(unittest.TestCase): def _check_test_key(self, value): @@ -17,15 +18,6 @@ class LoggingContextTestCase(unittest.TestCase): context_one.test_key = "test" self._check_test_key("test") - def test_chaining(self): - with LoggingContext() as context_one: - context_one.test_key = "one" - with LoggingContext() as context_two: - self._check_test_key("one") - context_two.test_key = "two" - self._check_test_key("two") - self._check_test_key("one") - @defer.inlineCallbacks def test_sleep(self): @defer.inlineCallbacks From f28cc4518368955f692a1262a688e907cb22d226 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 16:01:40 +0000 Subject: [PATCH 283/294] Pass in current state to push action handler --- synapse/handlers/_base.py | 31 ++++++++++-------------- synapse/push/action_generator.py | 6 +++-- synapse/push/bulk_push_rule_evaluator.py | 10 +++----- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index fa83d3e46..d3f722b22 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -53,25 +53,10 @@ class BaseHandler(object): self.event_builder_factory = hs.get_event_builder_factory() @defer.inlineCallbacks - def _filter_events_for_clients(self, user_tuples, events): + def _filter_events_for_clients(self, user_tuples, events, event_id_to_state): """ Returns dict of user_id -> list of events that user is allowed to see. """ - # If there is only one user, just get the state for that one user, - # otherwise just get all the state. - if len(user_tuples) == 1: - types = ( - (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.Member, user_tuples[0][0]), - ) - else: - types = None - - event_id_to_state = yield self.store.get_state_for_events( - frozenset(e.event_id for e in events), - types=types - ) - forgotten = yield defer.gatherResults([ self.store.who_forgot_in_room( room_id, @@ -135,7 +120,17 @@ class BaseHandler(object): @defer.inlineCallbacks def _filter_events_for_client(self, user_id, events, is_peeking=False): # Assumes that user has at some point joined the room if not is_guest. - res = yield self._filter_events_for_clients([(user_id, is_peeking)], events) + types = ( + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, user_id), + ) + event_id_to_state = yield self.store.get_state_for_events( + frozenset(e.event_id for e in events), + types=types + ) + res = yield self._filter_events_for_clients( + [(user_id, is_peeking)], events, event_id_to_state + ) defer.returnValue(res.get(user_id, [])) def ratelimit(self, user_id): @@ -275,7 +270,7 @@ class BaseHandler(object): action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( - event, self + event, self, context.current_state ) destinations = set() diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 1d2e558f9..d8f8256a1 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -36,7 +36,7 @@ class ActionGenerator: # tag (ie. we just need all the users). @defer.inlineCallbacks - def handle_push_actions_for_event(self, event, handler): + def handle_push_actions_for_event(self, event, handler, current_state): if event.type == EventTypes.Redaction and event.redacts is not None: yield self.store.remove_push_actions_for_event_id( event.room_id, event.redacts @@ -46,7 +46,9 @@ class ActionGenerator: event.room_id, self.hs, self.store ) - actions_by_user = yield bulk_evaluator.action_for_event_by_user(event, handler) + actions_by_user = yield bulk_evaluator.action_for_event_by_user( + event, handler, current_state + ) yield self.store.set_push_actions_for_event_and_users( event, diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 20c60422b..8ac5ceb9e 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -98,25 +98,21 @@ class BulkPushRuleEvaluator: self.store = store @defer.inlineCallbacks - def action_for_event_by_user(self, event, handler): + def action_for_event_by_user(self, event, handler, current_state): actions_by_user = {} users_dict = yield self.store.are_guests(self.rules_by_user.keys()) filtered_by_user = yield handler._filter_events_for_clients( - users_dict.items(), [event] + users_dict.items(), [event], {event.event_id: current_state} ) evaluator = PushRuleEvaluatorForEvent(event, len(self.users_in_room)) condition_cache = {} - member_state = yield self.store.get_state_for_event( - event.event_id, - ) - display_names = {} - for ev in member_state.values(): + for ev in current_state.values(): nm = ev.content.get("displayname", None) if nm and ev.type == EventTypes.Member: display_names[ev.state_key] = nm From 7b0d846407a593ccd204f82aaa1090b8af8df84c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 16:19:15 +0000 Subject: [PATCH 284/294] Atomically persit push actions when we persist the event --- synapse/events/snapshot.py | 1 + synapse/handlers/_base.py | 10 +++--- synapse/handlers/federation.py | 12 +++---- synapse/push/action_generator.py | 20 +++--------- synapse/storage/event_push_actions.py | 45 ++++++++++----------------- synapse/storage/events.py | 26 ++++++++++------ 6 files changed, 49 insertions(+), 65 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index f51200d18..8a475417a 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -20,3 +20,4 @@ class EventContext(object): self.current_state = current_state self.state_group = None self.rejected = False + self.push_actions = [] diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d3f722b22..064e8723c 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -264,13 +264,13 @@ class BaseHandler(object): "You don't have permission to redact events" ) - (event_stream_id, max_stream_id) = yield self.store.persist_event( - event, context=context - ) - action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( - event, self, context.current_state + event, context, self + ) + + (event_stream_id, max_stream_id) = yield self.store.persist_event( + event, context=context ) destinations = set() diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b78b0502d..da55d4354 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -236,12 +236,6 @@ class FederationHandler(BaseHandler): user = UserID.from_string(event.state_key) yield user_joined_room(self.distributor, user, event.room_id) - if not backfilled and not event.internal_metadata.is_outlier(): - action_generator = ActionGenerator(self.hs) - yield action_generator.handle_push_actions_for_event( - event, self - ) - @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): event_to_state = yield self.store.get_state_for_events( @@ -1073,6 +1067,12 @@ class FederationHandler(BaseHandler): auth_events=auth_events, ) + if not backfilled and not event.internal_metadata.is_outlier(): + action_generator = ActionGenerator(self.hs) + yield action_generator.handle_push_actions_for_event( + event, context, self + ) + event_stream_id, max_stream_id = yield self.store.persist_event( event, context=context, diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index d8f8256a1..e0da0868e 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -19,8 +19,6 @@ import bulk_push_rule_evaluator import logging -from synapse.api.constants import EventTypes - logger = logging.getLogger(__name__) @@ -36,23 +34,15 @@ class ActionGenerator: # tag (ie. we just need all the users). @defer.inlineCallbacks - def handle_push_actions_for_event(self, event, handler, current_state): - if event.type == EventTypes.Redaction and event.redacts is not None: - yield self.store.remove_push_actions_for_event_id( - event.room_id, event.redacts - ) - + def handle_push_actions_for_event(self, event, context, handler): bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( event.room_id, self.hs, self.store ) actions_by_user = yield bulk_evaluator.action_for_event_by_user( - event, handler, current_state + event, handler, context.current_state ) - yield self.store.set_push_actions_for_event_and_users( - event, - [ - (uid, None, actions) for uid, actions in actions_by_user.items() - ] - ) + context.push_actions = [ + (uid, None, actions) for uid, actions in actions_by_user.items() + ] diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index d0a969f50..466f07a1c 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -24,8 +24,7 @@ logger = logging.getLogger(__name__) class EventPushActionsStore(SQLBaseStore): - @defer.inlineCallbacks - def set_push_actions_for_event_and_users(self, event, tuples): + def _set_push_actions_for_event_and_users(self, txn, event, tuples): """ :param event: the event set actions for :param tuples: list of tuples of (user_id, profile_tag, actions) @@ -44,18 +43,12 @@ class EventPushActionsStore(SQLBaseStore): 'highlight': 1 if _action_has_highlight(actions) else 0, }) - def f(txn): - for uid, _, __ in tuples: - txn.call_after( - self.get_unread_event_push_actions_by_room_for_user.invalidate_many, - (event.room_id, uid) - ) - return self._simple_insert_many_txn(txn, "event_push_actions", values) - - yield self.runInteraction( - "set_actions_for_event_and_users", - f, - ) + for uid, _, __ in tuples: + txn.call_after( + self.get_unread_event_push_actions_by_room_for_user.invalidate_many, + (event.room_id, uid) + ) + self._simple_insert_many_txn(txn, "event_push_actions", values) @cachedInlineCallbacks(num_args=3, lru=True, tree=True) def get_unread_event_push_actions_by_room_for_user( @@ -107,21 +100,15 @@ class EventPushActionsStore(SQLBaseStore): ) defer.returnValue(ret) - @defer.inlineCallbacks - def remove_push_actions_for_event_id(self, room_id, event_id): - def f(txn): - # Sad that we have to blow away the cache for the whole room here - txn.call_after( - self.get_unread_event_push_actions_by_room_for_user.invalidate_many, - (room_id,) - ) - txn.execute( - "DELETE FROM event_push_actions WHERE room_id = ? AND event_id = ?", - (room_id, event_id) - ) - yield self.runInteraction( - "remove_push_actions_for_event_id", - f + def _remove_push_actions_for_event_id(self, txn, room_id, event_id): + # Sad that we have to blow away the cache for the whole room here + txn.call_after( + self.get_unread_event_push_actions_by_room_for_user.invalidate_many, + (room_id,) + ) + txn.execute( + "DELETE FROM event_push_actions WHERE room_id = ? AND event_id = ?", + (room_id, event_id) ) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index c6ed54721..7d4012c41 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -205,23 +205,29 @@ class EventsStore(SQLBaseStore): @log_function def _persist_events_txn(self, txn, events_and_contexts, backfilled, is_new_state=True): - - # Remove the any existing cache entries for the event_ids - for event, _ in events_and_contexts: + depth_updates = {} + for event, context in events_and_contexts: + # Remove the any existing cache entries for the event_ids txn.call_after(self._invalidate_get_event_cache, event.event_id) - if not backfilled: txn.call_after( self._events_stream_cache.entity_has_changed, event.room_id, event.internal_metadata.stream_ordering, ) - depth_updates = {} - for event, _ in events_and_contexts: - if event.internal_metadata.is_outlier(): - continue - depth_updates[event.room_id] = max( - event.depth, depth_updates.get(event.room_id, event.depth) + if not event.internal_metadata.is_outlier(): + depth_updates[event.room_id] = max( + event.depth, depth_updates.get(event.room_id, event.depth) + ) + + if context.push_actions: + self._set_push_actions_for_event_and_users( + txn, event, context.push_actions + ) + + if event.type == EventTypes.Redaction and event.redacts is not None: + self._remove_push_actions_for_event_id( + txn, event.room_id, event.redacts ) for room_id, depth in depth_updates.items(): From 78a54822670c94763e6de708797fd561260bbcf5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Feb 2016 16:23:11 +0000 Subject: [PATCH 285/294] Typo --- tests/config/test_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/test_load.py b/tests/config/test_load.py index 528e87853..fbbbf93fe 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -54,7 +54,7 @@ class ConfigLoadingTestCase(unittest.TestCase): "was: %r" % (config.macaroon_secret_key,) ) - def test_load_suceeds_if_macaroon_secret_key_missing(self): + def test_load_succeeds_if_macaroon_secret_key_missing(self): self.generate_config_and_remove_lines_containing("macaroon") config1 = HomeServerConfig.load_config("", ["-c", self.file]) config2 = HomeServerConfig.load_config("", ["-c", self.file]) From 02147452396c67e7874b201460f8b1cc8996a90a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 11:09:56 +0000 Subject: [PATCH 286/294] Rename functions --- synapse/storage/event_push_actions.py | 4 ++-- synapse/storage/events.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 466f07a1c..d77a81768 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) class EventPushActionsStore(SQLBaseStore): - def _set_push_actions_for_event_and_users(self, txn, event, tuples): + def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): """ :param event: the event set actions for :param tuples: list of tuples of (user_id, profile_tag, actions) @@ -100,7 +100,7 @@ class EventPushActionsStore(SQLBaseStore): ) defer.returnValue(ret) - def _remove_push_actions_for_event_id(self, txn, room_id, event_id): + def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): # Sad that we have to blow away the cache for the whole room here txn.call_after( self.get_unread_event_push_actions_by_room_for_user.invalidate_many, diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 7d4012c41..3a5c6ee4b 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -221,12 +221,12 @@ class EventsStore(SQLBaseStore): ) if context.push_actions: - self._set_push_actions_for_event_and_users( + self._set_push_actions_for_event_and_users_txn( txn, event, context.push_actions ) if event.type == EventTypes.Redaction and event.redacts is not None: - self._remove_push_actions_for_event_id( + self._remove_push_actions_for_event_id_txn( txn, event.room_id, event.redacts ) From 9777c5f49ab354206f5aaecdc20880fbb8b17660 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 11:23:32 +0000 Subject: [PATCH 287/294] Set parent context on instansiation --- synapse/util/logcontext.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 301b30d58..ffab03819 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -79,7 +79,7 @@ class LoggingContext(object): sentinel = Sentinel() def __init__(self, name=None): - self.parent_context = None + self.parent_context = LoggingContext.current_context() self.name = name self.ru_stime = 0. self.ru_utime = 0. @@ -116,7 +116,12 @@ class LoggingContext(object): def __enter__(self): """Enters this logging context into thread local storage""" - self.parent_context = self.set_current_context(self) + old_context = self.set_current_context(self) + if self.parent_context != old_context: + logger.warn( + "Expected parent context %r, found %r", + self.parent_context, old_context + ) self.alive = True return self From 00c9ad49dffe92b766ebfaf4f2e0ec82100a3009 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 11:25:19 +0000 Subject: [PATCH 288/294] s/parent_context/previous_context/ --- synapse/util/logcontext.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index ffab03819..e1c55390a 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -47,7 +47,7 @@ class LoggingContext(object): """ __slots__ = [ - "parent_context", "name", "usage_start", "usage_end", "main_thread", + "previous_context", "name", "usage_start", "usage_end", "main_thread", "__dict__", "tag", "alive", ] @@ -79,7 +79,7 @@ class LoggingContext(object): sentinel = Sentinel() def __init__(self, name=None): - self.parent_context = LoggingContext.current_context() + self.previous_context = LoggingContext.current_context() self.name = name self.ru_stime = 0. self.ru_utime = 0. @@ -117,10 +117,10 @@ class LoggingContext(object): def __enter__(self): """Enters this logging context into thread local storage""" old_context = self.set_current_context(self) - if self.parent_context != old_context: + if self.previous_context != old_context: logger.warn( - "Expected parent context %r, found %r", - self.parent_context, old_context + "Expected previous context %r, found %r", + self.previous_context, old_context ) self.alive = True return self @@ -131,7 +131,7 @@ class LoggingContext(object): Returns: None to avoid suppressing any exeptions that were thrown. """ - current = self.set_current_context(self.parent_context) + current = self.set_current_context(self.previous_context) if current is not self: if current is self.sentinel: logger.debug("Expected logging context %s has been lost", self) @@ -141,11 +141,11 @@ class LoggingContext(object): current, self ) - self.parent_context = None + self.previous_context = None self.alive = False def copy_to(self, record): - """Copy fields from this context and its parents to the record""" + """Copy fields from this context to the record""" for key, value in self.__dict__.items(): setattr(record, key, value) @@ -226,7 +226,7 @@ class PreserveLoggingContext(object): ) if self.current_context: - self.has_parent = self.current_context.parent_context is not None + self.has_parent = self.current_context.previous_context is not None if not self.current_context.alive: logger.debug( "Entering dead context: %s", From 4eb8f9ca8a4e69d11591584c3e9b89c388b01f87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 11:29:21 +0000 Subject: [PATCH 289/294] Remove comment --- synapse/util/logcontext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index e1c55390a..5316259d1 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -41,7 +41,7 @@ except: class LoggingContext(object): """Additional context for log formatting. Contexts are scoped within a - "with" block. Contexts inherit the state of their parent contexts. + "with" block. Args: name (str): Name for the context for debugging. """ From 8e49892b218fa10313b2502b4913ae8416321051 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 11:41:04 +0000 Subject: [PATCH 290/294] Only calculate initial sync for 10 rooms at a time This helps to ensure we don't completely starve other requests. --- synapse/handlers/sync.py | 53 ++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ddeed2796..84f29e386 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,7 +18,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes from synapse.util import unwrapFirstError -from synapse.util.logcontext import LoggingContext, PreserveLoggingContext +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext, preserve_fn from synapse.util.metrics import Measure from twisted.internet import defer @@ -228,10 +228,14 @@ class SyncHandler(BaseHandler): invited = [] archived = [] deferreds = [] - for event in room_list: - if event.membership == Membership.JOIN: - with PreserveLoggingContext(LoggingContext.current_context()): - room_sync_deferred = self.full_state_sync_for_joined_room( + + room_list_chunks = [room_list[i:i + 10] for i in xrange(0, len(room_list), 10)] + for room_list_chunk in room_list_chunks: + for event in room_list_chunk: + if event.membership == Membership.JOIN: + room_sync_deferred = preserve_fn( + self.full_state_sync_for_joined_room + )( room_id=event.room_id, sync_config=sync_config, now_token=now_token, @@ -240,20 +244,21 @@ class SyncHandler(BaseHandler): tags_by_room=tags_by_room, account_data_by_room=account_data_by_room, ) - room_sync_deferred.addCallback(joined.append) - deferreds.append(room_sync_deferred) - elif event.membership == Membership.INVITE: - invite = yield self.store.get_event(event.event_id) - invited.append(InvitedSyncResult( - room_id=event.room_id, - invite=invite, - )) - elif event.membership in (Membership.LEAVE, Membership.BAN): - leave_token = now_token.copy_and_replace( - "room_key", "s%d" % (event.stream_ordering,) - ) - with PreserveLoggingContext(LoggingContext.current_context()): - room_sync_deferred = self.full_state_sync_for_archived_room( + room_sync_deferred.addCallback(joined.append) + deferreds.append(room_sync_deferred) + elif event.membership == Membership.INVITE: + invite = yield self.store.get_event(event.event_id) + invited.append(InvitedSyncResult( + room_id=event.room_id, + invite=invite, + )) + elif event.membership in (Membership.LEAVE, Membership.BAN): + leave_token = now_token.copy_and_replace( + "room_key", "s%d" % (event.stream_ordering,) + ) + room_sync_deferred = preserve_fn( + self.full_state_sync_for_archived_room + )( sync_config=sync_config, room_id=event.room_id, leave_event_id=event.event_id, @@ -262,12 +267,12 @@ class SyncHandler(BaseHandler): tags_by_room=tags_by_room, account_data_by_room=account_data_by_room, ) - room_sync_deferred.addCallback(archived.append) - deferreds.append(room_sync_deferred) + room_sync_deferred.addCallback(archived.append) + deferreds.append(room_sync_deferred) - yield defer.gatherResults( - deferreds, consumeErrors=True - ).addErrback(unwrapFirstError) + yield defer.gatherResults( + deferreds, consumeErrors=True + ).addErrback(unwrapFirstError) account_data_for_user = sync_config.filter_collection.filter_account_data( self.account_data_for_user(account_data) From 24f00a6c33900cf701330ff324b0479c1898d5ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 12:57:50 +0000 Subject: [PATCH 291/294] Use _simple_select_many for _get_state_group_for_events --- synapse/handlers/sync.py | 2 +- synapse/storage/state.py | 26 ++++++++++---------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 84f29e386..1d0f0058a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,7 +18,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes from synapse.util import unwrapFirstError -from synapse.util.logcontext import LoggingContext, PreserveLoggingContext, preserve_fn +from synapse.util.logcontext import LoggingContext, preserve_fn from synapse.util.metrics import Measure from twisted.internet import defer diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 6c32e8f7b..90ec50bb5 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -264,26 +264,20 @@ class StateStore(SQLBaseStore): ) @cachedList(cache=_get_state_group_for_event.cache, list_name="event_ids", - num_args=1) + num_args=1, inlineCallbacks=True) def _get_state_group_for_events(self, event_ids): """Returns mapping event_id -> state_group """ - def f(txn): - results = {} - for event_id in event_ids: - results[event_id] = self._simple_select_one_onecol_txn( - txn, - table="event_to_state_groups", - keyvalues={ - "event_id": event_id, - }, - retcol="state_group", - allow_none=True, - ) + rows = yield self._simple_select_many_batch( + table="event_to_state_groups", + column="event_id", + iterable=event_ids, + keyvalues={}, + retcols=("event_id", "state_group",), + desc="_get_state_group_for_events", + ) - return results - - return self.runInteraction("_get_state_group_for_events", f) + defer.returnValue({row["event_id"]: row["state_group"] for row in rows}) def _get_some_state_from_cache(self, group, types): """Checks if group is in cache. See `_get_state_for_groups` From 5189bfdef4c87a7b0527de603eae52ac27bd500c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 13:24:42 +0000 Subject: [PATCH 292/294] Batch fetch _get_state_groups_from_groups --- synapse/storage/state.py | 72 +++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 90ec50bb5..372b54000 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -171,41 +171,43 @@ class StateStore(SQLBaseStore): events = yield self._get_events(event_ids, get_prev_content=False) defer.returnValue(events) - def _get_state_groups_from_groups(self, groups_and_types): + def _get_state_groups_from_groups(self, groups, types): """Returns dictionary state_group -> state event ids - - Args: - groups_and_types (list): list of 2-tuple (`group`, `types`) """ - def f(txn): + def f(txn, groups): + if types is not None: + where_clause = "AND (%s)" % ( + " OR ".join(["(type = ? AND state_key = ?)"] * len(types)), + ) + else: + where_clause = "" + + sql = ( + "SELECT state_group, event_id FROM state_groups_state WHERE" + " state_group IN (%s) %s" % ( + ",".join("?" for _ in groups), + where_clause, + ) + ) + + args = list(groups) + if types is not None: + args.extend([i for typ in types for i in typ]) + + txn.execute(sql, args) + rows = self.cursor_to_dict(txn) + results = {} - for group, types in groups_and_types: - if types is not None: - where_clause = "AND (%s)" % ( - " OR ".join(["(type = ? AND state_key = ?)"] * len(types)), - ) - else: - where_clause = "" - - sql = ( - "SELECT event_id FROM state_groups_state WHERE" - " state_group = ? %s" - ) % (where_clause,) - - args = [group] - if types is not None: - args.extend([i for typ in types for i in typ]) - - txn.execute(sql, args) - - results[group] = [r[0] for r in txn.fetchall()] - + for row in rows: + results.setdefault(row["state_group"], []).append(row["event_id"]) return results - return self.runInteraction( - "_get_state_groups_from_groups", - f, - ) + chunks = [groups[i:i + 100] for i in xrange(0, len(groups), 100)] + for chunk in chunks: + return self.runInteraction( + "_get_state_groups_from_groups", + f, chunk + ) @defer.inlineCallbacks def get_state_for_events(self, event_ids, types): @@ -349,7 +351,7 @@ class StateStore(SQLBaseStore): all events are returned. """ results = {} - missing_groups_and_types = [] + missing_groups = [] if types is not None: for group in set(groups): state_dict, missing_types, got_all = self._get_some_state_from_cache( @@ -358,7 +360,7 @@ class StateStore(SQLBaseStore): results[group] = state_dict if not got_all: - missing_groups_and_types.append((group, missing_types)) + missing_groups.append(group) else: for group in set(groups): state_dict, got_all = self._get_all_state_from_cache( @@ -367,9 +369,9 @@ class StateStore(SQLBaseStore): results[group] = state_dict if not got_all: - missing_groups_and_types.append((group, None)) + missing_groups.append(group) - if not missing_groups_and_types: + if not missing_groups: defer.returnValue({ group: { type_tuple: event @@ -383,7 +385,7 @@ class StateStore(SQLBaseStore): cache_seq_num = self._state_group_cache.sequence group_state_dict = yield self._get_state_groups_from_groups( - missing_groups_and_types + missing_groups, types ) state_events = yield self._get_events( From 6e89e69d08fe47b3dde4723fe24911231ed390c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 14:36:06 +0000 Subject: [PATCH 293/294] Bump version and changelog --- CHANGES.rst | 22 ++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b9c956fd0..3ebbdd482 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,25 @@ +Changes in synapse v0.13.0 (2016-02-10) +======================================= + +This version includes an upgrade of the schema, specifically adding an index to +the ``events`` table, which may take a minute to complete. + +Changes: + +* Improve general performance (PR #540, #543. #544, #54, #549, #567) +* Change guest user ids to be incrementing integers (PR #550) +* Improve performance of public room list API (PR #552) +* Change profile API to omit keys rather than return null (PR #557) +* Change to forward ``/media/r0`` to ``/media/v1`` (PR #595) + +Bug fixes: + +* Fix bug with upgrading guest accounts where it would fail if you opened the + registration email on a different device (PR #547) +* Fix bug where unread count could be wrong (PR #568) + + + Changes in synapse v0.12.1-rc1 (2016-01-29) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index aacacda4f..426330cc2 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.12.1-rc1" +__version__ = "0.13.0" From 6c3126d950d31aa88020e967d8fd1634f6312ccc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Feb 2016 14:49:48 +0000 Subject: [PATCH 294/294] Update CHANGES --- CHANGES.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3ebbdd482..f1c67b20e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,7 +2,8 @@ Changes in synapse v0.13.0 (2016-02-10) ======================================= This version includes an upgrade of the schema, specifically adding an index to -the ``events`` table, which may take a minute to complete. +the ``events`` table. This may cause synapse to pause for several minutes the +first time it is started after the upgrade. Changes: @@ -10,7 +11,8 @@ Changes: * Change guest user ids to be incrementing integers (PR #550) * Improve performance of public room list API (PR #552) * Change profile API to omit keys rather than return null (PR #557) -* Change to forward ``/media/r0`` to ``/media/v1`` (PR #595) +* Add ``/media/r0`` endpoint prefix, which is equivalent to ``/media/v1/`` + (PR #595) Bug fixes: