mirror of
https://mau.dev/maunium/synapse.git
synced 2024-11-17 15:31:19 +01:00
bb81e78ec6
Splits the state_group_cache in two. One half contains normal state events; the other contains member events. The idea is that the lazyloading common case of: "I want a subset of member events plus all of the other state" can be accomplished efficiently by splitting the cache into two, and asking for "all events" from the non-members cache, and "just these keys" from the members cache. This means we can avoid having to make DictionaryCache aware of these sort of complicated queries, whilst letting LL requests benefit from the caching. Previously we were unable to sensibly use the caching and had to pull all state from the DB irrespective of the filtering, which made things slow. Hopefully fixes https://github.com/matrix-org/synapse/issues/3720.
435 lines
14 KiB
Python
435 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2018 New Vector Ltd
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import logging
|
|
|
|
from twisted.internet import defer
|
|
|
|
from synapse.api.constants import EventTypes, Membership
|
|
from synapse.types import RoomID, UserID
|
|
|
|
import tests.unittest
|
|
import tests.utils
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class StateStoreTestCase(tests.unittest.TestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
super(StateStoreTestCase, self).__init__(*args, **kwargs)
|
|
self.store = None # type: synapse.storage.DataStore
|
|
|
|
@defer.inlineCallbacks
|
|
def setUp(self):
|
|
hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
|
|
|
|
self.store = hs.get_datastore()
|
|
self.event_builder_factory = hs.get_event_builder_factory()
|
|
self.event_creation_handler = hs.get_event_creation_handler()
|
|
|
|
self.u_alice = UserID.from_string("@alice:test")
|
|
self.u_bob = UserID.from_string("@bob:test")
|
|
|
|
self.room = RoomID.from_string("!abc123:test")
|
|
|
|
yield self.store.store_room(
|
|
self.room.to_string(), room_creator_user_id="@creator:text", is_public=True
|
|
)
|
|
|
|
@defer.inlineCallbacks
|
|
def inject_state_event(self, room, sender, typ, state_key, content):
|
|
builder = self.event_builder_factory.new(
|
|
{
|
|
"type": typ,
|
|
"sender": sender.to_string(),
|
|
"state_key": state_key,
|
|
"room_id": room.to_string(),
|
|
"content": content,
|
|
}
|
|
)
|
|
|
|
event, context = yield self.event_creation_handler.create_new_client_event(
|
|
builder
|
|
)
|
|
|
|
yield self.store.persist_event(event, context)
|
|
|
|
defer.returnValue(event)
|
|
|
|
def assertStateMapEqual(self, s1, s2):
|
|
for t in s1:
|
|
# just compare event IDs for simplicity
|
|
self.assertEqual(s1[t].event_id, s2[t].event_id)
|
|
self.assertEqual(len(s1), len(s2))
|
|
|
|
@defer.inlineCallbacks
|
|
def test_get_state_for_event(self):
|
|
|
|
# this defaults to a linear DAG as each new injection defaults to whatever
|
|
# forward extremities are currently in the DB for this room.
|
|
e1 = yield self.inject_state_event(
|
|
self.room, self.u_alice, EventTypes.Create, '', {}
|
|
)
|
|
e2 = yield self.inject_state_event(
|
|
self.room, self.u_alice, EventTypes.Name, '', {"name": "test room"}
|
|
)
|
|
e3 = yield self.inject_state_event(
|
|
self.room,
|
|
self.u_alice,
|
|
EventTypes.Member,
|
|
self.u_alice.to_string(),
|
|
{"membership": Membership.JOIN},
|
|
)
|
|
e4 = yield self.inject_state_event(
|
|
self.room,
|
|
self.u_bob,
|
|
EventTypes.Member,
|
|
self.u_bob.to_string(),
|
|
{"membership": Membership.JOIN},
|
|
)
|
|
e5 = yield self.inject_state_event(
|
|
self.room,
|
|
self.u_bob,
|
|
EventTypes.Member,
|
|
self.u_bob.to_string(),
|
|
{"membership": Membership.LEAVE},
|
|
)
|
|
|
|
# check we get the full state as of the final event
|
|
state = yield self.store.get_state_for_event(
|
|
e5.event_id, None, filtered_types=None
|
|
)
|
|
|
|
self.assertIsNotNone(e4)
|
|
|
|
self.assertStateMapEqual(
|
|
{
|
|
(e1.type, e1.state_key): e1,
|
|
(e2.type, e2.state_key): e2,
|
|
(e3.type, e3.state_key): e3,
|
|
# e4 is overwritten by e5
|
|
(e5.type, e5.state_key): e5,
|
|
},
|
|
state,
|
|
)
|
|
|
|
# check we can filter to the m.room.name event (with a '' state key)
|
|
state = yield self.store.get_state_for_event(
|
|
e5.event_id, [(EventTypes.Name, '')], filtered_types=None
|
|
)
|
|
|
|
self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
|
|
|
|
# check we can filter to the m.room.name event (with a wildcard None state key)
|
|
state = yield self.store.get_state_for_event(
|
|
e5.event_id, [(EventTypes.Name, None)], filtered_types=None
|
|
)
|
|
|
|
self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
|
|
|
|
# check we can grab the m.room.member events (with a wildcard None state key)
|
|
state = yield self.store.get_state_for_event(
|
|
e5.event_id, [(EventTypes.Member, None)], filtered_types=None
|
|
)
|
|
|
|
self.assertStateMapEqual(
|
|
{(e3.type, e3.state_key): e3, (e5.type, e5.state_key): e5}, state
|
|
)
|
|
|
|
# check we can use filtered_types to grab a specific room member
|
|
# without filtering out the other event types
|
|
state = yield self.store.get_state_for_event(
|
|
e5.event_id,
|
|
[(EventTypes.Member, self.u_alice.to_string())],
|
|
filtered_types=[EventTypes.Member],
|
|
)
|
|
|
|
self.assertStateMapEqual(
|
|
{
|
|
(e1.type, e1.state_key): e1,
|
|
(e2.type, e2.state_key): e2,
|
|
(e3.type, e3.state_key): e3,
|
|
},
|
|
state,
|
|
)
|
|
|
|
# check that types=[], filtered_types=[EventTypes.Member]
|
|
# doesn't return all members
|
|
state = yield self.store.get_state_for_event(
|
|
e5.event_id, [], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertStateMapEqual(
|
|
{(e1.type, e1.state_key): e1, (e2.type, e2.state_key): e2}, state
|
|
)
|
|
|
|
#######################################################
|
|
# _get_some_state_from_cache tests against a full cache
|
|
#######################################################
|
|
|
|
room_id = self.room.to_string()
|
|
group_ids = yield self.store.get_state_groups_ids(room_id, [e5.event_id])
|
|
group = list(group_ids.keys())[0]
|
|
|
|
# test _get_some_state_from_cache correctly filters out members with types=[]
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_cache,
|
|
group, [], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e1.type, e1.state_key): e1.event_id,
|
|
(e2.type, e2.state_key): e2.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group, [], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{},
|
|
state_dict,
|
|
)
|
|
|
|
# test _get_some_state_from_cache correctly filters in members with wildcard types
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_cache,
|
|
group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e1.type, e1.state_key): e1.event_id,
|
|
(e2.type, e2.state_key): e2.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e3.type, e3.state_key): e3.event_id,
|
|
# e4 is overwritten by e5
|
|
(e5.type, e5.state_key): e5.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
# test _get_some_state_from_cache correctly filters in members with specific types
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_cache,
|
|
group,
|
|
[(EventTypes.Member, e5.state_key)],
|
|
filtered_types=[EventTypes.Member],
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e1.type, e1.state_key): e1.event_id,
|
|
(e2.type, e2.state_key): e2.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group,
|
|
[(EventTypes.Member, e5.state_key)],
|
|
filtered_types=[EventTypes.Member],
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e5.type, e5.state_key): e5.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
# test _get_some_state_from_cache correctly filters in members with specific types
|
|
# and no filtered_types
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group, [(EventTypes.Member, e5.state_key)], filtered_types=None
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
|
|
|
|
#######################################################
|
|
# deliberately remove e2 (room name) from the _state_group_cache
|
|
|
|
(is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(
|
|
group
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertEqual(known_absent, set())
|
|
self.assertDictEqual(
|
|
state_dict_ids,
|
|
{
|
|
(e1.type, e1.state_key): e1.event_id,
|
|
(e2.type, e2.state_key): e2.event_id,
|
|
},
|
|
)
|
|
|
|
state_dict_ids.pop((e2.type, e2.state_key))
|
|
self.store._state_group_cache.invalidate(group)
|
|
self.store._state_group_cache.update(
|
|
sequence=self.store._state_group_cache.sequence,
|
|
key=group,
|
|
value=state_dict_ids,
|
|
# list fetched keys so it knows it's partial
|
|
fetched_keys=(
|
|
(e1.type, e1.state_key),
|
|
),
|
|
)
|
|
|
|
(is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(
|
|
group
|
|
)
|
|
|
|
self.assertEqual(is_all, False)
|
|
self.assertEqual(
|
|
known_absent,
|
|
set(
|
|
[
|
|
(e1.type, e1.state_key),
|
|
]
|
|
),
|
|
)
|
|
self.assertDictEqual(
|
|
state_dict_ids,
|
|
{
|
|
(e1.type, e1.state_key): e1.event_id,
|
|
},
|
|
)
|
|
|
|
############################################
|
|
# test that things work with a partial cache
|
|
|
|
# test _get_some_state_from_cache correctly filters out members with types=[]
|
|
room_id = self.room.to_string()
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_cache,
|
|
group, [], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, False)
|
|
self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
|
|
|
|
room_id = self.room.to_string()
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group, [], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual({}, state_dict)
|
|
|
|
# test _get_some_state_from_cache correctly filters in members wildcard types
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_cache,
|
|
group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, False)
|
|
self.assertDictEqual(
|
|
{
|
|
(e1.type, e1.state_key): e1.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e3.type, e3.state_key): e3.event_id,
|
|
(e5.type, e5.state_key): e5.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
# test _get_some_state_from_cache correctly filters in members with specific types
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_cache,
|
|
group,
|
|
[(EventTypes.Member, e5.state_key)],
|
|
filtered_types=[EventTypes.Member],
|
|
)
|
|
|
|
self.assertEqual(is_all, False)
|
|
self.assertDictEqual(
|
|
{
|
|
(e1.type, e1.state_key): e1.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group,
|
|
[(EventTypes.Member, e5.state_key)],
|
|
filtered_types=[EventTypes.Member],
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e5.type, e5.state_key): e5.event_id,
|
|
},
|
|
state_dict,
|
|
)
|
|
|
|
# test _get_some_state_from_cache correctly filters in members with specific types
|
|
# and no filtered_types
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_cache,
|
|
group, [(EventTypes.Member, e5.state_key)], filtered_types=None
|
|
)
|
|
|
|
self.assertEqual(is_all, False)
|
|
self.assertDictEqual({}, state_dict)
|
|
|
|
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
|
|
self.store._state_group_members_cache,
|
|
group, [(EventTypes.Member, e5.state_key)], filtered_types=None
|
|
)
|
|
|
|
self.assertEqual(is_all, True)
|
|
self.assertDictEqual(
|
|
{
|
|
(e5.type, e5.state_key): e5.event_id,
|
|
},
|
|
state_dict,
|
|
)
|