Merge pull request #6329 from matrix-org/babolivier/context_filters

Filter state, events_before and events_after in /context requests

changelog.d/6329.bugfix

@@ -0,0 +1 @@
+Correctly apply the event filter to the `state`, `events_before` and `events_after` fields in the response to `/context` requests.

synapse/api/constants.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/filtering.py

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/room.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -900,6 +901,10 @@ class RoomContextHandler(object):
             room_id, event_id, before_limit, after_limit, event_filter
         )
 
+        if event_filter:
+            results["events_before"] = event_filter.filter(results["events_before"])
+            results["events_after"] = event_filter.filter(results["events_after"])
+
         results["events_before"] = yield filter_evts(results["events_before"])
         results["events_after"] = yield filter_evts(results["events_after"])
         results["event"] = event
@@ -928,7 +933,12 @@ class RoomContextHandler(object):
         state = yield self.state_store.get_state_for_events(
             [last_event_id], state_filter=state_filter
         )
-        results["state"] = list(state[last_event_id].values())
+
+        state_events = list(state[last_event_id].values())
+        if event_filter:
+            state_events = event_filter.filter(state_events)
+
+        results["state"] = state_events
 
         # We use a dummy token here as we only care about the room portion of
         # the token, which we replace.

synapse/rest/client/versions.py

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/data_stores/main/stream.py

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

tests/api/test_filtering.py

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

tests/rest/client/v1/test_rooms.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
+# Copyright 2018-2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -813,105 +815,6 @@ class RoomMessageListTestCase(RoomBase):
         self.assertTrue("chunk" in channel.json_body)
         self.assertTrue("end" in channel.json_body)
 
-    def test_filter_labels(self):
-        """Test that we can filter by a label."""
-        message_filter = json.dumps(
-            {"types": [EventTypes.Message], "org.matrix.labels": ["#fun"]}
-        )
-
-        events = self._test_filter_labels(message_filter)
-
-        self.assertEqual(len(events), 2, [event["content"] for event in events])
-        self.assertEqual(events[0]["content"]["body"], "with right label", events[0])
-        self.assertEqual(events[1]["content"]["body"], "with right label", events[1])
-
-    def test_filter_not_labels(self):
-        """Test that we can filter by the absence of a label."""
-        message_filter = json.dumps(
-            {"types": [EventTypes.Message], "org.matrix.not_labels": ["#fun"]}
-        )
-
-        events = self._test_filter_labels(message_filter)
-
-        self.assertEqual(len(events), 3, [event["content"] for event in events])
-        self.assertEqual(events[0]["content"]["body"], "without label", events[0])
-        self.assertEqual(events[1]["content"]["body"], "with wrong label", events[1])
-        self.assertEqual(
-            events[2]["content"]["body"], "with two wrong labels", events[2]
-        )
-
-    def test_filter_labels_not_labels(self):
-        """Test that we can filter by both a label and the absence of another label."""
-        sync_filter = json.dumps(
-            {
-                "types": [EventTypes.Message],
-                "org.matrix.labels": ["#work"],
-                "org.matrix.not_labels": ["#notfun"],
-            }
-        )
-
-        events = self._test_filter_labels(sync_filter)
-
-        self.assertEqual(len(events), 1, [event["content"] for event in events])
-        self.assertEqual(events[0]["content"]["body"], "with wrong label", events[0])
-
-    def _test_filter_labels(self, message_filter):
-        self.helper.send_event(
-            room_id=self.room_id,
-            type=EventTypes.Message,
-            content={
-                "msgtype": "m.text",
-                "body": "with right label",
-                EventContentFields.LABELS: ["#fun"],
-            },
-        )
-
-        self.helper.send_event(
-            room_id=self.room_id,
-            type=EventTypes.Message,
-            content={"msgtype": "m.text", "body": "without label"},
-        )
-
-        self.helper.send_event(
-            room_id=self.room_id,
-            type=EventTypes.Message,
-            content={
-                "msgtype": "m.text",
-                "body": "with wrong label",
-                EventContentFields.LABELS: ["#work"],
-            },
-        )
-
-        self.helper.send_event(
-            room_id=self.room_id,
-            type=EventTypes.Message,
-            content={
-                "msgtype": "m.text",
-                "body": "with two wrong labels",
-                EventContentFields.LABELS: ["#work", "#notfun"],
-            },
-        )
-
-        self.helper.send_event(
-            room_id=self.room_id,
-            type=EventTypes.Message,
-            content={
-                "msgtype": "m.text",
-                "body": "with right label",
-                EventContentFields.LABELS: ["#fun"],
-            },
-        )
-
-        token = "s0_0_0_0_0_0_0_0_0"
-        request, channel = self.make_request(
-            "GET",
-            "/rooms/%s/messages?access_token=x&from=%s&filter=%s"
-            % (self.room_id, token, message_filter),
-        )
-        self.render(request)
-
-        return channel.json_body["chunk"]
-
     def test_room_messages_purge(self):
         store = self.hs.get_datastore()
         pagination_handler = self.hs.get_pagination_handler()
@@ -1320,3 +1223,377 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase):
         event_content = channel.json_body
 
         self.assertEqual(event_content.get("reason"), reason, channel.result)
+
+
+class LabelsTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+        profile.register_servlets,
+    ]
+
+    # Filter that should only catch messages with the label "#fun".
+    FILTER_LABELS = {
+        "types": [EventTypes.Message],
+        "org.matrix.labels": ["#fun"],
+    }
+    # Filter that should only catch messages without the label "#fun".
+    FILTER_NOT_LABELS = {
+        "types": [EventTypes.Message],
+        "org.matrix.not_labels": ["#fun"],
+    }
+    # Filter that should only catch messages with the label "#work" but without the label
+    # "#notfun".
+    FILTER_LABELS_NOT_LABELS = {
+        "types": [EventTypes.Message],
+        "org.matrix.labels": ["#work"],
+        "org.matrix.not_labels": ["#notfun"],
+    }
+
+    def prepare(self, reactor, clock, homeserver):
+        self.user_id = self.register_user("test", "test")
+        self.tok = self.login("test", "test")
+        self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok)
+
+    def test_context_filter_labels(self):
+        """Test that we can filter by a label on a /context request."""
+        event_id = self._send_labelled_messages_in_room()
+
+        request, channel = self.make_request(
+            "GET",
+            "/rooms/%s/context/%s?filter=%s"
+            % (self.room_id, event_id, json.dumps(self.FILTER_LABELS)),
+            access_token=self.tok,
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200, channel.result)
+
+        events_before = channel.json_body["events_before"]
+
+        self.assertEqual(
+            len(events_before), 1, [event["content"] for event in events_before]
+        )
+        self.assertEqual(
+            events_before[0]["content"]["body"], "with right label", events_before[0]
+        )
+
+        events_after = channel.json_body["events_before"]
+
+        self.assertEqual(
+            len(events_after), 1, [event["content"] for event in events_after]
+        )
+        self.assertEqual(
+            events_after[0]["content"]["body"], "with right label", events_after[0]
+        )
+
+    def test_context_filter_not_labels(self):
+        """Test that we can filter by the absence of a label on a /context request."""
+        event_id = self._send_labelled_messages_in_room()
+
+        request, channel = self.make_request(
+            "GET",
+            "/rooms/%s/context/%s?filter=%s"
+            % (self.room_id, event_id, json.dumps(self.FILTER_NOT_LABELS)),
+            access_token=self.tok,
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200, channel.result)
+
+        events_before = channel.json_body["events_before"]
+
+        self.assertEqual(
+            len(events_before), 1, [event["content"] for event in events_before]
+        )
+        self.assertEqual(
+            events_before[0]["content"]["body"], "without label", events_before[0]
+        )
+
+        events_after = channel.json_body["events_after"]
+
+        self.assertEqual(
+            len(events_after), 2, [event["content"] for event in events_after]
+        )
+        self.assertEqual(
+            events_after[0]["content"]["body"], "with wrong label", events_after[0]
+        )
+        self.assertEqual(
+            events_after[1]["content"]["body"], "with two wrong labels", events_after[1]
+        )
+
+    def test_context_filter_labels_not_labels(self):
+        """Test that we can filter by both a label and the absence of another label on a
+        /context request.
+        """
+        event_id = self._send_labelled_messages_in_room()
+
+        request, channel = self.make_request(
+            "GET",
+            "/rooms/%s/context/%s?filter=%s"
+            % (self.room_id, event_id, json.dumps(self.FILTER_LABELS_NOT_LABELS)),
+            access_token=self.tok,
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200, channel.result)
+
+        events_before = channel.json_body["events_before"]
+
+        self.assertEqual(
+            len(events_before), 0, [event["content"] for event in events_before]
+        )
+
+        events_after = channel.json_body["events_after"]
+
+        self.assertEqual(
+            len(events_after), 1, [event["content"] for event in events_after]
+        )
+        self.assertEqual(
+            events_after[0]["content"]["body"], "with wrong label", events_after[0]
+        )
+
+    def test_messages_filter_labels(self):
+        """Test that we can filter by a label on a /messages request."""
+        self._send_labelled_messages_in_room()
+
+        token = "s0_0_0_0_0_0_0_0_0"
+        request, channel = self.make_request(
+            "GET",
+            "/rooms/%s/messages?access_token=%s&from=%s&filter=%s"
+            % (self.room_id, self.tok, token, json.dumps(self.FILTER_LABELS)),
+        )
+        self.render(request)
+
+        events = channel.json_body["chunk"]
+
+        self.assertEqual(len(events), 2, [event["content"] for event in events])
+        self.assertEqual(events[0]["content"]["body"], "with right label", events[0])
+        self.assertEqual(events[1]["content"]["body"], "with right label", events[1])
+
+    def test_messages_filter_not_labels(self):
+        """Test that we can filter by the absence of a label on a /messages request."""
+        self._send_labelled_messages_in_room()
+
+        token = "s0_0_0_0_0_0_0_0_0"
+        request, channel = self.make_request(
+            "GET",
+            "/rooms/%s/messages?access_token=%s&from=%s&filter=%s"
+            % (self.room_id, self.tok, token, json.dumps(self.FILTER_NOT_LABELS)),
+        )
+        self.render(request)
+
+        events = channel.json_body["chunk"]
+
+        self.assertEqual(len(events), 4, [event["content"] for event in events])
+        self.assertEqual(events[0]["content"]["body"], "without label", events[0])
+        self.assertEqual(events[1]["content"]["body"], "without label", events[1])
+        self.assertEqual(events[2]["content"]["body"], "with wrong label", events[2])
+        self.assertEqual(
+            events[3]["content"]["body"], "with two wrong labels", events[3]
+        )
+
+    def test_messages_filter_labels_not_labels(self):
+        """Test that we can filter by both a label and the absence of another label on a
+        /messages request.
+        """
+        self._send_labelled_messages_in_room()
+
+        token = "s0_0_0_0_0_0_0_0_0"
+        request, channel = self.make_request(
+            "GET",
+            "/rooms/%s/messages?access_token=%s&from=%s&filter=%s"
+            % (
+                self.room_id,
+                self.tok,
+                token,
+                json.dumps(self.FILTER_LABELS_NOT_LABELS),
+            ),
+        )
+        self.render(request)
+
+        events = channel.json_body["chunk"]
+
+        self.assertEqual(len(events), 1, [event["content"] for event in events])
+        self.assertEqual(events[0]["content"]["body"], "with wrong label", events[0])
+
+    def test_search_filter_labels(self):
+        """Test that we can filter by a label on a /search request."""
+        request_data = json.dumps(
+            {
+                "search_categories": {
+                    "room_events": {
+                        "search_term": "label",
+                        "filter": self.FILTER_LABELS,
+                    }
+                }
+            }
+        )
+
+        self._send_labelled_messages_in_room()
+
+        request, channel = self.make_request(
+            "POST", "/search?access_token=%s" % self.tok, request_data
+        )
+        self.render(request)
+
+        results = channel.json_body["search_categories"]["room_events"]["results"]
+
+        self.assertEqual(
+            len(results), 2, [result["result"]["content"] for result in results],
+        )
+        self.assertEqual(
+            results[0]["result"]["content"]["body"],
+            "with right label",
+            results[0]["result"]["content"]["body"],
+        )
+        self.assertEqual(
+            results[1]["result"]["content"]["body"],
+            "with right label",
+            results[1]["result"]["content"]["body"],
+        )
+
+    def test_search_filter_not_labels(self):
+        """Test that we can filter by the absence of a label on a /search request."""
+        request_data = json.dumps(
+            {
+                "search_categories": {
+                    "room_events": {
+                        "search_term": "label",
+                        "filter": self.FILTER_NOT_LABELS,
+                    }
+                }
+            }
+        )
+
+        self._send_labelled_messages_in_room()
+
+        request, channel = self.make_request(
+            "POST", "/search?access_token=%s" % self.tok, request_data
+        )
+        self.render(request)
+
+        results = channel.json_body["search_categories"]["room_events"]["results"]
+
+        self.assertEqual(
+            len(results), 4, [result["result"]["content"] for result in results],
+        )
+        self.assertEqual(
+            results[0]["result"]["content"]["body"],
+            "without label",
+            results[0]["result"]["content"]["body"],
+        )
+        self.assertEqual(
+            results[1]["result"]["content"]["body"],
+            "without label",
+            results[1]["result"]["content"]["body"],
+        )
+        self.assertEqual(
+            results[2]["result"]["content"]["body"],
+            "with wrong label",
+            results[2]["result"]["content"]["body"],
+        )
+        self.assertEqual(
+            results[3]["result"]["content"]["body"],
+            "with two wrong labels",
+            results[3]["result"]["content"]["body"],
+        )
+
+    def test_search_filter_labels_not_labels(self):
+        """Test that we can filter by both a label and the absence of another label on a
+        /search request.
+        """
+        request_data = json.dumps(
+            {
+                "search_categories": {
+                    "room_events": {
+                        "search_term": "label",
+                        "filter": self.FILTER_LABELS_NOT_LABELS,
+                    }
+                }
+            }
+        )
+
+        self._send_labelled_messages_in_room()
+
+        request, channel = self.make_request(
+            "POST", "/search?access_token=%s" % self.tok, request_data
+        )
+        self.render(request)
+
+        results = channel.json_body["search_categories"]["room_events"]["results"]
+
+        self.assertEqual(
+            len(results), 1, [result["result"]["content"] for result in results],
+        )
+        self.assertEqual(
+            results[0]["result"]["content"]["body"],
+            "with wrong label",
+            results[0]["result"]["content"]["body"],
+        )
+
+    def _send_labelled_messages_in_room(self):
+        """Sends several messages to a room with different labels (or without any) to test
+        filtering by label.
+        Returns:
+            The ID of the event to use if we're testing filtering on /context.
+        """
+        self.helper.send_event(
+            room_id=self.room_id,
+            type=EventTypes.Message,
+            content={
+                "msgtype": "m.text",
+                "body": "with right label",
+                EventContentFields.LABELS: ["#fun"],
+            },
+            tok=self.tok,
+        )
+
+        self.helper.send_event(
+            room_id=self.room_id,
+            type=EventTypes.Message,
+            content={"msgtype": "m.text", "body": "without label"},
+            tok=self.tok,
+        )
+
+        res = self.helper.send_event(
+            room_id=self.room_id,
+            type=EventTypes.Message,
+            content={"msgtype": "m.text", "body": "without label"},
+            tok=self.tok,
+        )
+        # Return this event's ID when we test filtering in /context requests.
+        event_id = res["event_id"]
+
+        self.helper.send_event(
+            room_id=self.room_id,
+            type=EventTypes.Message,
+            content={
+                "msgtype": "m.text",
+                "body": "with wrong label",
+                EventContentFields.LABELS: ["#work"],
+            },
+            tok=self.tok,
+        )
+
+        self.helper.send_event(
+            room_id=self.room_id,
+            type=EventTypes.Message,
+            content={
+                "msgtype": "m.text",
+                "body": "with two wrong labels",
+                EventContentFields.LABELS: ["#work", "#notfun"],
+            },
+            tok=self.tok,
+        )
+
+        self.helper.send_event(
+            room_id=self.room_id,
+            type=EventTypes.Message,
+            content={
+                "msgtype": "m.text",
+                "body": "with right label",
+                EventContentFields.LABELS: ["#fun"],
+            },
+            tok=self.tok,
+        )
+
+        return event_id

tests/rest/client/v1/utils.py

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

tests/rest/client/v2_alpha/test_sync.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright 2018 New Vector
+# Copyright 2018-2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.