Support for MSC2285 (hidden read receipts) (#10413)

Implementation of matrix-org/matrix-doc#2285

changelog.d/10413.feature

@@ -0,0 +1 @@
+Support for [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-doc/pull/2285). Contributed by @SimonBrandner.

synapse/api/constants.py

@@ -224,3 +224,7 @@ class HistoryVisibility:
     JOINED = "joined"
     SHARED = "shared"
     WORLD_READABLE = "world_readable"
+
+
+class ReadReceiptEventFields:
+    MSC2285_HIDDEN = "org.matrix.msc2285.hidden"

synapse/config/experimental.py

@@ -33,5 +33,8 @@ class ExperimentalConfig(Config):
         # MSC2716 (backfill existing history)
         self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
 
+        # MSC2285 (hidden read receipts)
+        self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False)
+
         # MSC3244 (room version capabilities)
         self.msc3244_enabled: bool = experimental.get("msc3244_enabled", False)

synapse/handlers/initial_sync.py

@@ -21,6 +21,7 @@ from synapse.api.constants import EduTypes, EventTypes, Membership
 from synapse.api.errors import SynapseError
 from synapse.events.validator import EventValidator
 from synapse.handlers.presence import format_user_presence_state
+from synapse.handlers.receipts import ReceiptEventSource
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.storage.roommember import RoomsForUser
 from synapse.streams.config import PaginationConfig
@@ -134,6 +135,8 @@ class InitialSyncHandler(BaseHandler):
             joined_rooms,
             to_key=int(now_token.receipt_key),
         )
+        if self.hs.config.experimental.msc2285_enabled:
+            receipt = ReceiptEventSource.filter_out_hidden(receipt, user_id)
 
         tags_by_room = await self.store.get_tags_for_user(user_id)
 
@@ -430,7 +433,9 @@ class InitialSyncHandler(BaseHandler):
                 room_id, to_key=now_token.receipt_key
             )
             if not receipts:
-                receipts = []
+                return []
+            if self.hs.config.experimental.msc2285_enabled:
+                receipts = ReceiptEventSource.filter_out_hidden(receipts, user_id)
             return receipts
 
         presence, receipts, (messages, token) = await make_deferred_yieldable(

synapse/handlers/receipts.py

@@ -14,9 +14,10 @@
 import logging
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
+from synapse.api.constants import ReadReceiptEventFields
 from synapse.appservice import ApplicationService
 from synapse.handlers._base import BaseHandler
-from synapse.types import JsonDict, ReadReceipt, get_domain_from_id
+from synapse.types import JsonDict, ReadReceipt, UserID, get_domain_from_id
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -137,7 +138,7 @@ class ReceiptsHandler(BaseHandler):
         return True
 
     async def received_client_receipt(
-        self, room_id: str, receipt_type: str, user_id: str, event_id: str
+        self, room_id: str, receipt_type: str, user_id: str, event_id: str, hidden: bool
     ) -> None:
         """Called when a client tells us a local user has read up to the given
         event_id in the room.
@@ -147,23 +148,67 @@ class ReceiptsHandler(BaseHandler):
             receipt_type=receipt_type,
             user_id=user_id,
             event_ids=[event_id],
-            data={"ts": int(self.clock.time_msec())},
+            data={"ts": int(self.clock.time_msec()), "hidden": hidden},
         )
 
         is_new = await self._handle_new_receipts([receipt])
         if not is_new:
             return
 
-        if self.federation_sender:
+        if self.federation_sender and not (
+            self.hs.config.experimental.msc2285_enabled and hidden
+        ):
             await self.federation_sender.send_read_receipt(receipt)
 
 
 class ReceiptEventSource:
     def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
+        self.config = hs.config
+
+    @staticmethod
+    def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]:
+        visible_events = []
+
+        # filter out hidden receipts the user shouldn't see
+        for event in events:
+            content = event.get("content", {})
+            new_event = event.copy()
+            new_event["content"] = {}
+
+            for event_id in content.keys():
+                event_content = content.get(event_id, {})
+                m_read = event_content.get("m.read", {})
+
+                # If m_read is missing copy over the original event_content as there is nothing to process here
+                if not m_read:
+                    new_event["content"][event_id] = event_content.copy()
+                    continue
+
+                new_users = {}
+                for rr_user_id, user_rr in m_read.items():
+                    hidden = user_rr.get("hidden", None)
+                    if hidden is not True or rr_user_id == user_id:
+                        new_users[rr_user_id] = user_rr.copy()
+                        # If hidden has a value replace hidden with the correct prefixed key
+                        if hidden is not None:
+                            new_users[rr_user_id].pop("hidden")
+                            new_users[rr_user_id][
+                                ReadReceiptEventFields.MSC2285_HIDDEN
+                            ] = hidden
+
+                # Set new users unless empty
+                if len(new_users.keys()) > 0:
+                    new_event["content"][event_id] = {"m.read": new_users}
+
+            # Append new_event to visible_events unless empty
+            if len(new_event["content"].keys()) > 0:
+                visible_events.append(new_event)
+
+        return visible_events
 
     async def get_new_events(
-        self, from_key: int, room_ids: List[str], **kwargs
+        self, from_key: int, room_ids: List[str], user: UserID, **kwargs
     ) -> Tuple[List[JsonDict], int]:
         from_key = int(from_key)
         to_key = self.get_current_key()
@@ -175,6 +220,9 @@ class ReceiptEventSource:
             room_ids, from_key=from_key, to_key=to_key
         )
 
+        if self.config.experimental.msc2285_enabled:
+            events = ReceiptEventSource.filter_out_hidden(events, user.to_string())
+
         return (events, to_key)
 
     async def get_new_events_as(

synapse/replication/tcp/client.py

@@ -393,6 +393,11 @@ class FederationSenderHandler:
             # we only want to send on receipts for our own users
             if not self._is_mine_id(receipt.user_id):
                 continue
+            if (
+                receipt.data.get("hidden", False)
+                and self._hs.config.experimental.msc2285_enabled
+            ):
+                continue
             receipt_info = ReadReceipt(
                 receipt.room_id,
                 receipt.receipt_type,

synapse/rest/client/v2_alpha/read_marker.py

@@ -14,6 +14,8 @@
 
 import logging
 
+from synapse.api.constants import ReadReceiptEventFields
+from synapse.api.errors import Codes, SynapseError
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 
 from ._base import client_patterns
@@ -37,14 +39,24 @@ class ReadMarkerRestServlet(RestServlet):
         await self.presence_handler.bump_presence_active_time(requester.user)
 
         body = parse_json_object_from_request(request)
-
         read_event_id = body.get("m.read", None)
+        hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False)
+
+        if not isinstance(hidden, bool):
+            raise SynapseError(
+                400,
+                "Param %s must be a boolean, if given"
+                % ReadReceiptEventFields.MSC2285_HIDDEN,
+                Codes.BAD_JSON,
+            )
+
         if read_event_id:
             await self.receipts_handler.received_client_receipt(
                 room_id,
                 "m.read",
                 user_id=requester.user.to_string(),
                 event_id=read_event_id,
+                hidden=hidden,
             )
 
         read_marker_event_id = body.get("m.fully_read", None)

synapse/rest/client/v2_alpha/receipts.py

@@ -14,8 +14,9 @@
 
 import logging
 
-from synapse.api.errors import SynapseError
+from synapse.api.constants import ReadReceiptEventFields
-from synapse.http.servlet import RestServlet
+from synapse.api.errors import Codes, SynapseError
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 
 from ._base import client_patterns
 
@@ -42,10 +43,25 @@ class ReceiptRestServlet(RestServlet):
         if receipt_type != "m.read":
             raise SynapseError(400, "Receipt type must be 'm.read'")
 
+        body = parse_json_object_from_request(request)
+        hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False)
+
+        if not isinstance(hidden, bool):
+            raise SynapseError(
+                400,
+                "Param %s must be a boolean, if given"
+                % ReadReceiptEventFields.MSC2285_HIDDEN,
+                Codes.BAD_JSON,
+            )
+
         await self.presence_handler.bump_presence_active_time(requester.user)
 
         await self.receipts_handler.received_client_receipt(
-            room_id, receipt_type, user_id=requester.user.to_string(), event_id=event_id
+            room_id,
+            receipt_type,
+            user_id=requester.user.to_string(),
+            event_id=event_id,
+            hidden=hidden,
         )
 
         return 200, {}

synapse/rest/client/versions.py

@@ -82,6 +82,8 @@ class VersionsRestServlet(RestServlet):
                     "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private,
                     # Supports the busy presence state described in MSC3026.
                     "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled,
+                    # Supports receiving hidden read receipts as per MSC2285
+                    "org.matrix.msc2285": self.config.experimental.msc2285_enabled,
                 },
             },
         )

tests/handlers/test_receipts.py

@@ -0,0 +1,294 @@
+# Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List
+
+from synapse.api.constants import ReadReceiptEventFields
+from synapse.types import JsonDict
+
+from tests import unittest
+
+
+class ReceiptsTestCase(unittest.HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
+        self.event_source = hs.get_event_sources().sources["receipt"]
+
+    # In the first param of _test_filters_hidden we use "hidden" instead of
+    # ReadReceiptEventFields.MSC2285_HIDDEN. We do this because we're mocking
+    # the data from the database which doesn't use the prefix
+
+    def test_filters_out_hidden_receipt(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [],
+        )
+
+    def test_does_not_filter_out_our_hidden_receipt(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$1435641916hfgh4394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@me:server.org": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$1435641916hfgh4394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@me:server.org": {
+                                    "ts": 1436451550453,
+                                    ReadReceiptEventFields.MSC2285_HIDDEN: True,
+                                },
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_filters_out_hidden_receipt_and_ignores_rest(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$1dgdgrd5641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                },
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$1dgdgrd5641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$14356419edgd14394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                            }
+                        },
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_handles_missing_content_of_m_read(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$14356419ggffg114394fHBLK:matrix.org": {"m.read": {}},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$14356419ggffg114394fHBLK:matrix.org": {"m.read": {}},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_handles_empty_event(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$143564gdfg6114394fHBLK:matrix.org": {},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$143564gdfg6114394fHBLK:matrix.org": {},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$14356419edgd14394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                },
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                },
+            ],
+            [
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def _test_filters_hidden(
+        self, events: List[JsonDict], expected_output: List[JsonDict]
+    ):
+        """Tests that the _filter_out_hidden returns the expected output"""
+        filtered_events = self.event_source.filter_out_hidden(events, "@me:server.org")
+        self.assertEquals(filtered_events, expected_output)

tests/rest/client/v2_alpha/test_sync.py

@@ -15,9 +15,14 @@
 import json
 
 import synapse.rest.admin
-from synapse.api.constants import EventContentFields, EventTypes, RelationTypes
+from synapse.api.constants import (
+    EventContentFields,
+    EventTypes,
+    ReadReceiptEventFields,
+    RelationTypes,
+)
 from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import knock, read_marker, sync
+from synapse.rest.client.v2_alpha import knock, read_marker, receipts, sync
 
 from tests import unittest
 from tests.federation.transport.test_knocking import (
@@ -368,6 +373,76 @@ class SyncKnockTestCase(
         )
 
 
+class ReadReceiptsTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+        receipts.register_servlets,
+        room.register_servlets,
+        sync.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.url = "/sync?since=%s"
+        self.next_batch = "s0"
+
+        # Register the first user
+        self.user_id = self.register_user("kermit", "monkey")
+        self.tok = self.login("kermit", "monkey")
+
+        # Create the room
+        self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok)
+
+        # Register the second user
+        self.user2 = self.register_user("kermit2", "monkey")
+        self.tok2 = self.login("kermit2", "monkey")
+
+        # Join the second user
+        self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2)
+
+    @override_config({"experimental_features": {"msc2285_enabled": True}})
+    def test_hidden_read_receipts(self):
+        # Send a message as the first user
+        res = self.helper.send(self.room_id, body="hello", tok=self.tok)
+
+        # Send a read receipt to tell the server the first user's message was read
+        body = json.dumps({ReadReceiptEventFields.MSC2285_HIDDEN: True}).encode("utf8")
+        channel = self.make_request(
+            "POST",
+            "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]),
+            body,
+            access_token=self.tok2,
+        )
+        self.assertEqual(channel.code, 200)
+
+        # Test that the first user can't see the other user's hidden read receipt
+        self.assertEqual(self._get_read_receipt(), None)
+
+    def _get_read_receipt(self):
+        """Syncs and returns the read receipt."""
+
+        # Checks if event is a read receipt
+        def is_read_receipt(event):
+            return event["type"] == "m.receipt"
+
+        # Sync
+        channel = self.make_request(
+            "GET",
+            self.url % self.next_batch,
+            access_token=self.tok,
+        )
+        self.assertEqual(channel.code, 200)
+
+        # Store the next batch for the next request.
+        self.next_batch = channel.json_body["next_batch"]
+
+        # Return the read receipt
+        ephemeral_events = channel.json_body["rooms"]["join"][self.room_id][
+            "ephemeral"
+        ]["events"]
+        return next(filter(is_read_receipt, ephemeral_events), None)
+
+
 class UnreadMessagesTestCase(unittest.HomeserverTestCase):
     servlets = [
         synapse.rest.admin.register_servlets,
@@ -375,6 +450,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase):
         read_marker.register_servlets,
         room.register_servlets,
         sync.register_servlets,
+        receipts.register_servlets,
     ]
 
     def prepare(self, reactor, clock, hs):
@@ -448,6 +524,23 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase):
         # Check that the unread counter is back to 0.
         self._check_unread_count(0)
 
+        # Check that hidden read receipts don't break unread counts
+        res = self.helper.send(self.room_id, "hello", tok=self.tok2)
+        self._check_unread_count(1)
+
+        # Send a read receipt to tell the server we've read the latest event.
+        body = json.dumps({ReadReceiptEventFields.MSC2285_HIDDEN: True}).encode("utf8")
+        channel = self.make_request(
+            "POST",
+            "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]),
+            body,
+            access_token=self.tok,
+        )
+        self.assertEqual(channel.code, 200, channel.json_body)
+
+        # Check that the unread counter is back to 0.
+        self._check_unread_count(0)
+
         # Check that room name changes increase the unread counter.
         self.helper.send_state(
             self.room_id,