Merge branch 'rav/fix_cookie_path' into matrix-org-hotfixes

Merge the cookie fix to hotfixes

changelog.d/9400.feature

@@ -0,0 +1 @@
+Add the shadow-banning status to the display user admin API.

changelog.d/9408.misc

@@ -0,0 +1 @@
+Clean up an unused method in the presence handler code.

changelog.d/9425.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug in the deduplication of old presence, resulting in no deduplication.

docs/admin_api/user_admin_api.rst

@@ -29,8 +29,9 @@ It returns a JSON body like the following:
             }
         ],
         "avatar_url": "<avatar_url>",
-        "admin": false,
-        "deactivated": false,
+        "admin": 0,
+        "deactivated": 0,
+        "shadow_banned": 0,
         "password_hash": "$2b$12$p9B4GkqYdRTPGD",
         "creation_ts": 1560432506,
         "appservice_id": null,
@@ -150,6 +151,7 @@ A JSON body is returned with the following shape:
                 "admin": 0,
                 "user_type": null,
                 "deactivated": 0,
+                "shadow_banned": 0,
                 "displayname": "<User One>",
                 "avatar_url": null
             }, {
@@ -158,6 +160,7 @@ A JSON body is returned with the following shape:
                 "admin": 1,
                 "user_type": null,
                 "deactivated": 0,
+                "shadow_banned": 0,
                 "displayname": "<User Two>",
                 "avatar_url": "<avatar_url>"
             }
@@ -262,7 +265,7 @@ The following actions are performed when deactivating an user:
 - Reject all pending invites
 - Remove all account validity information related to the user
 
-The following additional actions are performed during deactivation if``erase``
+The following additional actions are performed during deactivation if ``erase``
 is set to ``true``:
 
 - Remove the user's display name

docs/workers.md

@@ -276,7 +276,8 @@ using):
 
 Ensure that all SSO logins go to a single process.
 For multiple workers not handling the SSO endpoints properly, see
-[#7530](https://github.com/matrix-org/synapse/issues/7530).
+[#7530](https://github.com/matrix-org/synapse/issues/7530) and 
+[#9427](https://github.com/matrix-org/synapse/issues/9427).
 
 Note that a HTTP listener with `client` and `federation` resources must be
 configured in the `worker_listeners` option in the worker config.

synapse/handlers/presence.py

@@ -349,10 +349,13 @@ class PresenceHandler(BasePresenceHandler):
                 [self.user_to_current_state[user_id] for user_id in unpersisted]
             )
 
-    async def _update_states(self, new_states):
+    async def _update_states(self, new_states: Iterable[UserPresenceState]) -> None:
         """Updates presence of users. Sets the appropriate timeouts. Pokes
         the notifier and federation if and only if the changed presence state
         should be sent to clients/servers.
+
+        Args:
+            new_states: The new user presence state updates to process.
         """
         now = self.clock.time_msec()
 
@@ -368,7 +371,7 @@ class PresenceHandler(BasePresenceHandler):
             new_states_dict = {}
             for new_state in new_states:
                 new_states_dict[new_state.user_id] = new_state
-            new_state = new_states_dict.values()
+            new_states = new_states_dict.values()
 
             for new_state in new_states:
                 user_id = new_state.user_id
@@ -657,17 +660,6 @@ class PresenceHandler(BasePresenceHandler):
 
         self._push_to_remotes(states)
 
-    async def notify_for_states(self, state, stream_id):
-        parties = await get_interested_parties(self.store, [state])
-        room_ids_to_states, users_to_states = parties
-
-        self.notifier.on_new_event(
-            "presence_key",
-            stream_id,
-            rooms=room_ids_to_states.keys(),
-            users=[UserID.from_string(u) for u in users_to_states],
-        )
-
     def _push_to_remotes(self, states):
         """Sends state updates to remote servers.
 

synapse/rest/client/v1/login.py

@@ -354,6 +354,7 @@ class SsoRedirectServlet(RestServlet):
             hs.get_oidc_handler()
         self._sso_handler = hs.get_sso_handler()
         self._msc2858_enabled = hs.config.experimental.msc2858_enabled
+        self._public_baseurl = hs.config.public_baseurl
 
     def register(self, http_server: HttpServer) -> None:
         super().register(http_server)
@@ -373,6 +374,28 @@ class SsoRedirectServlet(RestServlet):
     async def on_GET(
         self, request: SynapseRequest, idp_id: Optional[str] = None
     ) -> None:
+        if not self._public_baseurl:
+            raise SynapseError(400, "SSO requires a valid public_baseurl")
+
+        # if this isn't the expected hostname, redirect to the right one, so that we
+        # get our cookies back.
+        requested_uri = b"%s://%s%s" % (
+            b"https" if request.isSecure() else b"http",
+            request.getHeader(b"host"),
+            request.uri,
+        )
+        baseurl_bytes = self._public_baseurl.encode("utf-8")
+        if not requested_uri.startswith(baseurl_bytes):
+            i = requested_uri.index(b"/_matrix")
+            new_uri = baseurl_bytes[:-1] + requested_uri[i:]
+            logger.info(
+                "Requested URI %s is not canonical: redirecting to %s",
+                requested_uri.decode("utf-8", errors="replace"),
+                new_uri.decode("utf-8", errors="replace"),
+            )
+            request.redirect(new_uri)
+            finish_request(request)
+
         client_redirect_url = parse_string(
             request, "redirectUrl", required=True, encoding=None
         )

synapse/storage/databases/main/__init__.py

@@ -340,7 +340,7 @@ class DataStore(
             count = txn.fetchone()[0]
 
             sql = (
-                "SELECT name, user_type, is_guest, admin, deactivated, displayname, avatar_url "
+                "SELECT name, user_type, is_guest, admin, deactivated, shadow_banned, displayname, avatar_url "
                 + sql_base
                 + " ORDER BY u.name LIMIT ? OFFSET ?"
             )

synapse/storage/databases/main/registration.py

@@ -113,6 +113,7 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore):
                 "creation_ts",
                 "user_type",
                 "deactivated",
+                "shadow_banned",
             ],
             allow_none=True,
             desc="get_user_by_id",
@@ -372,23 +373,25 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore):
         """
 
         def set_shadow_banned_txn(txn):
+            user_id = user.to_string()
             self.db_pool.simple_update_one_txn(
                 txn,
                 table="users",
-                keyvalues={"name": user.to_string()},
+                keyvalues={"name": user_id},
                 updatevalues={"shadow_banned": shadow_banned},
             )
             # In order for this to apply immediately, clear the cache for this user.
             tokens = self.db_pool.simple_select_onecol_txn(
                 txn,
                 table="access_tokens",
-                keyvalues={"user_id": user.to_string()},
+                keyvalues={"user_id": user_id},
                 retcol="token",
             )
             for token in tokens:
                 self._invalidate_cache_and_stream(
                     txn, self.get_user_by_access_token, (token,)
                 )
+            self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
 
         await self.db_pool.runInteraction("set_shadow_banned", set_shadow_banned_txn)
 

tests/rest/admin/test_user.py

@@ -769,6 +769,7 @@ class UsersListTestCase(unittest.HomeserverTestCase):
             self.assertIn("admin", u)
             self.assertIn("user_type", u)
             self.assertIn("deactivated", u)
+            self.assertIn("shadow_banned", u)
             self.assertIn("displayname", u)
             self.assertIn("avatar_url", u)
 
@@ -1146,6 +1147,7 @@ class UserRestTestCase(unittest.HomeserverTestCase):
         self.assertEqual(False, channel.json_body["admin"])
         self.assertEqual(False, channel.json_body["is_guest"])
         self.assertEqual(False, channel.json_body["deactivated"])
+        self.assertEqual(False, channel.json_body["shadow_banned"])
         self.assertEqual("mxc://fibble/wibble", channel.json_body["avatar_url"])
 
     @override_config(

tests/storage/test_registration.py

@@ -52,6 +52,7 @@ class RegistrationStoreTestCase(unittest.TestCase):
                 "creation_ts": 1000,
                 "user_type": None,
                 "deactivated": 0,
+                "shadow_banned": 0,
             },
             (yield defer.ensureDeferred(self.store.get_user_by_id(self.user_id))),
         )