diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index c55221c6d..fa903c251 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -67,6 +67,8 @@ class TypingNotificationHandler(BaseHandler):
         if target_user != auth_user:
             raise AuthError(400, "Cannot set another user's typing state")
 
+        yield self.auth.check_joined_room(room_id, target_user.to_string())
+
         logger.debug(
             "%s has started typing in %s", target_user.to_string(), room_id
         )
@@ -102,6 +104,8 @@ class TypingNotificationHandler(BaseHandler):
         if target_user != auth_user:
             raise AuthError(400, "Cannot set another user's typing state")
 
+        yield self.auth.check_joined_room(room_id, target_user.to_string())
+
         logger.debug(
             "%s has stopped typing in %s", target_user.to_string(), room_id
         )
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index b858f9632..bc19db8df 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -22,6 +22,7 @@ import json
 
 from ..utils import MockHttpResource, MockClock, DeferredMockCallable, MockKey
 
+from synapse.api.errors import AuthError
 from synapse.server import HomeServer
 from synapse.handlers.typing import TypingNotificationHandler
 
@@ -68,7 +69,10 @@ class TypingNotificationsTestCase(unittest.TestCase):
         mock_notifier = Mock(spec=["on_new_user_event"])
         self.on_new_user_event = mock_notifier.on_new_user_event
 
+        self.auth = Mock(spec=[])
+
         hs = HomeServer("test",
+                auth=self.auth,
                 clock=self.clock,
                 db_pool=None,
                 datastore=Mock(spec=[
@@ -142,6 +146,12 @@ class TypingNotificationsTestCase(unittest.TestCase):
         self.room_member_handler.fetch_room_distributions_into = (
                 fetch_room_distributions_into)
 
+        def check_joined_room(room_id, user_id):
+            if user_id not in [u.to_string() for u in self.room_members]:
+                raise AuthError(401, "User is not in the room")
+
+        self.auth.check_joined_room = check_joined_room
+
         # Some local users to test with
         self.u_apple = hs.parse_userid("@apple:test")
         self.u_banana = hs.parse_userid("@banana:test")