diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index f929bcf85..571eacd34 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -720,14 +720,24 @@ class PresenceHandler(BaseHandler):
                 statuscache=statuscache,
             )
 
+            user_id = user.to_string()
+
             if state["presence"] == PresenceState.OFFLINE:
                 self._remote_offline_serials.insert(
                     0,
-                    (self._user_cachemap_latest_serial, set([user.to_string()]))
+                    (self._user_cachemap_latest_serial, set([user_id]))
                 )
                 while len(self._remote_offline_serials) > MAX_OFFLINE_SERIALS:
                     self._remote_offline_serials.pop()  # remove the oldest
                 del self._user_cachemap[user]
+            else:
+                # Remove the user from remote_offline_serials now that they're
+                # no longer offline
+                for idx, elem in enumerate(self._remote_offline_serials):
+                    (_, user_ids) = elem
+                    user_ids.discard(user_id)
+                    if not user_ids:
+                        self._remote_offline_serials.pop(idx)
 
         for poll in content.get("poll", []):
             user = UserID.from_string(poll)
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index bb497b6f0..9f5580c09 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -916,6 +916,33 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase):
             ]
         )
 
+        yield self.mock_federation_resource.trigger("PUT",
+            "/_matrix/federation/v1/send/1000001/",
+            _make_edu_json("elsewhere", "m.presence",
+                content={
+                    "push": [
+                        {"user_id": "@potato:remote",
+                         "presence": "online"},
+                    ],
+                }
+            )
+        )
+
+        self.assertEquals(self.event_source.get_current_key(), 2)
+
+        (events, _) = yield self.event_source.get_new_events_for_user(
+            self.u_apple, 0, None
+        )
+        self.assertEquals(events,
+            [
+                {"type": "m.presence",
+                 "content": {
+                     "user_id": "@potato:remote",
+                     "presence": ONLINE,
+                }}
+            ]
+        )
+
     @defer.inlineCallbacks
     def test_join_room_local(self):
         self.room_members = [self.u_apple, self.u_banana]