From d02e41dcb299c7588bc9fa26bd0b5321fd7c5751 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Aug 2019 13:41:27 +0100 Subject: [PATCH 1/4] Handle pusher being deleted during processing. Instead of throwing a StoreError lets break out of processing loop and mark the pusher as stopped. --- synapse/push/emailpusher.py | 19 +++++++++++++------ synapse/push/httppusher.py | 27 ++++++++++++++++++++------- synapse/storage/pusher.py | 30 ++++++++++++++++++++++-------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 424ffa8b6..f688d4152 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -234,13 +234,20 @@ class EmailPusher(object): return self.last_stream_ordering = last_stream_ordering - yield self.store.update_pusher_last_stream_ordering_and_success( - self.app_id, - self.email, - self.user_id, - last_stream_ordering, - self.clock.time_msec(), + pusher_still_exists = ( + yield self.store.update_pusher_last_stream_ordering_and_success( + self.app_id, + self.email, + self.user_id, + last_stream_ordering, + self.clock.time_msec(), + ) ) + if not pusher_still_exists: + # The pusher has been deleted while we were processing, so + # lets just stop and return. + self.on_stop() + return def seconds_until(self, ts_msec): secs = (ts_msec - self.clock.time_msec()) / 1000 diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 5b15b0dbe..bd5d53af9 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -199,13 +199,21 @@ class HttpPusher(object): http_push_processed_counter.inc() self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC self.last_stream_ordering = push_action["stream_ordering"] - yield self.store.update_pusher_last_stream_ordering_and_success( - self.app_id, - self.pushkey, - self.user_id, - self.last_stream_ordering, - self.clock.time_msec(), + pusher_still_exists = ( + yield self.store.update_pusher_last_stream_ordering_and_success( + self.app_id, + self.pushkey, + self.user_id, + self.last_stream_ordering, + self.clock.time_msec(), + ) ) + if not pusher_still_exists: + # The pusher has been deleted while we were processing, so + # lets just stop and return. + self.on_stop() + return + if self.failing_since: self.failing_since = None yield self.store.update_pusher_failing_since( @@ -234,12 +242,17 @@ class HttpPusher(object): ) self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC self.last_stream_ordering = push_action["stream_ordering"] - yield self.store.update_pusher_last_stream_ordering( + pusher_still_exists = yield self.store.update_pusher_last_stream_ordering( self.app_id, self.pushkey, self.user_id, self.last_stream_ordering, ) + if not pusher_still_exists: + # The pusher has been deleted while we were processing, so + # lets just stop and return. + self.on_stop() + return self.failing_since = None yield self.store.update_pusher_failing_since( diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index be3d4d9de..888035fe8 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -308,22 +308,36 @@ class PusherStore(PusherWorkerStore): def update_pusher_last_stream_ordering_and_success( self, app_id, pushkey, user_id, last_stream_ordering, last_success ): - yield self._simple_update_one( - "pushers", - {"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, - { + """Update the last stream ordering position we've processed up to for + the given pusher. + + Args: + app_id (str) + pushkey (str) + last_stream_ordering (int) + last_success (int) + + Returns: + Deferred[bool]: Whether the pusher stil exists or not. + """ + updated = yield self._simple_update( + table="pushers", + keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, + updatevalues={ "last_stream_ordering": last_stream_ordering, "last_success": last_success, }, desc="update_pusher_last_stream_ordering_and_success", ) + return bool(updated) + @defer.inlineCallbacks def update_pusher_failing_since(self, app_id, pushkey, user_id, failing_since): - yield self._simple_update_one( - "pushers", - {"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, - {"failing_since": failing_since}, + yield self._simple_update( + table="pushers", + keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, + updatevalues={"failing_since": failing_since}, desc="update_pusher_failing_since", ) From 312cc48e2ba8bfb703ed9f55be76714179723f67 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Aug 2019 13:45:09 +0100 Subject: [PATCH 2/4] Newsfile --- changelog.d/5809.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5809.misc diff --git a/changelog.d/5809.misc b/changelog.d/5809.misc new file mode 100644 index 000000000..82a812480 --- /dev/null +++ b/changelog.d/5809.misc @@ -0,0 +1 @@ +Handle pusher being deleted during processing rather than logging an exception. From fb3469f53ac86c4771caa9fdfc946eaa298977b9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Aug 2019 10:17:23 +0100 Subject: [PATCH 3/4] Clarify docstring Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- synapse/storage/pusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 888035fe8..b431d24b8 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -318,7 +318,7 @@ class PusherStore(PusherWorkerStore): last_success (int) Returns: - Deferred[bool]: Whether the pusher stil exists or not. + Deferred[bool]: True if the pusher still exists; False if it has been deleted. """ updated = yield self._simple_update( table="pushers", From 96bdd661b88f373826ea25baa94199cf2b8d25f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Aug 2019 12:50:36 +0100 Subject: [PATCH 4/4] Remove redundant return --- synapse/push/emailpusher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index f688d4152..42e5b0c0a 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -247,7 +247,6 @@ class EmailPusher(object): # The pusher has been deleted while we were processing, so # lets just stop and return. self.on_stop() - return def seconds_until(self, ts_msec): secs = (ts_msec - self.clock.time_msec()) / 1000