diff --git a/changelog.d/5342.bugfix b/changelog.d/5342.bugfix
new file mode 100644
index 000000000..66a307629
--- /dev/null
+++ b/changelog.d/5342.bugfix
@@ -0,0 +1 @@
+Fix failure when fetching batches of events during backfill, etc.
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 916ff487c..d55960538 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -279,6 +279,7 @@ class FederationClient(FederationBase):
                     "Failed to get PDU %s from %s because %s",
                     event_id, destination, e,
                 )
+                continue
             except NotRetryingDestination as e:
                 logger.info(str(e))
                 continue