diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 8530f9078..b70d6a5e1 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -405,11 +405,12 @@ userLoop:
 			case *fedsenderapi.FederationClientError:
 				if e.RetryAfter > 0 {
 					waitTime = e.RetryAfter
-					break userLoop
 				} else if e.Blacklisted {
 					waitTime = time.Hour * 8
-					break userLoop
+				} else {
+					waitTime = time.Hour
 				}
+				break userLoop
 			case net.Error:
 				// Use the default waitTime, if it's a timeout.
 				// It probably doesn't make sense to try further users.