maxmustermann/synapse
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add some randomness to the high-cpu backoff hack

Browse source
This commit is contained in:
Richard van der Hoff 2019-10-11 09:15:56 +01:00
parent b852a8247d
commit 15b2a50817
2 changed files with 12 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
synapse/federation/sender/__init__.py
View file

@ -165,8 +165,8 @@ class FederationSender(object):
and not self._transaction_manager.deprioritise_transmission
):
logger.warning(
"Event processing loop is getting behind: deprioritising "
"transaction transmission"
"Event queue is getting behind: deprioritising transaction "
"transmission"
)
self._transaction_manager.deprioritise_transmission = True

12
synapse/federation/sender/per_destination_queue.py
View file

@ -15,6 +15,7 @@
# limitations under the License.
import datetime
import logging
import random
from prometheus_client import Counter
@ -36,6 +37,8 @@ from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
# This is defined in the Matrix spec and enforced by the receiver.
MAX_EDUS_PER_TRANSACTION = 100
DEPRIORITISE_SLEEP_TIME = 10
logger = logging.getLogger(__name__)
@ -191,8 +194,13 @@ class PerDestinationQueue(object):
while True:
if self._transaction_manager.deprioritise_transmission:
# if the event-processing loop has got behind, sleep to give it
# a chance to catch up
yield self._clock.sleep(2)
# a chance to catch up. Add some randomness so that the transmitters
# don't all wake up in sync.
sleeptime = random.uniform(
DEPRIORITISE_SLEEP_TIME, DEPRIORITISE_SLEEP_TIME * 2
)
logger.info("TX [%s]: sleeping for %f seconds", sleeptime)
yield self._clock.sleep(sleeptime)
# We have to keep 2 free slots for presence and rr_edus
limit = MAX_EDUS_PER_TRANSACTION - 2