Merge pull request #4008 from matrix-org/erikj/log_looping_exceptions

Log looping call exceptions
This commit is contained in:
Erik Johnston 2018-10-09 10:12:10 +01:00 committed by GitHub
commit 9eb1a79100
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 36 additions and 13 deletions

1
changelog.d/4008.misc Normal file
View file

@ -0,0 +1 @@
Log exceptions in looping calls

View file

@ -28,6 +28,7 @@ from synapse.metrics import (
event_processing_loop_room_count, event_processing_loop_room_count,
) )
from synapse.metrics.background_process_metrics import run_as_background_process from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.util import log_failure
from synapse.util.logcontext import make_deferred_yieldable, run_in_background from synapse.util.logcontext import make_deferred_yieldable, run_in_background
from synapse.util.metrics import Measure from synapse.util.metrics import Measure
@ -36,17 +37,6 @@ logger = logging.getLogger(__name__)
events_processed_counter = Counter("synapse_handlers_appservice_events_processed", "") events_processed_counter = Counter("synapse_handlers_appservice_events_processed", "")
def log_failure(failure):
logger.error(
"Application Services Failure",
exc_info=(
failure.type,
failure.value,
failure.getTracebackObject()
)
)
class ApplicationServicesHandler(object): class ApplicationServicesHandler(object):
def __init__(self, hs): def __init__(self, hs):
@ -112,7 +102,10 @@ class ApplicationServicesHandler(object):
if not self.started_scheduler: if not self.started_scheduler:
def start_scheduler(): def start_scheduler():
return self.scheduler.start().addErrback(log_failure) return self.scheduler.start().addErrback(
log_failure, "Application Services Failure",
)
run_as_background_process("as_scheduler", start_scheduler) run_as_background_process("as_scheduler", start_scheduler)
self.started_scheduler = True self.started_scheduler = True

View file

@ -68,7 +68,10 @@ class Clock(object):
""" """
call = task.LoopingCall(f) call = task.LoopingCall(f)
call.clock = self._reactor call.clock = self._reactor
call.start(msec / 1000.0, now=False) d = call.start(msec / 1000.0, now=False)
d.addErrback(
log_failure, "Looping call died", consumeErrors=False,
)
return call return call
def call_later(self, delay, callback, *args, **kwargs): def call_later(self, delay, callback, *args, **kwargs):
@ -109,3 +112,29 @@ def batch_iter(iterable, size):
sourceiter = iter(iterable) sourceiter = iter(iterable)
# call islice until it returns an empty tuple # call islice until it returns an empty tuple
return iter(lambda: tuple(islice(sourceiter, size)), ()) return iter(lambda: tuple(islice(sourceiter, size)), ())
def log_failure(failure, msg, consumeErrors=True):
"""Creates a function suitable for passing to `Deferred.addErrback` that
logs any failures that occur.
Args:
msg (str): Message to log
consumeErrors (bool): If true consumes the failure, otherwise passes
on down the callback chain
Returns:
func(Failure)
"""
logger.error(
msg,
exc_info=(
failure.type,
failure.value,
failure.getTracebackObject()
)
)
if not consumeErrors:
return failure