Merge remote-tracking branch 'origin/develop' into matrix-org-hotfixes

changelog.d/7703.misc

@@ -0,0 +1 @@
+Make Tox actions work on Debian 10.

changelog.d/7738.misc

@@ -0,0 +1 @@
+Move `flake8` to the end of `scripts-dev/lint.sh` as it takes the longest and could cause the script to exit early.

changelog.d/7746.bugfix

@@ -0,0 +1 @@
+Fix large state resolutions from stalling Synapse for seconds at a time.

scripts-dev/lint.sh

@@ -2,8 +2,8 @@
 #
 # Runs linting scripts over the local Synapse checkout
 # isort - sorts import statements
-# flake8 - lints and finds mistakes
 # black - opinionated code formatter
+# flake8 - lints and finds mistakes
 
 set -e
 
@@ -16,6 +16,6 @@ fi
 
 echo "Linting these locations: $files"
 isort -y -rc $files
-flake8 $files
 python3 -m black $files
 ./scripts-dev/config-lint.sh
+flake8 $files

synapse/state/v2.py

@@ -126,6 +126,7 @@ def resolve_events_with_store(
 
     # Now sequentially auth each one
     resolved_state = yield _iterative_auth_checks(
+        clock,
         room_id,
         room_version,
         sorted_power_events,
@@ -154,6 +155,7 @@ def resolve_events_with_store(
     logger.debug("resolving remaining events")
 
     resolved_state = yield _iterative_auth_checks(
+        clock,
         room_id,
         room_version,
         leftover_events,
@@ -378,12 +380,13 @@ def _reverse_topological_power_sort(
 
 @defer.inlineCallbacks
 def _iterative_auth_checks(
-    room_id, room_version, event_ids, base_state, event_map, state_res_store
+    clock, room_id, room_version, event_ids, base_state, event_map, state_res_store
 ):
     """Sequentially apply auth checks to each event in given list, updating the
     state as it goes along.
 
     Args:
+        clock (Clock)
         room_id (str)
         room_version (str)
         event_ids (list[str]): Ordered list of events to apply auth checks to
@@ -397,7 +400,7 @@ def _iterative_auth_checks(
     resolved_state = base_state.copy()
     room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
 
-    for event_id in event_ids:
+    for idx, event_id in enumerate(event_ids, start=1):
         event = event_map[event_id]
 
         auth_events = {}
@@ -435,6 +438,11 @@ def _iterative_auth_checks(
         except AuthError:
             pass
 
+        # We yield occasionally when we're working with large data sets to
+        # ensure that we don't block the reactor loop for too long.
+        if idx % _YIELD_AFTER_ITERATIONS == 0:
+            yield clock.sleep(0)
+
     return resolved_state
 
 

tox.ini

@@ -2,7 +2,6 @@
 envlist = packaging, py35, py36, py37, py38, check_codestyle, check_isort
 
 [base]
-basepython = python3.7
 deps =
     mock
     python-subunit
@@ -120,11 +119,11 @@ commands =
 
 [testenv:check_codestyle]
 skip_install = True
-basepython = python3.6
 deps =
     flake8
     flake8-comprehensions
-    black==19.10b0  # We pin so that our tests don't start failing on new releases of black.
+    # We pin so that our tests don't start failing on new releases of black.
+    black==19.10b0
 commands =
     python -m black --check --diff .
     /bin/sh -c "flake8 synapse tests scripts scripts-dev synctl {env:PEP8SUFFIX:}"
@@ -140,7 +139,6 @@ skip_install = True
 deps = towncrier>=18.6.0rc1
 commands =
    python -m towncrier.check --compare-with=origin/develop
-basepython = python3.6
 
 [testenv:check-sampleconfig]
 commands = {toxinidir}/scripts-dev/generate_sample_config --check