From 0df618f81324a9f2cc2b064c33bd9dfc22391402 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 10 Jun 2020 18:27:49 +0100
Subject: [PATCH] Take out a lock before modifying _CACHES (#7663)

This should fix #7610.
---
 changelog.d/7663.bugfix |  1 +
 synapse/config/cache.py | 20 +++++++++++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/7663.bugfix

diff --git a/changelog.d/7663.bugfix b/changelog.d/7663.bugfix
new file mode 100644
index 000000000..b58316b34
--- /dev/null
+++ b/changelog.d/7663.bugfix
@@ -0,0 +1 @@
+Fix intermittent exception during startup, introduced in Synapse 1.14.0.
diff --git a/synapse/config/cache.py b/synapse/config/cache.py
index 067253879..aff5b21ab 100644
--- a/synapse/config/cache.py
+++ b/synapse/config/cache.py
@@ -15,6 +15,7 @@
 
 import os
 import re
+import threading
 from typing import Callable, Dict
 
 from ._base import Config, ConfigError
@@ -25,6 +26,9 @@ _CACHE_PREFIX = "SYNAPSE_CACHE_FACTOR"
 # Map from canonicalised cache name to cache.
 _CACHES = {}
 
+# a lock on the contents of _CACHES
+_CACHES_LOCK = threading.Lock()
+
 _DEFAULT_FACTOR_SIZE = 0.5
 _DEFAULT_EVENT_CACHE_SIZE = "10K"
 
@@ -66,7 +70,10 @@ def add_resizable_cache(cache_name: str, cache_resize_callback: Callable):
     # Some caches have '*' in them which we strip out.
     cache_name = _canonicalise_cache_name(cache_name)
 
-    _CACHES[cache_name] = cache_resize_callback
+    # sometimes caches are initialised from background threads, so we need to make
+    # sure we don't conflict with another thread running a resize operation
+    with _CACHES_LOCK:
+        _CACHES[cache_name] = cache_resize_callback
 
     # Ensure all loaded caches are sized appropriately
     #
@@ -87,7 +94,8 @@ class CacheConfig(Config):
             os.environ.get(_CACHE_PREFIX, _DEFAULT_FACTOR_SIZE)
         )
         properties.resize_all_caches_func = None
-        _CACHES.clear()
+        with _CACHES_LOCK:
+            _CACHES.clear()
 
     def generate_config_section(self, **kwargs):
         return """\
@@ -193,6 +201,8 @@ class CacheConfig(Config):
         For each cache, run the mapped callback function with either
         a specific cache factor or the default, global one.
         """
-        for cache_name, callback in _CACHES.items():
-            new_factor = self.cache_factors.get(cache_name, self.global_factor)
-            callback(new_factor)
+        # block other threads from modifying _CACHES while we iterate it.
+        with _CACHES_LOCK:
+            for cache_name, callback in _CACHES.items():
+                new_factor = self.cache_factors.get(cache_name, self.global_factor)
+                callback(new_factor)