forked from MirrorHub/synapse
Merge pull request #612 from matrix-org/erikj/cache_size
Add environment variable SYNAPSE_CACHE_FACTOR, default it to 0.1
This commit is contained in:
commit
2a78dac60d
7 changed files with 41 additions and 9 deletions
19
README.rst
19
README.rst
|
@ -565,4 +565,21 @@ sphinxcontrib-napoleon::
|
|||
Building internal API documentation::
|
||||
|
||||
python setup.py build_sphinx
|
||||
|
||||
|
||||
|
||||
|
||||
Halp!! Synapse eats all my RAM!
|
||||
===============================
|
||||
|
||||
Synapse's architecture is quite RAM hungry currently - we deliberately
|
||||
cache a lot of recent room data and metadata in RAM in order to speed up
|
||||
common requests. We'll improve this in future, but for now the easiest
|
||||
way to either reduce the RAM usage (at the risk of slowing things down)
|
||||
is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
|
||||
variable. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1.0 will max out
|
||||
at around 3-4GB of resident memory - this is what we currently run the
|
||||
matrix.org on. The default setting is currently 0.1, which is probably
|
||||
around a ~700MB footprint. You can dial it down further to 0.02 if
|
||||
desired, which targets roughly ~512MB. Conversely you can dial it up if
|
||||
you need performance for lots of users and have a box with a lot of RAM.
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
: ${WORKSPACE:="$(pwd)"}
|
||||
|
||||
export PYTHONDONTWRITEBYTECODE=yep
|
||||
export SYNAPSE_CACHE_FACTOR=1
|
||||
|
||||
# Output test results as junit xml
|
||||
export TRIAL_FLAGS="--reporter=subunit"
|
||||
|
|
|
@ -130,6 +130,10 @@ class PresenceHandler(BaseHandler):
|
|||
for state in active_presence
|
||||
}
|
||||
|
||||
metrics.register_callback(
|
||||
"user_to_current_state_size", lambda: len(self.user_to_current_state)
|
||||
)
|
||||
|
||||
now = self.clock.time_msec()
|
||||
for state in active_presence:
|
||||
self.wheel_timer.insert(
|
||||
|
|
|
@ -36,8 +36,6 @@ class ReceiptsHandler(BaseHandler):
|
|||
)
|
||||
self.clock = self.hs.get_clock()
|
||||
|
||||
self._receipt_cache = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def received_client_receipt(self, room_id, receipt_type, user_id,
|
||||
event_id):
|
||||
|
|
|
@ -28,6 +28,7 @@ from twisted.internet import defer
|
|||
|
||||
from collections import OrderedDict
|
||||
|
||||
import os
|
||||
import functools
|
||||
import inspect
|
||||
import threading
|
||||
|
@ -38,6 +39,9 @@ logger = logging.getLogger(__name__)
|
|||
_CacheSentinel = object()
|
||||
|
||||
|
||||
CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
|
||||
|
||||
|
||||
class Cache(object):
|
||||
|
||||
def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False):
|
||||
|
@ -140,6 +144,8 @@ class CacheDescriptor(object):
|
|||
"""
|
||||
def __init__(self, orig, max_entries=1000, num_args=1, lru=True, tree=False,
|
||||
inlineCallbacks=False):
|
||||
max_entries = int(max_entries * CACHE_SIZE_FACTOR)
|
||||
|
||||
self.orig = orig
|
||||
|
||||
if inlineCallbacks:
|
||||
|
|
|
@ -18,11 +18,15 @@ from synapse.util.caches import cache_counter, caches_by_name
|
|||
|
||||
from blist import sorteddict
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
|
||||
|
||||
|
||||
class StreamChangeCache(object):
|
||||
"""Keeps track of the stream positions of the latest change in a set of entities.
|
||||
|
||||
|
@ -33,7 +37,7 @@ class StreamChangeCache(object):
|
|||
old then the cache will simply return all given entities.
|
||||
"""
|
||||
def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}):
|
||||
self._max_size = max_size
|
||||
self._max_size = int(max_size * CACHE_SIZE_FACTOR)
|
||||
self._entity_to_key = {}
|
||||
self._cache = sorteddict()
|
||||
self._earliest_known_stream_pos = current_stream_pos
|
||||
|
|
|
@ -35,7 +35,8 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
|
|||
def setUp(self):
|
||||
self.as_yaml_files = []
|
||||
config = Mock(
|
||||
app_service_config_files=self.as_yaml_files
|
||||
app_service_config_files=self.as_yaml_files,
|
||||
event_cache_size=1,
|
||||
)
|
||||
hs = yield setup_test_homeserver(config=config)
|
||||
|
||||
|
@ -109,7 +110,8 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
|
|||
self.as_yaml_files = []
|
||||
|
||||
config = Mock(
|
||||
app_service_config_files=self.as_yaml_files
|
||||
app_service_config_files=self.as_yaml_files,
|
||||
event_cache_size=1,
|
||||
)
|
||||
hs = yield setup_test_homeserver(config=config)
|
||||
self.db_pool = hs.get_db_pool()
|
||||
|
@ -438,7 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
|
|||
f1 = self._write_config(suffix="1")
|
||||
f2 = self._write_config(suffix="2")
|
||||
|
||||
config = Mock(app_service_config_files=[f1, f2])
|
||||
config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
|
||||
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
||||
|
||||
ApplicationServiceStore(hs)
|
||||
|
@ -448,7 +450,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
|
|||
f1 = self._write_config(id="id", suffix="1")
|
||||
f2 = self._write_config(id="id", suffix="2")
|
||||
|
||||
config = Mock(app_service_config_files=[f1, f2])
|
||||
config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
|
||||
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
||||
|
||||
with self.assertRaises(ConfigError) as cm:
|
||||
|
@ -464,7 +466,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
|
|||
f1 = self._write_config(as_token="as_token", suffix="1")
|
||||
f2 = self._write_config(as_token="as_token", suffix="2")
|
||||
|
||||
config = Mock(app_service_config_files=[f1, f2])
|
||||
config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
|
||||
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
||||
|
||||
with self.assertRaises(ConfigError) as cm:
|
||||
|
|
Loading…
Reference in a new issue