forked from MirrorHub/synapse
Option to suppress resource exceeded alerting (#6173)
The expected use case is to suppress MAU limiting on small instances
This commit is contained in:
parent
92e88a71d3
commit
2794b79052
8 changed files with 160 additions and 46 deletions
1
changelog.d/6173.feature
Normal file
1
changelog.d/6173.feature
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Add config option to suppress client side resource limit alerting.
|
|
@ -241,7 +241,6 @@ listeners:
|
||||||
#
|
#
|
||||||
#hs_disabled: false
|
#hs_disabled: false
|
||||||
#hs_disabled_message: 'Human readable reason for why the HS is blocked'
|
#hs_disabled_message: 'Human readable reason for why the HS is blocked'
|
||||||
#hs_disabled_limit_type: 'error code(str), to help clients decode reason'
|
|
||||||
|
|
||||||
# Monthly Active User Blocking
|
# Monthly Active User Blocking
|
||||||
#
|
#
|
||||||
|
@ -261,9 +260,16 @@ listeners:
|
||||||
# sign up in a short space of time never to return after their initial
|
# sign up in a short space of time never to return after their initial
|
||||||
# session.
|
# session.
|
||||||
#
|
#
|
||||||
|
# 'mau_limit_alerting' is a means of limiting client side alerting
|
||||||
|
# should the mau limit be reached. This is useful for small instances
|
||||||
|
# where the admin has 5 mau seats (say) for 5 specific people and no
|
||||||
|
# interest increasing the mau limit further. Defaults to True, which
|
||||||
|
# means that alerting is enabled
|
||||||
|
#
|
||||||
#limit_usage_by_mau: false
|
#limit_usage_by_mau: false
|
||||||
#max_mau_value: 50
|
#max_mau_value: 50
|
||||||
#mau_trial_days: 2
|
#mau_trial_days: 2
|
||||||
|
#mau_limit_alerting: false
|
||||||
|
|
||||||
# If enabled, the metrics for the number of monthly active users will
|
# If enabled, the metrics for the number of monthly active users will
|
||||||
# be populated, however no one will be limited. If limit_usage_by_mau
|
# be populated, however no one will be limited. If limit_usage_by_mau
|
||||||
|
|
|
@ -25,7 +25,13 @@ from twisted.internet import defer
|
||||||
import synapse.logging.opentracing as opentracing
|
import synapse.logging.opentracing as opentracing
|
||||||
import synapse.types
|
import synapse.types
|
||||||
from synapse import event_auth
|
from synapse import event_auth
|
||||||
from synapse.api.constants import EventTypes, JoinRules, Membership, UserTypes
|
from synapse.api.constants import (
|
||||||
|
EventTypes,
|
||||||
|
JoinRules,
|
||||||
|
LimitBlockingTypes,
|
||||||
|
Membership,
|
||||||
|
UserTypes,
|
||||||
|
)
|
||||||
from synapse.api.errors import (
|
from synapse.api.errors import (
|
||||||
AuthError,
|
AuthError,
|
||||||
Codes,
|
Codes,
|
||||||
|
@ -726,7 +732,7 @@ class Auth(object):
|
||||||
self.hs.config.hs_disabled_message,
|
self.hs.config.hs_disabled_message,
|
||||||
errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
|
errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
|
||||||
admin_contact=self.hs.config.admin_contact,
|
admin_contact=self.hs.config.admin_contact,
|
||||||
limit_type=self.hs.config.hs_disabled_limit_type,
|
limit_type=LimitBlockingTypes.HS_DISABLED,
|
||||||
)
|
)
|
||||||
if self.hs.config.limit_usage_by_mau is True:
|
if self.hs.config.limit_usage_by_mau is True:
|
||||||
assert not (user_id and threepid)
|
assert not (user_id and threepid)
|
||||||
|
@ -759,5 +765,5 @@ class Auth(object):
|
||||||
"Monthly Active User Limit Exceeded",
|
"Monthly Active User Limit Exceeded",
|
||||||
admin_contact=self.hs.config.admin_contact,
|
admin_contact=self.hs.config.admin_contact,
|
||||||
errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
|
errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
|
||||||
limit_type="monthly_active_user",
|
limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER,
|
||||||
)
|
)
|
||||||
|
|
|
@ -131,3 +131,10 @@ class RelationTypes(object):
|
||||||
ANNOTATION = "m.annotation"
|
ANNOTATION = "m.annotation"
|
||||||
REPLACE = "m.replace"
|
REPLACE = "m.replace"
|
||||||
REFERENCE = "m.reference"
|
REFERENCE = "m.reference"
|
||||||
|
|
||||||
|
|
||||||
|
class LimitBlockingTypes(object):
|
||||||
|
"""Reasons that a server may be blocked"""
|
||||||
|
|
||||||
|
MONTHLY_ACTIVE_USER = "monthly_active_user"
|
||||||
|
HS_DISABLED = "hs_disabled"
|
||||||
|
|
|
@ -171,6 +171,7 @@ class ServerConfig(Config):
|
||||||
)
|
)
|
||||||
|
|
||||||
self.mau_trial_days = config.get("mau_trial_days", 0)
|
self.mau_trial_days = config.get("mau_trial_days", 0)
|
||||||
|
self.mau_limit_alerting = config.get("mau_limit_alerting", True)
|
||||||
|
|
||||||
# How long to keep redacted events in the database in unredacted form
|
# How long to keep redacted events in the database in unredacted form
|
||||||
# before redacting them.
|
# before redacting them.
|
||||||
|
@ -192,7 +193,6 @@ class ServerConfig(Config):
|
||||||
# Options to disable HS
|
# Options to disable HS
|
||||||
self.hs_disabled = config.get("hs_disabled", False)
|
self.hs_disabled = config.get("hs_disabled", False)
|
||||||
self.hs_disabled_message = config.get("hs_disabled_message", "")
|
self.hs_disabled_message = config.get("hs_disabled_message", "")
|
||||||
self.hs_disabled_limit_type = config.get("hs_disabled_limit_type", "")
|
|
||||||
|
|
||||||
# Admin uri to direct users at should their instance become blocked
|
# Admin uri to direct users at should their instance become blocked
|
||||||
# due to resource constraints
|
# due to resource constraints
|
||||||
|
@ -675,7 +675,6 @@ class ServerConfig(Config):
|
||||||
#
|
#
|
||||||
#hs_disabled: false
|
#hs_disabled: false
|
||||||
#hs_disabled_message: 'Human readable reason for why the HS is blocked'
|
#hs_disabled_message: 'Human readable reason for why the HS is blocked'
|
||||||
#hs_disabled_limit_type: 'error code(str), to help clients decode reason'
|
|
||||||
|
|
||||||
# Monthly Active User Blocking
|
# Monthly Active User Blocking
|
||||||
#
|
#
|
||||||
|
@ -695,9 +694,16 @@ class ServerConfig(Config):
|
||||||
# sign up in a short space of time never to return after their initial
|
# sign up in a short space of time never to return after their initial
|
||||||
# session.
|
# session.
|
||||||
#
|
#
|
||||||
|
# 'mau_limit_alerting' is a means of limiting client side alerting
|
||||||
|
# should the mau limit be reached. This is useful for small instances
|
||||||
|
# where the admin has 5 mau seats (say) for 5 specific people and no
|
||||||
|
# interest increasing the mau limit further. Defaults to True, which
|
||||||
|
# means that alerting is enabled
|
||||||
|
#
|
||||||
#limit_usage_by_mau: false
|
#limit_usage_by_mau: false
|
||||||
#max_mau_value: 50
|
#max_mau_value: 50
|
||||||
#mau_trial_days: 2
|
#mau_trial_days: 2
|
||||||
|
#mau_limit_alerting: false
|
||||||
|
|
||||||
# If enabled, the metrics for the number of monthly active users will
|
# If enabled, the metrics for the number of monthly active users will
|
||||||
# be populated, however no one will be limited. If limit_usage_by_mau
|
# be populated, however no one will be limited. If limit_usage_by_mau
|
||||||
|
|
|
@ -20,6 +20,7 @@ from twisted.internet import defer
|
||||||
|
|
||||||
from synapse.api.constants import (
|
from synapse.api.constants import (
|
||||||
EventTypes,
|
EventTypes,
|
||||||
|
LimitBlockingTypes,
|
||||||
ServerNoticeLimitReached,
|
ServerNoticeLimitReached,
|
||||||
ServerNoticeMsgType,
|
ServerNoticeMsgType,
|
||||||
)
|
)
|
||||||
|
@ -70,7 +71,7 @@ class ResourceLimitsServerNotices(object):
|
||||||
return
|
return
|
||||||
|
|
||||||
if not self._server_notices_manager.is_enabled():
|
if not self._server_notices_manager.is_enabled():
|
||||||
# Don't try and send server notices unles they've been enabled
|
# Don't try and send server notices unless they've been enabled
|
||||||
return
|
return
|
||||||
|
|
||||||
timestamp = yield self._store.user_last_seen_monthly_active(user_id)
|
timestamp = yield self._store.user_last_seen_monthly_active(user_id)
|
||||||
|
@ -79,8 +80,6 @@ class ResourceLimitsServerNotices(object):
|
||||||
# In practice, not sure we can ever get here
|
# In practice, not sure we can ever get here
|
||||||
return
|
return
|
||||||
|
|
||||||
# Determine current state of room
|
|
||||||
|
|
||||||
room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
|
room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
|
||||||
|
|
||||||
if not room_id:
|
if not room_id:
|
||||||
|
@ -88,50 +87,85 @@ class ResourceLimitsServerNotices(object):
|
||||||
return
|
return
|
||||||
|
|
||||||
yield self._check_and_set_tags(user_id, room_id)
|
yield self._check_and_set_tags(user_id, room_id)
|
||||||
|
|
||||||
|
# Determine current state of room
|
||||||
currently_blocked, ref_events = yield self._is_room_currently_blocked(room_id)
|
currently_blocked, ref_events = yield self._is_room_currently_blocked(room_id)
|
||||||
|
|
||||||
|
limit_msg = None
|
||||||
|
limit_type = None
|
||||||
try:
|
try:
|
||||||
# Normally should always pass in user_id if you have it, but in
|
# Normally should always pass in user_id to check_auth_blocking
|
||||||
# this case are checking what would happen to other users if they
|
# if you have it, but in this case are checking what would happen
|
||||||
# were to arrive.
|
# to other users if they were to arrive.
|
||||||
try:
|
yield self._auth.check_auth_blocking()
|
||||||
yield self._auth.check_auth_blocking()
|
except ResourceLimitError as e:
|
||||||
is_auth_blocking = False
|
limit_msg = e.msg
|
||||||
except ResourceLimitError as e:
|
limit_type = e.limit_type
|
||||||
is_auth_blocking = True
|
|
||||||
event_content = e.msg
|
|
||||||
event_limit_type = e.limit_type
|
|
||||||
|
|
||||||
if currently_blocked and not is_auth_blocking:
|
try:
|
||||||
|
if (
|
||||||
|
limit_type == LimitBlockingTypes.MONTHLY_ACTIVE_USER
|
||||||
|
and not self._config.mau_limit_alerting
|
||||||
|
):
|
||||||
|
# We have hit the MAU limit, but MAU alerting is disabled:
|
||||||
|
# reset room if necessary and return
|
||||||
|
if currently_blocked:
|
||||||
|
self._remove_limit_block_notification(user_id, ref_events)
|
||||||
|
return
|
||||||
|
|
||||||
|
if currently_blocked and not limit_msg:
|
||||||
# Room is notifying of a block, when it ought not to be.
|
# Room is notifying of a block, when it ought not to be.
|
||||||
# Remove block notification
|
yield self._remove_limit_block_notification(user_id, ref_events)
|
||||||
content = {"pinned": ref_events}
|
elif not currently_blocked and limit_msg:
|
||||||
yield self._server_notices_manager.send_notice(
|
|
||||||
user_id, content, EventTypes.Pinned, ""
|
|
||||||
)
|
|
||||||
|
|
||||||
elif not currently_blocked and is_auth_blocking:
|
|
||||||
# Room is not notifying of a block, when it ought to be.
|
# Room is not notifying of a block, when it ought to be.
|
||||||
# Add block notification
|
yield self._apply_limit_block_notification(
|
||||||
content = {
|
user_id, limit_msg, limit_type
|
||||||
"body": event_content,
|
|
||||||
"msgtype": ServerNoticeMsgType,
|
|
||||||
"server_notice_type": ServerNoticeLimitReached,
|
|
||||||
"admin_contact": self._config.admin_contact,
|
|
||||||
"limit_type": event_limit_type,
|
|
||||||
}
|
|
||||||
event = yield self._server_notices_manager.send_notice(
|
|
||||||
user_id, content, EventTypes.Message
|
|
||||||
)
|
)
|
||||||
|
|
||||||
content = {"pinned": [event.event_id]}
|
|
||||||
yield self._server_notices_manager.send_notice(
|
|
||||||
user_id, content, EventTypes.Pinned, ""
|
|
||||||
)
|
|
||||||
|
|
||||||
except SynapseError as e:
|
except SynapseError as e:
|
||||||
logger.error("Error sending resource limits server notice: %s", e)
|
logger.error("Error sending resource limits server notice: %s", e)
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def _remove_limit_block_notification(self, user_id, ref_events):
|
||||||
|
"""Utility method to remove limit block notifications from the server
|
||||||
|
notices room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
user_id (str): user to notify
|
||||||
|
ref_events (list[str]): The event_ids of pinned events that are unrelated to
|
||||||
|
limit blocking and need to be preserved.
|
||||||
|
"""
|
||||||
|
content = {"pinned": ref_events}
|
||||||
|
yield self._server_notices_manager.send_notice(
|
||||||
|
user_id, content, EventTypes.Pinned, ""
|
||||||
|
)
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def _apply_limit_block_notification(self, user_id, event_body, event_limit_type):
|
||||||
|
"""Utility method to apply limit block notifications in the server
|
||||||
|
notices room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
user_id (str): user to notify
|
||||||
|
event_body(str): The human readable text that describes the block.
|
||||||
|
event_limit_type(str): Specifies the type of block e.g. monthly active user
|
||||||
|
limit has been exceeded.
|
||||||
|
"""
|
||||||
|
content = {
|
||||||
|
"body": event_body,
|
||||||
|
"msgtype": ServerNoticeMsgType,
|
||||||
|
"server_notice_type": ServerNoticeLimitReached,
|
||||||
|
"admin_contact": self._config.admin_contact,
|
||||||
|
"limit_type": event_limit_type,
|
||||||
|
}
|
||||||
|
event = yield self._server_notices_manager.send_notice(
|
||||||
|
user_id, content, EventTypes.Message
|
||||||
|
)
|
||||||
|
|
||||||
|
content = {"pinned": [event.event_id]}
|
||||||
|
yield self._server_notices_manager.send_notice(
|
||||||
|
user_id, content, EventTypes.Pinned, ""
|
||||||
|
)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _check_and_set_tags(self, user_id, room_id):
|
def _check_and_set_tags(self, user_id, room_id):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -17,7 +17,7 @@ from mock import Mock
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
|
|
||||||
from synapse.api.constants import EventTypes, ServerNoticeMsgType
|
from synapse.api.constants import EventTypes, LimitBlockingTypes, ServerNoticeMsgType
|
||||||
from synapse.api.errors import ResourceLimitError
|
from synapse.api.errors import ResourceLimitError
|
||||||
from synapse.server_notices.resource_limits_server_notices import (
|
from synapse.server_notices.resource_limits_server_notices import (
|
||||||
ResourceLimitsServerNotices,
|
ResourceLimitsServerNotices,
|
||||||
|
@ -133,7 +133,7 @@ class TestResourceLimitsServerNotices(unittest.HomeserverTestCase):
|
||||||
self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
|
self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
|
||||||
|
|
||||||
# Would be better to check contents, but 2 calls == set blocking event
|
# Would be better to check contents, but 2 calls == set blocking event
|
||||||
self.assertTrue(self._send_notice.call_count == 2)
|
self.assertEqual(self._send_notice.call_count, 2)
|
||||||
|
|
||||||
def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self):
|
def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self):
|
||||||
"""
|
"""
|
||||||
|
@ -158,6 +158,61 @@ class TestResourceLimitsServerNotices(unittest.HomeserverTestCase):
|
||||||
|
|
||||||
self._send_notice.assert_not_called()
|
self._send_notice.assert_not_called()
|
||||||
|
|
||||||
|
def test_maybe_send_server_notice_when_alerting_suppressed_room_unblocked(self):
|
||||||
|
"""
|
||||||
|
Test that when server is over MAU limit and alerting is suppressed, then
|
||||||
|
an alert message is not sent into the room
|
||||||
|
"""
|
||||||
|
self.hs.config.mau_limit_alerting = False
|
||||||
|
self._rlsn._auth.check_auth_blocking = Mock(
|
||||||
|
side_effect=ResourceLimitError(
|
||||||
|
403, "foo", limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
|
||||||
|
|
||||||
|
self.assertTrue(self._send_notice.call_count == 0)
|
||||||
|
|
||||||
|
def test_check_hs_disabled_unaffected_by_mau_alert_suppression(self):
|
||||||
|
"""
|
||||||
|
Test that when a server is disabled, that MAU limit alerting is ignored.
|
||||||
|
"""
|
||||||
|
self.hs.config.mau_limit_alerting = False
|
||||||
|
self._rlsn._auth.check_auth_blocking = Mock(
|
||||||
|
side_effect=ResourceLimitError(
|
||||||
|
403, "foo", limit_type=LimitBlockingTypes.HS_DISABLED
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
|
||||||
|
|
||||||
|
# Would be better to check contents, but 2 calls == set blocking event
|
||||||
|
self.assertEqual(self._send_notice.call_count, 2)
|
||||||
|
|
||||||
|
def test_maybe_send_server_notice_when_alerting_suppressed_room_blocked(self):
|
||||||
|
"""
|
||||||
|
When the room is already in a blocked state, test that when alerting
|
||||||
|
is suppressed that the room is returned to an unblocked state.
|
||||||
|
"""
|
||||||
|
self.hs.config.mau_limit_alerting = False
|
||||||
|
self._rlsn._auth.check_auth_blocking = Mock(
|
||||||
|
side_effect=ResourceLimitError(
|
||||||
|
403, "foo", limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self._rlsn._server_notices_manager.__is_room_currently_blocked = Mock(
|
||||||
|
return_value=defer.succeed((True, []))
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_event = Mock(
|
||||||
|
type=EventTypes.Message, content={"msgtype": ServerNoticeMsgType}
|
||||||
|
)
|
||||||
|
self._rlsn._store.get_events = Mock(
|
||||||
|
return_value=defer.succeed({"123": mock_event})
|
||||||
|
)
|
||||||
|
self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
|
||||||
|
|
||||||
|
self._send_notice.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase):
|
class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase):
|
||||||
def prepare(self, reactor, clock, hs):
|
def prepare(self, reactor, clock, hs):
|
||||||
|
|
|
@ -137,7 +137,6 @@ def default_config(name, parse=False):
|
||||||
"limit_usage_by_mau": False,
|
"limit_usage_by_mau": False,
|
||||||
"hs_disabled": False,
|
"hs_disabled": False,
|
||||||
"hs_disabled_message": "",
|
"hs_disabled_message": "",
|
||||||
"hs_disabled_limit_type": "",
|
|
||||||
"max_mau_value": 50,
|
"max_mau_value": 50,
|
||||||
"mau_trial_days": 0,
|
"mau_trial_days": 0,
|
||||||
"mau_stats_only": False,
|
"mau_stats_only": False,
|
||||||
|
|
Loading…
Reference in a new issue