From 1352ae2c079bba6d95a6d50ad5e0f4d20c892a58 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 17 Apr 2015 10:38:07 +0100 Subject: [PATCH 01/23] Add kick users script --- contrib/scripts/kick_users.py | 93 +++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 contrib/scripts/kick_users.py diff --git a/contrib/scripts/kick_users.py b/contrib/scripts/kick_users.py new file mode 100755 index 000000000..5dfaec3ad --- /dev/null +++ b/contrib/scripts/kick_users.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +from argparse import ArgumentParser +import json +import requests +import sys +import urllib + +def _mkurl(template, kws): + for key in kws: + template = template.replace(key, kws[key]) + return template + +def main(hs, room_id, access_token, user_id_prefix, why): + if not why: + why = "Automated kick." + print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix) + room_state_url = _mkurl( + "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN", + { + "$HS": hs, + "$ROOM": room_id, + "$TOKEN": access_token + } + ) + print "Getting room state => %s" % room_state_url + res = requests.get(room_state_url) + print "HTTP %s" % res.status_code + state_events = res.json() + if "error" in state_events: + print "FATAL" + print state_events + return + + kick_list = [] + room_name = room_id + for event in state_events: + if not event["type"] == "m.room.member": + if event["type"] == "m.room.name": + room_name = event["content"].get("name") + continue + if not event["content"].get("membership") == "join": + continue + if event["state_key"].startswith(user_id_prefix): + kick_list.append(event["state_key"]) + + if len(kick_list) == 0: + print "No user IDs match the prefix '%s'" % user_id_prefix + return + + print "The following user IDs will be kicked from %s" % room_name + for uid in kick_list: + print uid + doit = raw_input("Continue? [Y]es\n") + if len(doit) > 0 and doit.lower() == 'y': + print "Kicking members..." + # encode them all + kick_list = [urllib.quote(uid) for uid in kick_list] + for uid in kick_list: + kick_url = _mkurl( + "$HS/_matrix/client/api/v1/rooms/$ROOM/state/m.room.member/$UID?access_token=$TOKEN", + { + "$HS": hs, + "$UID": uid, + "$ROOM": room_id, + "$TOKEN": access_token + } + ) + kick_body = { + "membership": "leave", + "reason": why + } + print "Kicking %s" % uid + res = requests.put(kick_url, data=json.dumps(kick_body)) + if res.status_code != 200: + print "ERROR: HTTP %s" % res.status_code + if res.json().get("error"): + print "ERROR: JSON %s" % res.json() + + + +if __name__ == "__main__": + parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.") + parser.add_argument("-u","--user-id",help="The user ID prefix e.g. '@irc_'") + parser.add_argument("-t","--token",help="Your access_token") + parser.add_argument("-r","--room",help="The room ID to kick members in") + parser.add_argument("-s","--homeserver",help="The base HS url e.g. http://matrix.org") + parser.add_argument("-w","--why",help="Reason for the kick. Optional.") + args = parser.parse_args() + if not args.room or not args.token or not args.user_id or not args.homeserver: + parser.print_help() + sys.exit(1) + else: + main(args.homeserver, args.room, args.token, args.user_id, args.why) From 16dcdedc8a74967d5cafb466cc2f1af04b617458 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 13:24:55 +0100 Subject: [PATCH 02/23] As of version 2.7.9, urllib2 now checks SSL certs --- register_new_matrix_user | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/register_new_matrix_user b/register_new_matrix_user index daddadc30..974dfd398 100755 --- a/register_new_matrix_user +++ b/register_new_matrix_user @@ -49,7 +49,12 @@ def request_registration(user, password, server_location, shared_secret): headers={'Content-Type': 'application/json'} ) try: - f = urllib2.urlopen(req) + if sys.version_info[:3] >= (2, 7, 9): + # As of version 2.7.9, urllib2 now checks SSL certs + import ssl + f = urllib2.urlopen(req, ctx=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) + else: + f = urllib2.urlopen(req) f.read() f.close() print "Success." From ced39d019f124792a9dd38d6dfb558bdfdebd560 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 13:25:16 +0100 Subject: [PATCH 03/23] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index fd87c7e2d..56c10a84e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.8.1-r3" +__version__ = "0.8.1-r4" From fd4fa9097f4ea593a0896377ff5179397ffa94d7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Apr 2015 14:44:31 +0100 Subject: [PATCH 04/23] The new parameter to urlopen is "context" not "ctx" --- register_new_matrix_user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/register_new_matrix_user b/register_new_matrix_user index 974dfd398..4a520bdb5 100755 --- a/register_new_matrix_user +++ b/register_new_matrix_user @@ -52,7 +52,7 @@ def request_registration(user, password, server_location, shared_secret): if sys.version_info[:3] >= (2, 7, 9): # As of version 2.7.9, urllib2 now checks SSL certs import ssl - f = urllib2.urlopen(req, ctx=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) + f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) else: f = urllib2.urlopen(req) f.read() From 24d21887ed2311b67ccd9013387c450acd019628 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 14:14:19 +0100 Subject: [PATCH 05/23] SYN-350: Don't ratelimit the individual events generated during room creation --- synapse/handlers/room.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 823affc38..f9fc4a9c9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -124,7 +124,7 @@ class RoomCreationHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler for event in creation_events: - yield msg_handler.create_and_send_event(event) + yield msg_handler.create_and_send_event(event, ratelimit=False) if "name" in config: name = config["name"] @@ -134,7 +134,7 @@ class RoomCreationHandler(BaseHandler): "sender": user_id, "state_key": "", "content": {"name": name}, - }) + }, ratelimit=False) if "topic" in config: topic = config["topic"] @@ -144,7 +144,7 @@ class RoomCreationHandler(BaseHandler): "sender": user_id, "state_key": "", "content": {"topic": topic}, - }) + }, ratelimit=False) for invitee in invite_list: yield msg_handler.create_and_send_event({ @@ -153,7 +153,7 @@ class RoomCreationHandler(BaseHandler): "room_id": room_id, "sender": user_id, "content": {"membership": Membership.INVITE}, - }) + }, ratelimit=False) result = {"room_id": room_id} From 1ebff9736b3a74db3976a69db056438907b45e7b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 16:07:20 +0100 Subject: [PATCH 06/23] Split out the JsonResource request logging and error handling into a separate wrapper function --- synapse/http/server.py | 203 ++++++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 92 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index dee49b9e1..f1193f309 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -51,6 +51,71 @@ response_timer = metrics.register_distribution( labels=["method", "servlet"] ) +_next_request_id = 0 + + +def request_handler(request_handler): + """Wraps a method that acts as a request handler with the necessary logging + and exception handling. + The method must have a signature of "handle_foo(self, request)". + The argument "self" must have "version_string" and "clock" attributes. + The argument "request" must be a twisted HTTP request. + """ + + @defer.inlineCallbacks + def wrapped_request_handler(self, request): + global _next_request_id + request_id = "%s-%s" % (request.method, _next_request_id) + _next_request_id += 1 + with LoggingContext(request_id) as request_context: + request_context.request = request_id + code = None + start = self.clock.time_msec() + try: + logger.info( + "Received request: %s %s", + request.method, request.path + ) + yield request_handler(self, request) + code = request.code + except CodeMessageException as e: + code = e.code + if isinstance(e, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, e.msg + ) + else: + logger.exception(e) + outgoing_responses_counter.inc(request.method, str(code)) + respond_with_json( + request, code, cs_exception(e), send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + except: + code = 500 + logger.exception( + "Failed handle request %s.%s on %r: %r", + request_handler.__module__, + request_handler.__name__, + self, + request + ) + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) + finally: + code = str(code) if code else "-" + end = self.clock.time_msec() + logger.info( + "Processed request: %dms %s %s %s", + end-start, code, request.method, request.path + ) + return wrapped_request_handler + class HttpServer(object): """ Interface for registering callbacks on a HTTP server @@ -115,102 +180,56 @@ class JsonResource(HttpServer, resource.Resource): def render(self, request): """ This get's called by twisted every time someone sends us a request. """ - self._async_render_with_logging_context(request) + self._async_render(request) return server.NOT_DONE_YET - _request_id = 0 - - @defer.inlineCallbacks - def _async_render_with_logging_context(self, request): - request_id = "%s-%s" % (request.method, JsonResource._request_id) - JsonResource._request_id += 1 - with LoggingContext(request_id) as request_context: - request_context.request = request_id - yield self._async_render(request) - + @request_handler @defer.inlineCallbacks def _async_render(self, request): """ This get's called by twisted every time someone sends us a request. This checks if anyone has registered a callback for that method and path. """ - code = None start = self.clock.time_msec() - try: - # Just say yes to OPTIONS. - if request.method == "OPTIONS": - self._send_response(request, 200, {}) - return + if request.method == "OPTIONS": + self._send_response(request, 200, {}) + return + # Loop through all the registered callbacks to check if the method + # and path regex match + for path_entry in self.path_regexs.get(request.method, []): + m = path_entry.pattern.match(request.path) + if not m: + continue - # Loop through all the registered callbacks to check if the method - # and path regex match - for path_entry in self.path_regexs.get(request.method, []): - m = path_entry.pattern.match(request.path) - if not m: - continue + # We found a match! Trigger callback and then return the + # returned response. We pass both the request and any + # matched groups from the regex to the callback. - # We found a match! Trigger callback and then return the - # returned response. We pass both the request and any - # matched groups from the regex to the callback. + callback = path_entry.callback - callback = path_entry.callback - - servlet_instance = getattr(callback, "__self__", None) - if servlet_instance is not None: - servlet_classname = servlet_instance.__class__.__name__ - else: - servlet_classname = "%r" % callback - incoming_requests_counter.inc(request.method, servlet_classname) - - args = [ - urllib.unquote(u).decode("UTF-8") for u in m.groups() - ] - - logger.info( - "Received request: %s %s", - request.method, request.path - ) - - code, response = yield callback(request, *args) - - self._send_response(request, code, response) - response_timer.inc_by( - self.clock.time_msec() - start, request.method, servlet_classname - ) - - return - - # Huh. No one wanted to handle that? Fiiiiiine. Send 400. - raise UnrecognizedRequestError() - except CodeMessageException as e: - if isinstance(e, SynapseError): - logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) + servlet_instance = getattr(callback, "__self__", None) + if servlet_instance is not None: + servlet_classname = servlet_instance.__class__.__name__ else: - logger.exception(e) + servlet_classname = "%r" % callback + incoming_requests_counter.inc(request.method, servlet_classname) - code = e.code - self._send_response( - request, - code, - cs_exception(e), - response_code_message=e.response_code_message - ) - except Exception as e: - logger.exception(e) - self._send_response( - request, - 500, - {"error": "Internal server error"} - ) - finally: - code = str(code) if code else "-" + args = [ + urllib.unquote(u).decode("UTF-8") for u in m.groups() + ] - end = self.clock.time_msec() - logger.info( - "Processed request: %dms %s %s %s", - end-start, code, request.method, request.path + code, response = yield callback(request, *args) + + self._send_response(request, code, response) + response_timer.inc_by( + self.clock.time_msec() - start, request.method, servlet_classname ) + return + + # Huh. No one wanted to handle that? Fiiiiiine. Send 400. + raise UnrecognizedRequestError() + def _send_response(self, request, code, response_json_object, response_code_message=None): # could alternatively use request.notifyFinish() and flip a flag when @@ -229,20 +248,10 @@ class JsonResource(HttpServer, resource.Resource): request, code, response_json_object, send_cors=True, response_code_message=response_code_message, - pretty_print=self._request_user_agent_is_curl, + pretty_print=_request_user_agent_is_curl(request), version_string=self.version_string, ) - @staticmethod - def _request_user_agent_is_curl(request): - user_agents = request.requestHeaders.getRawHeaders( - "User-Agent", default=[] - ) - for user_agent in user_agents: - if "curl" in user_agent: - return True - return False - class RootRedirect(resource.Resource): """Redirects the root '/' path to another path.""" @@ -263,8 +272,8 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, response_code_message=None, pretty_print=False, version_string=""): - if not pretty_print: - json_bytes = encode_pretty_printed_json(json_object) + if pretty_print: + json_bytes = encode_pretty_printed_json(json_object) + "\n" else: json_bytes = encode_canonical_json(json_object) @@ -304,3 +313,13 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.write(json_bytes) request.finish() return NOT_DONE_YET + + +def _request_user_agent_is_curl(request): + user_agents = request.requestHeaders.getRawHeaders( + "User-Agent", default=[] + ) + for user_agent in user_agents: + if "curl" in user_agent: + return True + return False From 1967650bc4cbeea3deacaf540036f9cae4cbc330 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 16:35:53 +0100 Subject: [PATCH 07/23] Combine the request wrappers in rest/media/v1 and http/server into a single wrapper decorator --- synapse/http/server.py | 15 ++- synapse/http/servlet.py | 110 ++++++++++---------- synapse/rest/client/v2_alpha/sync.py | 20 ++-- synapse/rest/media/v1/base_resource.py | 80 +++----------- synapse/rest/media/v1/download_resource.py | 12 +-- synapse/rest/media/v1/thumbnail_resource.py | 16 +-- synapse/rest/media/v1/upload_resource.py | 85 +++++++-------- 7 files changed, 140 insertions(+), 198 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index f1193f309..b3706889a 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -57,9 +57,18 @@ _next_request_id = 0 def request_handler(request_handler): """Wraps a method that acts as a request handler with the necessary logging and exception handling. - The method must have a signature of "handle_foo(self, request)". - The argument "self" must have "version_string" and "clock" attributes. - The argument "request" must be a twisted HTTP request. + + The method must have a signature of "handle_foo(self, request)". The + argument "self" must have "version_string" and "clock" attributes. The + argument "request" must be a twisted HTTP request. + + The method must return a deferred. If the deferred succeeds we assume that + a response has been sent. If the deferred fails with a SynapseError we use + it to send a JSON response with the appropriate HTTP reponse code. If the + deferred fails with any other type of error we send a 500 reponse. + + We insert a unique request-id into the logging context for this request and + log the response and duration for this request. """ @defer.inlineCallbacks diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 265559a3e..9cda17fcf 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -23,6 +23,61 @@ import logging logger = logging.getLogger(__name__) +def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + +def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + +def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default + + class RestServlet(object): """ A Synapse REST Servlet. @@ -56,58 +111,3 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") - - @staticmethod - def parse_integer(request, name, default=None, required=False): - if name in request.args: - try: - return int(request.args[name][0]) - except: - message = "Query parameter %r must be an integer" % (name,) - raise SynapseError(400, message) - else: - if required: - message = "Missing integer query parameter %r" % (name,) - raise SynapseError(400, message) - else: - return default - - @staticmethod - def parse_boolean(request, name, default=None, required=False): - if name in request.args: - try: - return { - "true": True, - "false": False, - }[request.args[name][0]] - except: - message = ( - "Boolean query parameter %r must be one of" - " ['true', 'false']" - ) % (name,) - raise SynapseError(400, message) - else: - if required: - message = "Missing boolean query parameter %r" % (name,) - raise SynapseError(400, message) - else: - return default - - @staticmethod - def parse_string(request, name, default=None, required=False, - allowed_values=None, param_type="string"): - if name in request.args: - value = request.args[name][0] - if allowed_values is not None and value not in allowed_values: - message = "Query parameter %r must be one of [%s]" % ( - name, ", ".join(repr(v) for v in allowed_values) - ) - raise SynapseError(message) - else: - return value - else: - if required: - message = "Missing %s query parameter %r" % (param_type, name) - raise SynapseError(400, message) - else: - return default diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 3056ec45c..f2fd0b9f3 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -15,7 +15,9 @@ from twisted.internet import defer -from synapse.http.servlet import RestServlet +from synapse.http.servlet import ( + RestServlet, parse_string, parse_integer, parse_boolean +) from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken from synapse.events.utils import ( @@ -87,20 +89,20 @@ class SyncRestServlet(RestServlet): def on_GET(self, request): user, client = yield self.auth.get_user_by_req(request) - timeout = self.parse_integer(request, "timeout", default=0) - limit = self.parse_integer(request, "limit", required=True) - gap = self.parse_boolean(request, "gap", default=True) - sort = self.parse_string( + timeout = parse_integer(request, "timeout", default=0) + limit = parse_integer(request, "limit", required=True) + gap = parse_boolean(request, "gap", default=True) + sort = parse_string( request, "sort", default="timeline,asc", allowed_values=self.ALLOWED_SORT ) - since = self.parse_string(request, "since") - set_presence = self.parse_string( + since = parse_string(request, "since") + set_presence = parse_string( request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = self.parse_boolean(request, "backfill", default=False) - filter_id = self.parse_string(request, "filter", default=None) + backfill = parse_boolean(request, "backfill", default=False) + filter_id = parse_string(request, "filter", default=None) logger.info( "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r," diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index b10cbddb8..fc078fca8 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -18,7 +18,7 @@ from .thumbnailer import Thumbnailer from synapse.http.server import respond_with_json from synapse.util.stringutils import random_string from synapse.api.errors import ( - cs_exception, CodeMessageException, cs_error, Codes, SynapseError + cs_error, Codes, SynapseError ) from twisted.internet import defer @@ -32,6 +32,18 @@ import logging logger = logging.getLogger(__name__) +def parse_media_id(request): + try: + server_name, media_id = request.postpath + return (server_name, media_id) + except: + raise SynapseError( + 404, + "Invalid media id token %r" % (request.postpath,), + Codes.UNKNOWN, + ) + + class BaseMediaResource(Resource): isLeaf = True @@ -47,72 +59,6 @@ class BaseMediaResource(Resource): self.filepaths = filepaths self.downloads = {} - @staticmethod - def catch_errors(request_handler): - @defer.inlineCallbacks - def wrapped_request_handler(self, request): - try: - yield request_handler(self, request) - except CodeMessageException as e: - logger.info("Responding with error: %r", e) - respond_with_json( - request, e.code, cs_exception(e), send_cors=True - ) - except: - logger.exception( - "Failed handle request %s.%s on %r", - request_handler.__module__, - request_handler.__name__, - self, - ) - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True - ) - return wrapped_request_handler - - @staticmethod - def _parse_media_id(request): - try: - server_name, media_id = request.postpath - return (server_name, media_id) - except: - raise SynapseError( - 404, - "Invalid media id token %r" % (request.postpath,), - Codes.UNKNOWN, - ) - - @staticmethod - def _parse_integer(request, arg_name, default=None): - try: - if default is None: - return int(request.args[arg_name][0]) - else: - return int(request.args.get(arg_name, [default])[0]) - except: - raise SynapseError( - 400, - "Missing integer argument %r" % (arg_name,), - Codes.UNKNOWN, - ) - - @staticmethod - def _parse_string(request, arg_name, default=None): - try: - if default is None: - return request.args[arg_name][0] - else: - return request.args.get(arg_name, [default])[0] - except: - raise SynapseError( - 400, - "Missing string argument %r" % (arg_name,), - Codes.UNKNOWN, - ) - def _respond_404(self, request): respond_with_json( request, 404, diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index c585bb11f..0fe6abf64 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .base_resource import BaseMediaResource +from .base_resource import BaseMediaResource, parse_media_id +from synapse.http.server import request_handler from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -28,15 +29,10 @@ class DownloadResource(BaseMediaResource): self._async_render_GET(request) return NOT_DONE_YET - @BaseMediaResource.catch_errors + @request_handler @defer.inlineCallbacks def _async_render_GET(self, request): - try: - server_name, media_id = request.postpath - except: - self._respond_404(request) - return - + server_name, media_id = parse_media_id(request) if server_name == self.server_name: yield self._respond_local_file(request, media_id) else: diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 84f5e3463..1dadd880b 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -14,7 +14,9 @@ # limitations under the License. -from .base_resource import BaseMediaResource +from .base_resource import BaseMediaResource, parse_media_id +from synapse.http.servlet import parse_string, parse_integer +from synapse.http.server import request_handler from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -31,14 +33,14 @@ class ThumbnailResource(BaseMediaResource): self._async_render_GET(request) return NOT_DONE_YET - @BaseMediaResource.catch_errors + @request_handler @defer.inlineCallbacks def _async_render_GET(self, request): - server_name, media_id = self._parse_media_id(request) - width = self._parse_integer(request, "width") - height = self._parse_integer(request, "height") - method = self._parse_string(request, "method", "scale") - m_type = self._parse_string(request, "type", "image/png") + server_name, media_id = parse_media_id(request) + width = parse_integer(request, "width") + height = parse_integer(request, "height") + method = parse_string(request, "method", "scale") + m_type = parse_string(request, "type", "image/png") if server_name == self.server_name: yield self._respond_local_thumbnail( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index e5aba3af4..cc571976a 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import respond_with_json +from synapse.http.server import respond_with_json, request_handler from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException -) +from synapse.api.errors import SynapseError from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -69,53 +67,42 @@ class UploadResource(BaseMediaResource): defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) + @request_handler @defer.inlineCallbacks def _async_render_POST(self, request): - try: - auth_user, client = yield self.auth.get_user_by_req(request) - # TODO: The checks here are a bit late. The content will have - # already been uploaded to a tmp file at this point - content_length = request.getHeader("Content-Length") - if content_length is None: - raise SynapseError( - msg="Request must specify a Content-Length", code=400 - ) - if int(content_length) > self.max_upload_size: - raise SynapseError( - msg="Upload request body is too large", - code=413, - ) - - headers = request.requestHeaders - - if headers.hasHeader("Content-Type"): - media_type = headers.getRawHeaders("Content-Type")[0] - else: - raise SynapseError( - msg="Upload request missing 'Content-Type'", - code=400, - ) - - # if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] - # TODO(markjh): parse content-dispostion - - content_uri = yield self.create_content( - media_type, None, request.content.read(), - content_length, auth_user + auth_user, client = yield self.auth.get_user_by_req(request) + # TODO: The checks here are a bit late. The content will have + # already been uploaded to a tmp file at this point + content_length = request.getHeader("Content-Length") + if content_length is None: + raise SynapseError( + msg="Request must specify a Content-Length", code=400 + ) + if int(content_length) > self.max_upload_size: + raise SynapseError( + msg="Upload request body is too large", + code=413, ) - respond_with_json( - request, 200, {"content_uri": content_uri}, send_cors=True - ) - except CodeMessageException as e: - logger.exception(e) - respond_with_json(request, e.code, cs_exception(e), send_cors=True) - except: - logger.exception("Failed to store file") - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True + headers = request.requestHeaders + + if headers.hasHeader("Content-Type"): + media_type = headers.getRawHeaders("Content-Type")[0] + else: + raise SynapseError( + msg="Upload request missing 'Content-Type'", + code=400, ) + + # if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] + # TODO(markjh): parse content-dispostion + + content_uri = yield self.create_content( + media_type, None, request.content.read(), + content_length, auth_user + ) + + respond_with_json( + request, 200, {"content_uri": content_uri}, send_cors=True + ) From 812a99100bb86e3a09b3e5739c849e55c74cf448 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 16:43:58 +0100 Subject: [PATCH 08/23] Set a version_string in BaseMediaResource so that the request_handler wrapper works --- synapse/rest/media/v1/base_resource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index fc078fca8..edd4f7802 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -57,6 +57,7 @@ class BaseMediaResource(Resource): self.max_upload_size = hs.config.max_upload_size self.max_image_pixels = hs.config.max_image_pixels self.filepaths = filepaths + self.version_string = hs.version_string self.downloads = {} def _respond_404(self, request): From 6080830beffd38a08b08d73ee1588cf2289c0080 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 17:03:57 +0100 Subject: [PATCH 09/23] Bump syutil version to 0.0.5 --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index dac927d0a..ee72f774b 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.4": ["syutil"], + "syutil>=0.0.5": ["syutil"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -43,8 +43,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/syutil", - version="v0.0.4", - egg="syutil-0.0.4", + version="v0.0.5", + egg="syutil-0.0.5", ), github_link( project="matrix-org/matrix-angular-sdk", From 3a7d7a3f22fe7358b23250e1e8b8d5a9e4559f23 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:18:29 +0100 Subject: [PATCH 10/23] Sanitise a user's powerlevel to an int() before numerical comparison, because otherwise Python is "helpful" with it (SYN-351) --- synapse/api/auth.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e159e4503..c1b3ae173 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -189,6 +189,12 @@ class Auth(object): auth_events, ) + # TODO(paul): There's an awful lot of int()-casting in this code; + # surely we should be squashing strings to integers at a higher + # level, maybe when we insert? + if user_level is not None: + user_level = int(user_level) + ban_level, kick_level, redact_level = ( self._get_ops_level_from_event_state( event, @@ -269,6 +275,7 @@ class Auth(object): 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: + print "I wonder how user's level of %r compares to ban level of %r" % (user_level, ban_level) if user_level < ban_level: raise AuthError(403, "You don't have permission to ban") else: From b568c0231c708431532aa385ebcc121e0a8ef986 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:21:14 +0100 Subject: [PATCH 11/23] Remove debugging print statement accidentally committed --- synapse/api/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index c1b3ae173..43b21897b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -275,7 +275,6 @@ class Auth(object): 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: - print "I wonder how user's level of %r compares to ban level of %r" % (user_level, ban_level) if user_level < ban_level: raise AuthError(403, "You don't have permission to ban") else: From d3309933f52f4382470b72ec1079f403ca412904 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:53:23 +0100 Subject: [PATCH 12/23] Much neater fetching of defined powerlevels from m.room.power_levels state event --- synapse/api/auth.py | 52 ++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 43b21897b..9a5058a36 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -195,12 +195,8 @@ class Auth(object): if user_level is not None: user_level = int(user_level) - ban_level, kick_level, redact_level = ( - self._get_ops_level_from_event_state( - event, - auth_events, - ) - ) + # FIXME (erikj): What should we do here as the default? + ban_level = self._get_named_level(auth_events, "ban", 50) logger.debug( "is_membership_change_allowed: %s", @@ -216,11 +212,6 @@ class Auth(object): } ) - if ban_level: - ban_level = int(ban_level) - else: - ban_level = 50 # FIXME (erikj): What should we do here? - if Membership.JOIN != membership: # JOIN is the only action you can perform if you're not in the room if not caller_in_room: # caller isn't joined @@ -265,10 +256,7 @@ class Auth(object): 403, "You cannot unban user &s." % (target_user_id,) ) elif target_user_id != event.user_id: - if kick_level: - kick_level = int(kick_level) - else: - kick_level = 50 # FIXME (erikj): What should we do here? + kick_level = self._get_named_level(auth_events, "kick", 50) if user_level < kick_level: raise AuthError( @@ -282,10 +270,14 @@ class Auth(object): return True - def _get_power_level_from_event_state(self, event, user_id, auth_events): + def _get_power_level_event(self, auth_events): key = (EventTypes.PowerLevels, "", ) - power_level_event = auth_events.get(key) + return auth_events.get(key) + + def _get_power_level_from_event_state(self, event, user_id, auth_events): + power_level_event = self._get_power_level_event(auth_events) level = None + if power_level_event: level = power_level_event.content.get("users", {}).get(user_id) if not level: @@ -299,17 +291,18 @@ class Auth(object): return level - def _get_ops_level_from_event_state(self, event, auth_events): - key = (EventTypes.PowerLevels, "", ) - power_level_event = auth_events.get(key) - if power_level_event: - return ( - power_level_event.content.get("ban", 50), - power_level_event.content.get("kick", 50), - power_level_event.content.get("redact", 50), - ) - return None, None, None, + def _get_named_level(self, auth_events, name, default): + power_level_event = self._get_power_level_event(auth_events) + + if not power_level_event: + return default + + level = power_level_event.content.get(name, None) + if level is not None: + return int(level) + else: + return default @defer.inlineCallbacks def get_user_by_req(self, request): @@ -551,10 +544,7 @@ class Auth(object): auth_events, ) - _, _, redact_level = self._get_ops_level_from_event_state( - event, - auth_events, - ) + redact_level = self._get_named_level(auth_events, "redact", 50) if user_level < redact_level: raise AuthError( From 7c50e3b81655b8d4236d0308cd21506ac35b593c Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 22 Apr 2015 08:38:26 +0100 Subject: [PATCH 13/23] Add info on breaking AS API changes --- UPGRADE.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/UPGRADE.rst b/UPGRADE.rst index 87dd6e04a..ab327a813 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,37 @@ +Upgrading to v0.x.x +=================== + +Application services have had a breaking API change in this version. + +They can no longer register themselves with a home server using the AS HTTP API. This +decision was made because a compromised application service with free reign to register +any regex in effect grants full read/write access to the home server if a regex of ``.*`` +is used. An attack where a compromised AS re-registers itself with ``.*`` was deemed too +big of a security risk to ignore, and so the ability to register with the HS remotely has +been removed. + +It has been replaced by specifying a list of application service registrations in +``homeserver.yaml``:: + + app_service_config_files: ["registration-01.yaml", "registration-02.yaml"] + +Where ``registration-01.yaml`` looks like:: + + url: # e.g. "https://my.application.service.com" + as_token: + hs_token: + sender_localpart: # This is a new field which denotes the user_id localpart when using the AS token + namespaces: + users: + - exclusive: + regex: # e.g. "@prefix_.*" + aliases: + - exclusive: + regex: + rooms: + - exclusive: + regex: + Upgrading to v0.8.0 =================== From f43063158afb33bc1601632583b9e6377ff76aca Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 22 Apr 2015 13:12:11 +0100 Subject: [PATCH 14/23] Appease pep8 --- synapse/api/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9a5058a36..bae210c57 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -291,7 +291,6 @@ class Auth(object): return level - def _get_named_level(self, auth_events, name, default): power_level_event = self._get_power_level_event(auth_events) From a16eaa0c337c29a932b5effddfddff78849836c9 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 22 Apr 2015 14:20:04 +0100 Subject: [PATCH 15/23] Neater fetching of user's auth level in a room - squash to int() at access time (SYN-353) --- synapse/api/auth.py | 47 ++++++++++++--------------------------------- 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index bae210c57..a21120b31 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -183,17 +183,7 @@ class Auth(object): else: join_rule = JoinRules.INVITE - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) - - # TODO(paul): There's an awful lot of int()-casting in this code; - # surely we should be squashing strings to integers at a higher - # level, maybe when we insert? - if user_level is not None: - user_level = int(user_level) + user_level = self._get_user_power_level(event.user_id, auth_events) # FIXME (erikj): What should we do here as the default? ban_level = self._get_named_level(auth_events, "ban", 50) @@ -274,22 +264,26 @@ class Auth(object): key = (EventTypes.PowerLevels, "", ) return auth_events.get(key) - def _get_power_level_from_event_state(self, event, user_id, auth_events): + def _get_user_power_level(self, user_id, auth_events): power_level_event = self._get_power_level_event(auth_events) - level = None if power_level_event: level = power_level_event.content.get("users", {}).get(user_id) if not level: level = power_level_event.content.get("users_default", 0) + + if level is None: + return 0 + else: + return int(level) else: key = (EventTypes.Create, "", ) create_event = auth_events.get(key) if (create_event is not None and create_event.content["creator"] == user_id): return 100 - - return level + else: + return 0 def _get_named_level(self, auth_events, name, default): power_level_event = self._get_power_level_event(auth_events) @@ -496,16 +490,7 @@ class Auth(object): else: send_level = 0 - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) - - if user_level: - user_level = int(user_level) - else: - user_level = 0 + user_level = self._get_user_power_level(event.user_id, auth_events) if user_level < send_level: raise AuthError( @@ -537,11 +522,7 @@ class Auth(object): return True def _check_redaction(self, event, auth_events): - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) + user_level = self._get_user_power_level(event.user_id, auth_events) redact_level = self._get_named_level(auth_events, "redact", 50) @@ -571,11 +552,7 @@ class Auth(object): if not current_state: return - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) + user_level = self._get_user_power_level(event.user_id, auth_events) # Check other levels: levels_to_check = [ From 6c994913473b70e164b13a4f551da8a8d448cc33 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 23 Apr 2015 16:07:49 +0100 Subject: [PATCH 16/23] prometheus/metrics howto from Leo --- docs/metrics-howto.rst | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/metrics-howto.rst diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst new file mode 100644 index 000000000..b3e71fc77 --- /dev/null +++ b/docs/metrics-howto.rst @@ -0,0 +1,49 @@ +How to monitor Synapse metrics using Prometheus +=============================================== + +1: install prometheus: + Follow instructions at http://prometheus.io/docs/introduction/install/ + +2: enable synapse metrics: + Simply setting a (local) port number will enable it. Pick a port. + prometheus itself defaults to 9090, so starting just above that for + locally monitored services seems reasonable. E.g. 9092: + + Add to homeserver.yaml + + metrics_port: 9092 + + Restart synapse + +3: check out synapse-prometheus-config + https://github.com/matrix-org/synapse-prometheus-config + +4: arrange for synapse.html to appear in prometheus's "consoles" + directory - symlink might be easiest to ensure `git pull` keeps it + updated. + +5: arrange for synapse.rules to be invoked from the main + prometheus.conf and add a synapse target. This is easiest if + prometheus runs on the same machine as synapse, as it can then just + use localhost:: + + global: { + rule_file: "synapse.rules" + } + + job: { + name: "synapse" + + target_group: { + target: "http://localhost:9092/" + } + } + +6: start prometheus:: + + ./prometheus -config.file=prometheus.conf + +7: wait a few seconds for it to start and perform the first scrape, + then visit the console: + + http://server-where-prometheus-runs:9090/consoles/synapse.html From 8c784142845bf462b255374b4cbacc22fd572847 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 16:14:08 +0100 Subject: [PATCH 17/23] Formatting / wording fixes to metrics doc --- docs/metrics-howto.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index b3e71fc77..99776cd50 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -1,10 +1,10 @@ How to monitor Synapse metrics using Prometheus =============================================== -1: install prometheus: +1: Install prometheus: Follow instructions at http://prometheus.io/docs/introduction/install/ -2: enable synapse metrics: +2: Enable synapse metrics: Simply setting a (local) port number will enable it. Pick a port. prometheus itself defaults to 9090, so starting just above that for locally monitored services seems reasonable. E.g. 9092: @@ -15,17 +15,18 @@ How to monitor Synapse metrics using Prometheus Restart synapse -3: check out synapse-prometheus-config +3: Check out synapse-prometheus-config https://github.com/matrix-org/synapse-prometheus-config -4: arrange for synapse.html to appear in prometheus's "consoles" - directory - symlink might be easiest to ensure `git pull` keeps it - updated. +4: Add `synapse.html` and `synapse.rules` + The `.html` file needs to appear in prometheus's "consoles" directory, and + the `.rules` file needs to be invoked somewhere in the main config file. + A symlink to each from the git checkout into the prometheus directory might be + easiest to ensure `git pull` keeps it updated. -5: arrange for synapse.rules to be invoked from the main - prometheus.conf and add a synapse target. This is easiest if - prometheus runs on the same machine as synapse, as it can then just - use localhost:: +5: Add a prometheus target for synapse + This is easiest if prometheus runs on the same machine as synapse, as it can + then just use localhost:: global: { rule_file: "synapse.rules" @@ -39,11 +40,11 @@ How to monitor Synapse metrics using Prometheus } } -6: start prometheus:: +6: Start prometheus:: ./prometheus -config.file=prometheus.conf -7: wait a few seconds for it to start and perform the first scrape, +7: Wait a few seconds for it to start and perform the first scrape, then visit the console: http://server-where-prometheus-runs:9090/consoles/synapse.html From 6d1540134133cfe07fbecfbf0c733aceade33a05 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 16:16:08 +0100 Subject: [PATCH 18/23] Mumble ReST mumble ``fixed-width`` mumble --- docs/metrics-howto.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 99776cd50..c1f5ae217 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -18,11 +18,11 @@ How to monitor Synapse metrics using Prometheus 3: Check out synapse-prometheus-config https://github.com/matrix-org/synapse-prometheus-config -4: Add `synapse.html` and `synapse.rules` - The `.html` file needs to appear in prometheus's "consoles" directory, and - the `.rules` file needs to be invoked somewhere in the main config file. - A symlink to each from the git checkout into the prometheus directory might be - easiest to ensure `git pull` keeps it updated. +4: Add ``synapse.html`` and ``synapse.rules`` + The ``.html`` file needs to appear in prometheus's ``consoles`` directory, + and the ``.rules`` file needs to be invoked somewhere in the main config + file. A symlink to each from the git checkout into the prometheus directory + might be easiest to ensure ``git pull`` keeps it updated. 5: Add a prometheus target for synapse This is easiest if prometheus runs on the same machine as synapse, as it can From 191f7f09cee4e148949af9e33c5c8f60184acf90 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 18:27:25 +0100 Subject: [PATCH 19/23] Generate presence event-stream JSON structures directly --- synapse/handlers/presence.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index bbc7a0f20..6332f5097 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -836,6 +836,8 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap + clock = self.clock + latest_serial = None updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. @@ -845,18 +847,17 @@ class PresenceEventSource(object): if cached.serial <= from_key: continue - if (yield self.is_visible(observer_user, observed_user)): - updates.append((observed_user, cached)) + if not (yield self.is_visible(observer_user, observed_user)): + continue + + if latest_serial is None or cached.serial > latest_serial: + latest_serial = cached.serial + updates.append(cached.make_event(user=observed_user, clock=clock)) # TODO(paul): limit if updates: - clock = self.clock - - latest_serial = max([x[1].serial for x in updates]) - data = [x[1].make_event(user=x[0], clock=clock) for x in updates] - - defer.returnValue((data, latest_serial)) + defer.returnValue((updates, latest_serial)) else: defer.returnValue(([], presence._user_cachemap_latest_serial)) From 8a785c3006327076245428d26e5ca1634e9caeb2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 18:40:19 +0100 Subject: [PATCH 20/23] Store a list of the presence serial number at which remote users went offline, so that when we delete them from the cachemap, we can still synthesize OFFLINE events for them (SYN-261) --- synapse/handlers/presence.py | 21 ++++++++++++++++++ tests/handlers/test_presence.py | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6332f5097..42fb622c4 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -135,6 +135,9 @@ class PresenceHandler(BaseHandler): self._remote_sendmap = {} # map remote users to sets of local users who're interested in them self._remote_recvmap = {} + # list of (serial, set of(userids)) tuples, ordered by serial, latest + # first + self._remote_offline_serials = [] # map any user to a UserPresenceCache self._user_cachemap = {} @@ -715,6 +718,10 @@ class PresenceHandler(BaseHandler): ) if state["presence"] == PresenceState.OFFLINE: + self._remote_offline_serials.insert( + 0, + (self._user_cachemap_latest_serial, set([user.to_string()])) + ) del self._user_cachemap[user] for poll in content.get("poll", []): @@ -856,6 +863,20 @@ class PresenceEventSource(object): # TODO(paul): limit + for serial, user_ids in presence._remote_offline_serials: + if serial < from_key: + break + + for u in user_ids: + updates.append({ + "type": "m.presence", + "content": {"user_id": u, "presence": PresenceState.OFFLINE}, + }) + # TODO(paul): For the v2 API we want to tell the client their from_key + # is too old if we fell off the end of the _remote_offline_serials + # list, and get them to invalidate+resync. In v1 we have no such + # concept so this is a best-effort result. + if updates: defer.returnValue((updates, latest_serial)) else: diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 04eba4289..bb497b6f0 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -878,6 +878,44 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): state ) + @defer.inlineCallbacks + def test_recv_remote_offline(self): + """ Various tests relating to SYN-261 """ + potato_set = self.handler._remote_recvmap.setdefault(self.u_potato, + set()) + potato_set.add(self.u_apple) + + self.room_members = [self.u_banana, self.u_potato] + + self.assertEquals(self.event_source.get_current_key(), 0) + + yield self.mock_federation_resource.trigger("PUT", + "/_matrix/federation/v1/send/1000000/", + _make_edu_json("elsewhere", "m.presence", + content={ + "push": [ + {"user_id": "@potato:remote", + "presence": "offline"}, + ], + } + ) + ) + + self.assertEquals(self.event_source.get_current_key(), 1) + + (events, _) = yield self.event_source.get_new_events_for_user( + self.u_apple, 0, None + ) + self.assertEquals(events, + [ + {"type": "m.presence", + "content": { + "user_id": "@potato:remote", + "presence": OFFLINE, + }} + ] + ) + @defer.inlineCallbacks def test_join_room_local(self): self.room_members = [self.u_apple, self.u_banana] From b3bda8a75f9745fd351d2c2093ffc68774e8a2e2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 18:40:47 +0100 Subject: [PATCH 21/23] Don't let the remote offline serial list grow arbitrarily large --- synapse/handlers/presence.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 42fb622c4..f929bcf85 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -36,6 +36,9 @@ metrics = synapse.metrics.get_metrics_for(__name__) # Don't bother bumping "last active" time if it differs by less than 60 seconds LAST_ACTIVE_GRANULARITY = 60*1000 +# Keep no more than this number of offline serial revisions +MAX_OFFLINE_SERIALS = 1000 + # TODO(paul): Maybe there's one of these I can steal from somewhere def partition(l, func): @@ -722,6 +725,8 @@ class PresenceHandler(BaseHandler): 0, (self._user_cachemap_latest_serial, set([user.to_string()])) ) + while len(self._remote_offline_serials) > MAX_OFFLINE_SERIALS: + self._remote_offline_serials.pop() # remove the oldest del self._user_cachemap[user] for poll in content.get("poll", []): From e1e5e53127540fbaa4e23fbc628113983efd767b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 19:01:37 +0100 Subject: [PATCH 22/23] Remove users from the remote_offline_serials list (and clean up empty elements) when they go online again --- synapse/handlers/presence.py | 12 +++++++++++- tests/handlers/test_presence.py | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index f929bcf85..571eacd34 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -720,14 +720,24 @@ class PresenceHandler(BaseHandler): statuscache=statuscache, ) + user_id = user.to_string() + if state["presence"] == PresenceState.OFFLINE: self._remote_offline_serials.insert( 0, - (self._user_cachemap_latest_serial, set([user.to_string()])) + (self._user_cachemap_latest_serial, set([user_id])) ) while len(self._remote_offline_serials) > MAX_OFFLINE_SERIALS: self._remote_offline_serials.pop() # remove the oldest del self._user_cachemap[user] + else: + # Remove the user from remote_offline_serials now that they're + # no longer offline + for idx, elem in enumerate(self._remote_offline_serials): + (_, user_ids) = elem + user_ids.discard(user_id) + if not user_ids: + self._remote_offline_serials.pop(idx) for poll in content.get("poll", []): user = UserID.from_string(poll) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index bb497b6f0..9f5580c09 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -916,6 +916,33 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): ] ) + yield self.mock_federation_resource.trigger("PUT", + "/_matrix/federation/v1/send/1000001/", + _make_edu_json("elsewhere", "m.presence", + content={ + "push": [ + {"user_id": "@potato:remote", + "presence": "online"}, + ], + } + ) + ) + + self.assertEquals(self.event_source.get_current_key(), 2) + + (events, _) = yield self.event_source.get_new_events_for_user( + self.u_apple, 0, None + ) + self.assertEquals(events, + [ + {"type": "m.presence", + "content": { + "user_id": "@potato:remote", + "presence": ONLINE, + }} + ] + ) + @defer.inlineCallbacks def test_join_room_local(self): self.room_members = [self.u_apple, self.u_banana] From 74270defdaf4070ba001713ae9f1f668790fc9a3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 09:27:42 +0100 Subject: [PATCH 23/23] No commas here, otherwise our error string constants become tuples. --- synapse/api/errors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index eddd88977..72d2bd5b4 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -35,8 +35,8 @@ class Codes(object): LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" - MISSING_PARAM = "M_MISSING_PARAM", - TOO_LARGE = "M_TOO_LARGE", + MISSING_PARAM = "M_MISSING_PARAM" + TOO_LARGE = "M_TOO_LARGE" EXCLUSIVE = "M_EXCLUSIVE"