Merge branch 'develop' into request_logging

Conflicts:
	synapse/config/logger.py

synapse/api/__init__.py

@@ -12,4 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

synapse/api/errors.py

@@ -54,7 +54,7 @@ class SynapseError(CodeMessageException):
         """Constructs a synapse error.
 
         Args:
-            code (int): The integer error code (typically an HTTP response code)
+            code (int): The integer error code (an HTTP response code)
             msg (str): The human-readable error message.
             err (str): The error code e.g 'M_FORBIDDEN'
         """
@@ -67,6 +67,7 @@ class SynapseError(CodeMessageException):
             self.errcode,
         )
 
+
 class RoomError(SynapseError):
     """An error raised when a room event fails."""
     pass
@@ -117,6 +118,7 @@ class InvalidCaptchaError(SynapseError):
             error_url=self.error_url,
         )
 
+
 class LimitExceededError(SynapseError):
     """A client has sent too many requests and is being throttled.
     """

synapse/app/__init__.py

@@ -12,4 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

synapse/config/_base.py

@@ -123,11 +123,18 @@ class Config(object):
                 #   style mode markers into the file, to hint to people that
                 #   this is a YAML file.
                 yaml.dump(config, config_file, default_flow_style=False)
-            print "A config file has been generated in %s for server name '%s') with corresponding SSL keys and self-signed certificates. Please review this file and customise it to your needs." % (config_args.config_path, config['server_name'])
+            print (
-            print "If this server name is incorrect, you will need to regenerate the SSL certificates"
+                "A config file has been generated in %s for server name"
+                " '%s' with corresponding SSL keys and self-signed"
+                " certificates. Please review this file and customise it to"
+                " your needs."
+            ) % (
+                config_args.config_path, config['server_name']
+            )
+            print (
+                "If this server name is incorrect, you will need to regenerate"
+                " the SSL certificates"
+            )
             sys.exit(0)
 
         return cls(args)
-
-
-

synapse/config/database.py

@@ -16,6 +16,7 @@
 from ._base import Config
 import os
 
+
 class DatabaseConfig(Config):
     def __init__(self, args):
         super(DatabaseConfig, self).__init__(args)
@@ -34,4 +35,3 @@ class DatabaseConfig(Config):
     def generate_config(cls, args, config_dir_path):
         super(DatabaseConfig, cls).generate_config(args, config_dir_path)
         args.database_path = os.path.abspath(args.database_path)
-

synapse/config/email.py

@@ -35,5 +35,8 @@ class EmailConfig(Config):
         email_group.add_argument(
             "--email-smtp-server",
             default="",
-            help="The SMTP server to send emails from (e.g. for password resets)."
+            help=(
+                "The SMTP server to send emails from (e.g. for password"
+                " resets)."
+            )
         )

synapse/config/ratelimiting.py

@@ -14,6 +14,7 @@
 
 from ._base import Config
 
+
 class RatelimitConfig(Config):
 
     def __init__(self, args):

synapse/config/repository.py

@@ -15,6 +15,7 @@
 
 from ._base import Config
 
+
 class ContentRepositoryConfig(Config):
     def __init__(self, args):
         super(ContentRepositoryConfig, self).__init__(args)

synapse/config/voip.py

@@ -33,7 +33,10 @@ class VoipConfig(Config):
         )
         group.add_argument(
             "--turn-shared-secret", type=str, default=None,
-            help="The shared secret used to compute passwords for the TURN server"
+            help=(
+                "The shared secret used to compute passwords for the TURN"
+                " server"
+            )
         )
         group.add_argument(
             "--turn-user-lifetime", type=int, default=(1000 * 60 * 60),

synapse/crypto/__init__.py

@@ -12,4 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

synapse/crypto/context_factory.py

@@ -20,6 +20,7 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+
 class ServerContextFactory(ssl.ContextFactory):
     """Factory for PyOpenSSL SSL contexts that are used to handle incoming
     connections and to make connections to remote servers."""
@@ -43,4 +44,3 @@ class ServerContextFactory(ssl.ContextFactory):
 
     def getContext(self):
         return self._context
-

synapse/crypto/keyclient.py

@@ -100,4 +100,3 @@ class SynapseKeyClientProtocol(HTTPClient):
 
 class SynapseKeyClientFactory(Factory):
     protocol = SynapseKeyClientProtocol
-

synapse/handlers/login.py

@@ -54,7 +54,7 @@ class LoginHandler(BaseHandler):
         # pull out the hash for this user if they exist
         user_info = yield self.store.get_user_by_id(user_id=user)
         if not user_info:
-            logger.warn("Attempted to login as %s but they do not exist.", user)
+            logger.warn("Attempted to login as %s but they do not exist", user)
             raise LoginError(403, "", errcode=Codes.FORBIDDEN)
 
         stored_hash = user_info[0]["password_hash"]

synapse/handlers/message.py

@@ -115,8 +115,12 @@ class MessageHandler(BaseHandler):
 
         user = self.hs.parse_userid(user_id)
 
-        events, next_token = yield data_source.get_pagination_rows(
+        events, next_key = yield data_source.get_pagination_rows(
-            user, pagin_config, room_id
+            user, pagin_config.get_source_config("room"), room_id
+        )
+
+        next_token = pagin_config.from_token.copy_and_replace(
+            "room_key", next_key
         )
 
         chunk = {
@@ -271,7 +275,7 @@ class MessageHandler(BaseHandler):
         presence_stream = self.hs.get_event_sources().sources["presence"]
         pagination_config = PaginationConfig(from_token=now_token)
         presence, _ = yield presence_stream.get_pagination_rows(
-            user, pagination_config, None
+            user, pagination_config.get_source_config("presence"), None
         )
 
         public_rooms = yield self.store.get_rooms(is_public=True)

synapse/handlers/presence.py

@@ -76,9 +76,7 @@ class PresenceHandler(BaseHandler):
             "stopped_user_eventstream", self.stopped_user_eventstream
         )
 
-        distributor.observe("user_joined_room",
+        distributor.observe("user_joined_room", self.user_joined_room)
-            self.user_joined_room
-        )
 
         distributor.declare("collect_presencelike_data")
 
@@ -156,14 +154,12 @@ class PresenceHandler(BaseHandler):
             defer.returnValue(True)
 
         if (yield self.store.user_rooms_intersect(
-            [u.to_string() for u in observer_user, observed_user]
+                [u.to_string() for u in observer_user, observed_user])):
-        )):
             defer.returnValue(True)
 
         if (yield self.store.is_presence_visible(
                 observed_localpart=observed_user.localpart,
-            observer_userid=observer_user.to_string(),
+                observer_userid=observer_user.to_string())):
-        )):
             defer.returnValue(True)
 
         defer.returnValue(False)
@@ -171,7 +167,8 @@ class PresenceHandler(BaseHandler):
     @defer.inlineCallbacks
     def get_state(self, target_user, auth_user):
         if target_user.is_mine:
-            visible = yield self.is_presence_visible(observer_user=auth_user,
+            visible = yield self.is_presence_visible(
+                observer_user=auth_user,
                 observed_user=target_user
             )
 
@@ -219,9 +216,9 @@ class PresenceHandler(BaseHandler):
                 )
 
         if state["presence"] not in self.STATE_LEVELS:
-            raise SynapseError(400, "'%s' is not a valid presence state" %
+            raise SynapseError(400, "'%s' is not a valid presence state" % (
-                state["presence"]
+                state["presence"],
-            )
+            ))
 
         logger.debug("Updating presence state of %s to %s",
                      target_user.localpart, state["presence"])
@@ -649,8 +646,9 @@ class PresenceHandler(BaseHandler):
             del state["user_id"]
 
             if "presence" not in state:
-                logger.warning("Received a presence 'push' EDU from %s without"
+                logger.warning(
-                    + " a 'presence' key", origin
+                    "Received a presence 'push' EDU from %s without a"
+                    " 'presence' key", origin
                 )
                 continue
 
@@ -765,8 +763,7 @@ class PresenceEventSource(object):
         presence = self.hs.get_handlers().presence_handler
 
         if (yield presence.store.user_rooms_intersect(
-            [u.to_string() for u in observer_user, observed_user]
+                [u.to_string() for u in observer_user, observed_user])):
-        )):
             defer.returnValue(True)
 
         if observed_user.is_mine:
@@ -823,15 +820,12 @@ class PresenceEventSource(object):
     def get_pagination_rows(self, user, pagination_config, key):
         # TODO (erikj): Does this make sense? Ordering?
 
-        from_token = pagination_config.from_token
-        to_token = pagination_config.to_token
-
         observer_user = user
 
-        from_key = int(from_token.presence_key)
+        from_key = int(pagination_config.from_key)
 
-        if to_token:
+        if pagination_config.to_key:
-            to_key = int(to_token.presence_key)
+            to_key = int(pagination_config.to_key)
         else:
             to_key = -1
 
@@ -841,7 +835,7 @@ class PresenceEventSource(object):
         updates = []
         # TODO(paul): use a DeferredList ? How to limit concurrency.
         for observed_user in cachemap.keys():
-            if not (to_key < cachemap[observed_user].serial < from_key):
+            if not (to_key < cachemap[observed_user].serial <= from_key):
                 continue
 
             if (yield self.is_visible(observer_user, observed_user)):
@@ -849,30 +843,15 @@ class PresenceEventSource(object):
 
         # TODO(paul): limit
 
-        updates = [(k, cachemap[k]) for k in cachemap
-                   if to_key < cachemap[k].serial < from_key]
-
         if updates:
             clock = self.clock
 
             earliest_serial = max([x[1].serial for x in updates])
             data = [x[1].make_event(user=x[0], clock=clock) for x in updates]
 
-            if to_token:
+            defer.returnValue((data, earliest_serial))
-                next_token = to_token
         else:
-                next_token = from_token
+            defer.returnValue(([], 0))
-
-            next_token = next_token.copy_and_replace(
-                "presence_key", earliest_serial
-            )
-            defer.returnValue((data, next_token))
-        else:
-            if not to_token:
-                to_token = from_token.copy_and_replace(
-                    "presence_key", 0
-                )
-            defer.returnValue(([], to_token))
 
 
 class UserPresenceCache(object):

synapse/handlers/register.py

@@ -64,9 +64,11 @@ class RegistrationHandler(BaseHandler):
             user_id = user.to_string()
 
             token = self._generate_token(user_id)
-            yield self.store.register(user_id=user_id,
+            yield self.store.register(
+                user_id=user_id,
                 token=token,
-                password_hash=password_hash)
+                password_hash=password_hash
+            )
 
             self.distributor.fire("registered_user", user)
         else:
@@ -181,8 +183,11 @@ class RegistrationHandler(BaseHandler):
         data = yield httpCli.post_urlencoded_get_json(
             creds['idServer'],
             "/_matrix/identity/api/v1/3pid/bind",
-            {'sid': creds['sid'], 'clientSecret': creds['clientSecret'],
+            {
-            'mxid': mxid}
+                'sid': creds['sid'],
+                'clientSecret': creds['clientSecret'],
+                'mxid': mxid,
+            }
         )
         defer.returnValue(data)
 
@@ -223,5 +228,3 @@ class RegistrationHandler(BaseHandler):
             }
         )
         defer.returnValue(data)
-
-

synapse/handlers/room.py

@@ -612,23 +612,14 @@ class RoomEventSource(object):
         return self.store.get_room_events_max_id()
 
     @defer.inlineCallbacks
-    def get_pagination_rows(self, user, pagination_config, key):
+    def get_pagination_rows(self, user, config, key):
-        from_token = pagination_config.from_token
-        to_token = pagination_config.to_token
-        limit = pagination_config.limit
-        direction = pagination_config.direction
-
-        to_key = to_token.room_key if to_token else None
-
         events, next_key = yield self.store.paginate_room_events(
             room_id=key,
-            from_key=from_token.room_key,
+            from_key=config.from_key,
-            to_key=to_key,
+            to_key=config.to_key,
-            direction=direction,
+            direction=config.direction,
-            limit=limit,
+            limit=config.limit,
             with_feedback=True
         )
 
-        next_token = from_token.copy_and_replace("room_key", next_key)
+        defer.returnValue((events, next_key))
-
-        defer.returnValue((events, next_token))

synapse/handlers/typing.py

@@ -96,9 +96,10 @@ class TypingNotificationHandler(BaseHandler):
         remotedomains = set()
 
         rm_handler = self.homeserver.get_handlers().room_member_handler
-        yield rm_handler.fetch_room_distributions_into(room_id,
+        yield rm_handler.fetch_room_distributions_into(
-                localusers=localusers, remotedomains=remotedomains,
+            room_id, localusers=localusers, remotedomains=remotedomains,
-                ignore_user=user)
+            ignore_user=user
+        )
 
         for u in localusers:
             self.push_update_to_clients(
@@ -130,8 +131,9 @@ class TypingNotificationHandler(BaseHandler):
         localusers = set()
 
         rm_handler = self.homeserver.get_handlers().room_member_handler
-        yield rm_handler.fetch_room_distributions_into(room_id,
+        yield rm_handler.fetch_room_distributions_into(
-                localusers=localusers)
+            room_id, localusers=localusers
+        )
 
         for u in localusers:
             self.push_update_to_clients(
@@ -158,4 +160,4 @@ class TypingNotificationEventSource(object):
         return 0
 
     def get_pagination_rows(self, user, pagination_config, key):
-        return ([], pagination_config.from_token)
+        return ([], pagination_config.from_key)

synapse/http/__init__.py

@@ -12,4 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

synapse/http/client.py

@@ -100,7 +100,7 @@ class BaseHttpClient(object):
 
         retries_left = 5
 
-        endpoint = self._getEndpoint(reactor, destination);
+        endpoint = self._getEndpoint(reactor, destination)
 
         while True:
 
@@ -280,7 +280,6 @@ class MatrixHttpClient(BaseHttpClient):
 
         defer.returnValue(json.loads(body))
 
-
     def _getEndpoint(self, reactor, destination):
         return matrix_endpoint(
             reactor, destination, timeout=10,
@@ -355,6 +354,7 @@ class IdentityServerHttpClient(BaseHttpClient):
 
         defer.returnValue(json.loads(body))
 
+
 class CaptchaServerHttpClient(MatrixHttpClient):
     """Separate HTTP client for talking to google's captcha servers"""
 
@@ -388,6 +388,7 @@ class CaptchaServerHttpClient(MatrixHttpClient):
             else:
                 raise e
 
+
 def _print_ex(e):
     if hasattr(e, "reasons") and e.reasons:
         for ex in e.reasons:

synapse/http/content_repository.py

@@ -38,8 +38,8 @@ class ContentRepoResource(resource.Resource):
 
     Uploads are POSTed to wherever this Resource is linked to. This resource
     returns a "content token" which can be used to GET this content again. The
-    token is typically a path, but it may not be. Tokens can expire, be one-time
+    token is typically a path, but it may not be. Tokens can expire, be
-    uses, etc.
+    one-time uses, etc.
 
     In this case, the token is a path to the file and contains 3 interesting
     sections:
@@ -175,10 +175,9 @@ class ContentRepoResource(resource.Resource):
             with open(fname, "wb") as f:
                 f.write(request.content.read())
 
-
             # FIXME (erikj): These should use constants.
             file_name = os.path.basename(fname)
-            # FIXME: we can't assume what the public mounted path of the repo is
+            # FIXME: we can't assume what the repo's public mounted path is
             # ...plus self-signed SSL won't work to remote clients anyway
             # ...and we can't assume that it's SSL anyway, as we might want to
             # server it via the non-SSL listener...
@@ -201,6 +200,3 @@ class ContentRepoResource(resource.Resource):
                 500,
                 json.dumps({"error": "Internal server error"}),
                 send_cors=True)
-
-
-

synapse/notifier.py

@@ -167,7 +167,8 @@ class Notifier(object):
                 )
 
         def eb(failure):
-            logger.error("Failed to notify listener",
+            logger.error(
+                "Failed to notify listener",
                 exc_info=(
                     failure.type,
                     failure.value,
@@ -207,7 +208,7 @@ class Notifier(object):
         )
 
         if timeout:
-            reactor.callLater(timeout/1000, self._timeout_listener, listener)
+            reactor.callLater(timeout/1000.0, self._timeout_listener, listener)
 
             self._register_with_keys(listener)
 

synapse/rest/register.py

@@ -60,40 +60,45 @@ class RegisterRestServlet(RestServlet):
 
     def on_GET(self, request):
         if self.hs.config.enable_registration_captcha:
-            return (200, {
+            return (
-                "flows": [
+                200,
+                {"flows": [
                     {
                         "type": LoginType.RECAPTCHA,
-                        "stages": ([LoginType.RECAPTCHA,
+                        "stages": [
+                            LoginType.RECAPTCHA,
                             LoginType.EMAIL_IDENTITY,
-                                    LoginType.PASSWORD])
+                            LoginType.PASSWORD
+                        ]
                     },
                     {
                         "type": LoginType.RECAPTCHA,
                         "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD]
                     }
-                ]
+                ]}
-            })
+            )
         else:
-            return (200, {
+            return (
-                "flows": [
+                200,
+                {"flows": [
                     {
                         "type": LoginType.EMAIL_IDENTITY,
-                        "stages": ([LoginType.EMAIL_IDENTITY,
+                        "stages": [
-                                    LoginType.PASSWORD])
+                            LoginType.EMAIL_IDENTITY, LoginType.PASSWORD
+                        ]
                     },
                     {
                         "type": LoginType.PASSWORD
                     }
-                ]
+                ]}
-            })
+            )
 
     @defer.inlineCallbacks
     def on_POST(self, request):
         register_json = _parse_json(request)
 
-        session = (register_json["session"] if "session" in register_json
+        session = (register_json["session"]
-                  else None)
+                   if "session" in register_json else None)
         login_type = None
         if "type" not in register_json:
             raise SynapseError(400, "Missing 'type' key.")
@@ -122,7 +127,9 @@ class RegisterRestServlet(RestServlet):
             defer.returnValue((200, response))
         except KeyError as e:
             logger.exception(e)
-            raise SynapseError(400, "Missing JSON keys for login type %s." % login_type)
+            raise SynapseError(400, "Missing JSON keys for login type %s." % (
+                login_type,
+            ))
 
     def on_OPTIONS(self, request):
         return (200, {})
@@ -183,8 +190,10 @@ class RegisterRestServlet(RestServlet):
                 session["user"] = register_json["user"]
                 defer.returnValue(None)
             else:
-                raise SynapseError(400, "Captcha bypass HMAC incorrect",
+                raise SynapseError(
-                    errcode=Codes.CAPTCHA_NEEDED)
+                    400, "Captcha bypass HMAC incorrect",
+                    errcode=Codes.CAPTCHA_NEEDED
+                )
 
         challenge = None
         user_response = None
@@ -230,12 +239,15 @@ class RegisterRestServlet(RestServlet):
 
         if ("user" in session and "user" in register_json and
                 session["user"] != register_json["user"]):
-            raise SynapseError(400, "Cannot change user ID during registration")
+            raise SynapseError(
+                400, "Cannot change user ID during registration"
+            )
 
         password = register_json["password"].encode("utf-8")
-        desired_user_id = (register_json["user"].encode("utf-8") if "user"
+        desired_user_id = (register_json["user"].encode("utf-8")
-                          in register_json else None)
+                           if "user" in register_json else None)
-        if desired_user_id and urllib.quote(desired_user_id) != desired_user_id:
+        if (desired_user_id
+                and urllib.quote(desired_user_id) != desired_user_id):
             raise SynapseError(
                 400,
                 "User ID must only contain characters which do not " +

synapse/rest/room.py

@@ -48,7 +48,9 @@ class RoomCreateRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, txn_id):
         try:
-            defer.returnValue(self.txns.get_client_transaction(request, txn_id))
+            defer.returnValue(
+                self.txns.get_client_transaction(request, txn_id)
+            )
         except KeyError:
             pass
 
@@ -98,7 +100,7 @@ class RoomStateEventRestServlet(RestServlet):
         no_state_key = "/rooms/(?P<room_id>[^/]*)/state/(?P<event_type>[^/]*)$"
 
         # /room/$roomid/state/$eventtype/$statekey
-        state_key = ("/rooms/(?P<room_id>[^/]*)/state/" +
+        state_key = ("/rooms/(?P<room_id>[^/]*)/state/"
                      "(?P<event_type>[^/]*)/(?P<state_key>[^/]*)$")
 
         http_server.register_path("GET",
@@ -133,7 +135,9 @@ class RoomStateEventRestServlet(RestServlet):
         )
 
         if not data:
-            raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND)
+            raise SynapseError(
+                404, "Event not found.", errcode=Codes.NOT_FOUND
+            )
         defer.returnValue((200, data[0].get_dict()["content"]))
 
     @defer.inlineCallbacks
@@ -195,7 +199,9 @@ class RoomSendEventRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, room_id, event_type, txn_id):
         try:
-            defer.returnValue(self.txns.get_client_transaction(request, txn_id))
+            defer.returnValue(
+                self.txns.get_client_transaction(request, txn_id)
+            )
         except KeyError:
             pass
 
@@ -254,7 +260,9 @@ class JoinRoomAliasServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, room_identifier, txn_id):
         try:
-            defer.returnValue(self.txns.get_client_transaction(request, txn_id))
+            defer.returnValue(
+                self.txns.get_client_transaction(request, txn_id)
+            )
         except KeyError:
             pass
 
@@ -293,7 +301,8 @@ class RoomMemberListRestServlet(RestServlet):
             target_user = self.hs.parse_userid(event["user_id"])
             # Presence is an optional cache; don't fail if we can't fetch it
             try:
-                presence_state = yield self.handlers.presence_handler.get_state(
+                presence_handler = self.handlers.presence_handler
+                presence_state = yield presence_handler.get_state(
                     target_user=target_user, auth_user=user
                 )
                 event["content"].update(presence_state)
@@ -359,11 +368,11 @@ class RoomInitialSyncRestServlet(RestServlet):
         #       { state event } , { state event }
         #   ]
         # }
-        # Probably worth keeping the keys room_id and membership for parity with
+        # Probably worth keeping the keys room_id and membership for parity
-        # /initialSync even though they must be joined to sync this and know the
+        # with /initialSync even though they must be joined to sync this and
-        # room ID, so clients can reuse the same code (room_id and membership
+        # know the room ID, so clients can reuse the same code (room_id and
-        # are MANDATORY for /initialSync, so the code will expect it to be
+        # membership are MANDATORY for /initialSync, so the code will expect
-        # there)
+        # it to be there)
         defer.returnValue((200, {}))
 
 
@@ -388,7 +397,7 @@ class RoomMembershipRestServlet(RestServlet):
 
     def register(self, http_server):
         # /rooms/$roomid/[invite|join|leave]
-        PATTERN = ("/rooms/(?P<room_id>[^/]*)/" +
+        PATTERN = ("/rooms/(?P<room_id>[^/]*)/"
                    "(?P<membership_action>join|invite|leave|ban|kick)")
         register_txn_path(self, PATTERN, http_server)
 
@@ -422,7 +431,9 @@ class RoomMembershipRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, room_id, membership_action, txn_id):
         try:
-            defer.returnValue(self.txns.get_client_transaction(request, txn_id))
+            defer.returnValue(
+                self.txns.get_client_transaction(request, txn_id)
+            )
         except KeyError:
             pass
 
@@ -431,6 +442,7 @@ class RoomMembershipRestServlet(RestServlet):
         self.txns.store_client_transaction(request, txn_id, response)
         defer.returnValue(response)
 
+
 class RoomRedactEventRestServlet(RestServlet):
     def register(self, http_server):
         PATTERN = ("/rooms/(?P<room_id>[^/]*)/redact/(?P<event_id>[^/]*)")
@@ -457,7 +469,9 @@ class RoomRedactEventRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, room_id, event_id, txn_id):
         try:
-            defer.returnValue(self.txns.get_client_transaction(request, txn_id))
+            defer.returnValue(
+                self.txns.get_client_transaction(request, txn_id)
+            )
         except KeyError:
             pass
 

synapse/rest/transactions.py

@@ -30,9 +30,9 @@ class HttpTransactionStore(object):
         """Retrieve a response for this request.
 
         Args:
-            key (str): A transaction-independent key for this request. Typically
+            key (str): A transaction-independent key for this request. Usually
-            this is a combination of the path (without the transaction id) and
+                this is a combination of the path (without the transaction id)
-            the user's access token.
+                and the user's access token.
             txn_id (str): The transaction ID for this request
         Returns:
             A tuple of (HTTP response code, response content) or None.
@@ -51,9 +51,9 @@ class HttpTransactionStore(object):
         """Stores an HTTP response tuple.
 
         Args:
-            key (str): A transaction-independent key for this request. Typically
+            key (str): A transaction-independent key for this request. Usually
-            this is a combination of the path (without the transaction id) and
+                this is a combination of the path (without the transaction id)
-            the user's access token.
+                and the user's access token.
             txn_id (str): The transaction ID for this request.
             response (tuple): A tuple of (HTTP response code, response content)
         """
@@ -92,5 +92,3 @@ class HttpTransactionStore(object):
         token = request.args["access_token"][0]
         path_without_txn_id = request.path.rsplit("/", 1)[0]
         return path_without_txn_id + "/" + token
-
-

synapse/rest/voip.py

@@ -40,9 +40,9 @@ class VoipRestServlet(RestServlet):
         username = "%d:%s" % (expiry, auth_user.to_string())
 
         mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1)
-        # We need to use standard base64 encoding here, *not* syutil's encode_base64
+        # We need to use standard base64 encoding here, *not* syutil's
-        # because we need to add the standard padding to get the same result as the
+        # encode_base64 because we need to add the standard padding to get the
-        # TURN server.
+        # same result as the TURN server.
         password = base64.b64encode(mac.digest())
 
         defer.returnValue((200, {

synapse/storage/keys.py

@@ -21,6 +21,7 @@ import OpenSSL
 from syutil.crypto.signing_key import decode_verify_key_bytes
 import hashlib
 
+
 class KeyStore(SQLBaseStore):
     """Persistence for signature verification keys and tls X.509 certificates
     """

synapse/storage/roommember.py

@@ -33,7 +33,9 @@ class RoomMemberStore(SQLBaseStore):
             target_user_id = event.state_key
             domain = self.hs.parse_userid(target_user_id).domain
         except:
-            logger.exception("Failed to parse target_user_id=%s", target_user_id)
+            logger.exception(
+                "Failed to parse target_user_id=%s", target_user_id
+            )
             raise
 
         logger.debug(
@@ -65,7 +67,8 @@ class RoomMemberStore(SQLBaseStore):
             # Check if this was the last person to have left.
             member_events = self._get_members_query_txn(
                 txn,
-                where_clause="c.room_id = ? AND m.membership = ? AND m.user_id != ?",
+                where_clause=("c.room_id = ? AND m.membership = ?"
+                              " AND m.user_id != ?"),
                 where_values=(event.room_id, Membership.JOIN, target_user_id,)
             )
 
@@ -120,7 +123,6 @@ class RoomMemberStore(SQLBaseStore):
         else:
             return None
 
-
     def get_room_members(self, room_id, membership=None):
         """Retrieve the current room member list for a room.
 

synapse/streams/config.py

@@ -22,6 +22,19 @@ import logging
 logger = logging.getLogger(__name__)
 
 
+class SourcePaginationConfig(object):
+
+    """A configuration object which stores pagination parameters for a
+    specific event source."""
+
+    def __init__(self, from_key=None, to_key=None, direction='f',
+                 limit=0):
+        self.from_key = from_key
+        self.to_key = to_key
+        self.direction = 'f' if direction == 'f' else 'b'
+        self.limit = int(limit)
+
+
 class PaginationConfig(object):
 
     """A configuration object which stores pagination parameters."""
@@ -82,3 +95,13 @@ class PaginationConfig(object):
             "<PaginationConfig from_tok=%s, to_tok=%s, "
             "direction=%s, limit=%s>"
         ) % (self.from_token, self.to_token, self.direction, self.limit)
+
+    def get_source_config(self, source_name):
+        keyname = "%s_key" % source_name
+
+        return SourcePaginationConfig(
+            from_key=getattr(self.from_token, keyname),
+            to_key=getattr(self.to_token, keyname) if self.to_token else None,
+            direction=self.direction,
+            limit=self.limit,
+        )

synapse/streams/events.py

@@ -35,7 +35,7 @@ class NullSource(object):
         return defer.succeed(0)
 
     def get_pagination_rows(self, user, pagination_config, key):
-        return defer.succeed(([], pagination_config.from_token))
+        return defer.succeed(([], pagination_config.from_key))
 
 
 class EventSources(object):

synapse/util/distributor.py

@@ -42,7 +42,8 @@ class Distributor(object):
         if name in self.signals:
             raise KeyError("%r already has a signal named %s" % (self, name))
 
-        self.signals[name] = Signal(name,
+        self.signals[name] = Signal(
+            name,
             suppress_failures=self.suppress_failures,
         )
 

synapse/util/emailutils.py

@@ -42,7 +42,7 @@ def send_email(smtp_server, from_addr, to_addr, subject, body):
         EmailException if there was a problem sending the mail.
     """
     if not smtp_server or not from_addr or not to_addr:
-        raise EmailException("Need SMTP server, from and to addresses. Check " +
+        raise EmailException("Need SMTP server, from and to addresses. Check"
                              " the config to set these.")
 
     msg = MIMEMultipart('alternative')

synapse/util/jsonobject.py

@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import copy
 
+
 class JsonEncodedObject(object):
     """ A common base class for defining protocol units that are represented
     as JSON.
@@ -89,6 +89,7 @@ class JsonEncodedObject(object):
     def __str__(self):
         return "(%s, %s)" % (self.__class__.__name__, repr(self.__dict__))
 
+
 def _encode(obj):
     if type(obj) is list:
         return [_encode(o) for o in obj]

tests/handlers/test_presence.py

@@ -29,6 +29,7 @@ from synapse.server import HomeServer
 from synapse.api.constants import PresenceState
 from synapse.api.errors import SynapseError
 from synapse.handlers.presence import PresenceHandler, UserPresenceCache
+from synapse.streams.config import SourcePaginationConfig
 
 
 OFFLINE = PresenceState.OFFLINE
@@ -676,6 +677,21 @@ class PresencePushTestCase(unittest.TestCase):
             msg="Presence event should be visible to self-reflection"
         )
 
+        config = SourcePaginationConfig(from_key=1, to_key=0)
+        (chunk, _) = yield self.event_source.get_pagination_rows(
+            self.u_apple, config, None
+        )
+        self.assertEquals(chunk,
+            [
+                {"type": "m.presence",
+                 "content": {
+                     "user_id": "@apple:test",
+                     "presence": ONLINE,
+                     "last_active_ago": 0,
+                }},
+            ]
+        )
+
         # Banana sees it because of presence subscription
         (events, _) = yield self.event_source.get_new_events_for_user(
             self.u_banana, 0, None

webclient/app-controller.js

@@ -53,7 +53,7 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
      * Open a given page.
      * @param {String} url url of the page
      */
-    $scope.goToPage = function(url) {
+    $rootScope.goToPage = function(url) {
         $location.url(url);
     };
     

webclient/app.css

@@ -416,7 +416,6 @@ textarea, input {
     text-align: right;
     float: right;
     margin-top: 15px;
-    width: 100%;
 }
 
 /*** Participant list ***/

webclient/components/matrix/event-handler-service.js

@@ -58,14 +58,29 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) {
         var shouldBing = false;
         
         // case-insensitive name check for user_id OR display_name if they exist
+        var userRegex = "";
         var myUserId = matrixService.config().user_id;
         if (myUserId) {
-            myUserId = myUserId.toLocaleLowerCase();
+            var localpart = getLocalPartFromUserId(myUserId);
+            if (localpart) {
+                localpart = localpart.toLocaleLowerCase();
+                userRegex += "\\b" + localpart + "\\b";
+            }
         }
         var myDisplayName = matrixService.config().display_name;
         if (myDisplayName) {
             myDisplayName = myDisplayName.toLocaleLowerCase();
+            if (userRegex.length > 0) {
+                userRegex += "|";
             }
+            userRegex += "\\b" + myDisplayName + "\\b";
+        }
+
+        var r = new RegExp(userRegex, 'i');
+        if (content.search(r) >= 0) {
+            shouldBing = true;
+        }
+
         if ( (myDisplayName && content.toLocaleLowerCase().indexOf(myDisplayName) != -1) ||
              (myUserId && content.toLocaleLowerCase().indexOf(myUserId) != -1) ) {
             shouldBing = true;
@@ -84,6 +99,18 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) {
         return shouldBing;
     };
 
+    var getLocalPartFromUserId = function(user_id) {
+        if (!user_id) {
+            return null;
+        }
+        var localpartRegex = /@(.*):\w+/i
+        var results = localpartRegex.exec(user_id);
+        if (results && results.length == 2) {
+            return results[1];
+        }
+        return null;
+    };
+
     var initialSyncDeferred;
 
     var reset = function() {
@@ -172,6 +199,17 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) {
     };
 
     var handleMessage = function(event, isLiveEvent) {
+        // Check for empty event content
+        var hasContent = false;
+        for (var prop in event.content) {
+            hasContent = true;
+            break;
+        }
+        if (!hasContent) {
+            // empty json object is a redacted event, so ignore.
+            return;
+        }
+
         if (isLiveEvent) {
             if (event.user_id === matrixService.config().user_id &&
                 (event.content.msgtype === "m.text" || event.content.msgtype === "m.emote") ) {
@@ -238,6 +276,12 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) {
                         "body": message,
                         "icon": member ? member.avatar_url : undefined
                     });
+
+                    notification.onclick = function() {
+                        console.log("notification.onclick() room=" + event.room_id);
+                        $rootScope.goToPage('room/' + (event.room_id)); 
+                    };
+
                     $timeout(function() {
                         notification.close();
                     }, 5 * 1000);
@@ -330,6 +374,31 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) {
         }
     };
 
+    var handleRedaction = function(event, isLiveEvent) {
+        if (!isLiveEvent) {
+            // we have nothing to remove, so just ignore it.
+            console.log("Received redacted event: "+JSON.stringify(event));
+            return;
+        }
+
+        // we need to remove something possibly: do we know the redacted
+        // event ID?
+        if (eventMap[event.redacts]) {
+            // remove event from list of messages in this room.
+            var eventList = $rootScope.events.rooms[event.room_id].messages;
+            for (var i=0; i<eventList.length; i++) {
+                if (eventList[i].event_id === event.redacts) {
+                    console.log("Removing event " + event.redacts);
+                    eventList.splice(i, 1);
+                    break;
+                }
+            }
+
+            // broadcast the redaction so controllers can nuke this
+            console.log("Redacted an event.");
+        }
+    }
+    
     /**
      * Get the index of the event in $rootScope.events.rooms[room_id].messages
      * @param {type} room_id the room id
@@ -491,6 +560,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) {
                     case 'm.room.topic':
                         handleRoomTopic(event, isLiveEvent, isStateEvent);
                         break;
+                    case 'm.room.redaction':
+                        handleRedaction(event, isLiveEvent);
+                        break;
                     default:
                         console.log("Unable to handle event type " + event.type);
                         console.log(JSON.stringify(event, undefined, 4));

webclient/components/matrix/matrix-filter.js

@@ -47,7 +47,6 @@ angular.module('matrixFilter', [])
             else if (room.members && !isPublicRoom) {    // Do not rename public room
             
                 var user_id = matrixService.config().user_id;
-
                 // Else, build the name from its users
                 // Limit the room renaming to 1:1 room
                 if (2 === Object.keys(room.members).length) {
@@ -65,8 +64,16 @@ angular.module('matrixFilter', [])
                     
                     var otherUserId;
 
-                    if (Object.keys(room.members)[0] && Object.keys(room.members)[0] !== user_id) {
+                    if (Object.keys(room.members)[0]) {
                         otherUserId = Object.keys(room.members)[0];
+                        // this could be an invite event (from event stream)
+                        if (otherUserId === user_id && 
+                                room.members[user_id].content.membership === "invite") {
+                            // this is us being invited to this room, so the
+                            // *user_id* is the other user ID and not the state
+                            // key.
+                            otherUserId = room.members[user_id].user_id;
+                        }
                     }
                     else {
                         // it's got to be an invite, or failing that a self-chat;

webclient/components/matrix/matrix-service.js

@@ -438,6 +438,14 @@ angular.module('matrixService', [])
             return this.sendMessage(room_id, msg_id, content);
         },
 
+        redactEvent: function(room_id, event_id) {
+            var path = "/rooms/$room_id/redact/$event_id";
+            path = path.replace("$room_id", room_id);
+            path = path.replace("$event_id", event_id);
+            var content = {};
+            return doRequest("POST", path, undefined, content);
+        },
+
         // get a snapshot of the members in a room.
         getMemberList: function(room_id) {
             // Like the cmd client, escape room ids

webclient/room/room-controller.js

@@ -133,7 +133,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput'])
         // Do not autoscroll to the bottom to display the new event if the user is not at the bottom.
         // Exception: in case where the event is from the user, we want to force scroll to the bottom
         var objDiv = document.getElementById("messageTableWrapper");
-        if ((objDiv.offsetHeight + objDiv.scrollTop >= objDiv.scrollHeight) || force) {
+        // add a 10px buffer to this check so if the message list is not *quite*
+        // at the bottom it still scrolls since it basically is at the bottom.
+        if ((10 + objDiv.offsetHeight + objDiv.scrollTop >= objDiv.scrollHeight) || force) {
             
             $timeout(function() {
                 objDiv.scrollTop = objDiv.scrollHeight;
@@ -983,10 +985,45 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput'])
     };
 
     $scope.openJson = function(content) {
-        console.log("Displaying modal dialog for " + JSON.stringify(content));
+        $scope.event_selected = content;
+        // scope this so the template can check power levels and enable/disable
+        // buttons
+        $scope.pow = matrixService.getUserPowerLevel;
+
         var modalInstance = $modal.open({
-            template: "<pre>" + angular.toJson(content, true) + "</pre>"
+            templateUrl: 'eventInfoTemplate.html',
+            controller: 'EventInfoController',
+            scope: $scope
+        });
+
+        modalInstance.result.then(function(action) {
+            if (action === "redact") {
+                var eventId = $scope.event_selected.event_id;
+                console.log("Redacting event ID " + eventId);
+                matrixService.redactEvent(
+                    $scope.event_selected.room_id,
+                    eventId
+                ).then(function(response) {
+                    console.log("Redaction = " + JSON.stringify(response));
+                }, function(error) {
+                    console.error("Failed to redact event: "+JSON.stringify(error));
+                    if (error.data.error) {
+                        $scope.feedback = error.data.error;
+                    }
+                });
+            }
+        }, function() {
+            // any dismiss code
         });
     };
 
-}]);
+}])
+.controller('EventInfoController', function($scope, $modalInstance) {
+    console.log("Displaying modal dialog for >>>> " + JSON.stringify($scope.event_selected));
+    $scope.redact = function() {
+        console.log("User level = "+$scope.pow($scope.room_id, $scope.state.user_id)+
+                    " Redact level = "+$scope.events.rooms[$scope.room_id]["m.room.ops_levels"].content.redact_level);
+        console.log("Redact event >> " + JSON.stringify($scope.event_selected));
+        $modalInstance.close("redact");
+    };
+});

webclient/room/room.html

@@ -1,5 +1,18 @@
 <div ng-controller="RoomController" data-ng-init="onInit()" class="room" style="height: 100%;">
 
+    <script type="text/ng-template" id="eventInfoTemplate.html">
+        <div class="modal-body">
+            <pre> {{event_selected | json}} </pre>
+        </div>
+        <div class="modal-footer">
+            <button ng-click="redact()" type="button" class="btn btn-danger" 
+             ng-disabled="!events.rooms[room_id]['m.room.ops_levels'].content.redact_level || !pow(room_id, state.user_id) || pow(room_id, state.user_id) < events.rooms[room_id]['m.room.ops_levels'].content.redact_level"
+             title="Delete this event on all home servers. This cannot be undone.">
+                Redact
+            </button>        
+        </div>
+    </script>
+
     <div id="roomHeader">
         <a href ng-click="goToPage('/')"><img src="img/logo-small.png" width="100" height="43" alt="[matrix]"/></a>
         <div class="roomHeaderInfo">
@@ -83,11 +96,11 @@
                          ng-hide="events.rooms[room_id].messages[$index - 1].user_id === msg.user_id || msg.user_id === state.user_id"/>
                 </td>
                 <td ng-class="(!msg.content.membership && ('m.room.topic' !== msg.type && 'm.room.name' !== msg.type))? (msg.content.msgtype === 'm.emote' ? 'emote text' : 'text') : 'membership text'">
-                    <div class="bubble">
+                    <div class="bubble" ng-click="openJson(msg)">
-                        <span ng-if="'join' === msg.content.membership && msg.changedKey === 'membership'" ng-click="openJson(msg)">
+                        <span ng-if="'join' === msg.content.membership && msg.changedKey === 'membership'">
                             {{ members[msg.state_key].displayname || msg.state_key }} joined
                         </span>
-                        <span ng-if="'leave' === msg.content.membership && msg.changedKey === 'membership'" ng-click="openJson(msg)">
+                        <span ng-if="'leave' === msg.content.membership && msg.changedKey === 'membership'">
                             <span ng-if="msg.user_id === msg.state_key">
                                 {{ members[msg.state_key].displayname || msg.state_key }} left
                             </span>
@@ -101,7 +114,7 @@
                             </span>
                         </span>
                         <span ng-if="'invite' === msg.content.membership && msg.changedKey === 'membership' || 
-                                     'ban' === msg.content.membership && msg.changedKey === 'membership'" ng-click="openJson(msg)">
+                                     'ban' === msg.content.membership && msg.changedKey === 'membership'">
                             {{ members[msg.user_id].displayname || msg.user_id }}
                             {{ {"invite": "invited", "ban": "banned"}[msg.content.membership] }}
                             {{ members[msg.state_key].displayname || msg.state_key }}
@@ -109,25 +122,24 @@
                                 : {{ msg.content.reason }}
                             </span>
                         </span>                        
-                        <span ng-if="msg.changedKey === 'displayname'" ng-click="openJson(msg)">
+                        <span ng-if="msg.changedKey === 'displayname'">
                             {{ msg.user_id }} changed their display name from {{ msg.prev_content.displayname }} to {{ msg.content.displayname }}
                         </span>
                         
                         <span ng-show='msg.content.msgtype === "m.emote"'
                               ng-class="msg.echo_msg_state"
                               ng-bind-html="'* ' + (members[msg.user_id].displayname || msg.user_id) + ' ' + msg.content.body | linky:'_blank'"
-                              ng-click="openJson(msg)"/>
+                              />
                         
                         <span ng-show='msg.content.msgtype === "m.text"' 
                               class="message"
-                              ng-click="openJson(msg)"
                               ng-class="containsBingWord(msg.content.body) && msg.user_id != state.user_id ? msg.echo_msg_state + ' messageBing' : msg.echo_msg_state"
                               ng-bind-html="(msg.content.msgtype === 'm.text' && msg.type === 'm.room.message' && msg.content.format === 'org.matrix.custom.html') ? 
                                                                                         (msg.content.formatted_body | unsanitizedLinky) :
                                              (msg.content.msgtype === 'm.text' && msg.type === 'm.room.message') ? (msg.content.body | linky:'_blank') : '' "/>
 
-                        <span ng-show='msg.type === "m.call.invite" && msg.user_id == state.user_id' ng-click="openJson(msg)">Outgoing Call{{ isWebRTCSupported ? '' : ' (But your browser does not support VoIP)' }}</span>
+                        <span ng-show='msg.type === "m.call.invite" && msg.user_id == state.user_id'>Outgoing Call{{ isWebRTCSupported ? '' : ' (But your browser does not support VoIP)' }}</span>
-                        <span ng-show='msg.type === "m.call.invite" && msg.user_id != state.user_id' ng-click="openJson(msg)">Incoming Call{{ isWebRTCSupported ? '' : ' (But your browser does not support VoIP)' }}</span>
+                        <span ng-show='msg.type === "m.call.invite" && msg.user_id != state.user_id'>Incoming Call{{ isWebRTCSupported ? '' : ' (But your browser does not support VoIP)' }}</span>
 
                         <div ng-show='msg.content.msgtype === "m.image"'>
                             <div ng-hide='msg.content.thumbnail_url' ng-style="msg.content.body.h && { 'height' : (msg.content.body.h < 320) ? msg.content.body.h : 320}">
@@ -135,15 +147,15 @@
                             </div>
                             <div ng-show='msg.content.thumbnail_url' ng-style="{ 'height' : msg.content.thumbnail_info.h }">
                                 <img class="image mouse-pointer" ng-src="{{ msg.content.thumbnail_url }}"
-                                     ng-click="$parent.fullScreenImageURL = msg.content.url"/>
+                                     ng-click="$parent.fullScreenImageURL = msg.content.url; $event.stopPropagation();"/>
                             </div>
                         </div>
 
-                        <span ng-if="'m.room.topic' === msg.type" ng-click="openJson(msg)">
+                        <span ng-if="'m.room.topic' === msg.type">
                             {{ members[msg.user_id].displayname || msg.user_id }} changed the topic to: {{ msg.content.topic }}
                         </span>
 
-                        <span ng-if="'m.room.name' === msg.type" ng-click="openJson(msg)">
+                        <span ng-if="'m.room.name' === msg.type">
                             {{ members[msg.user_id].displayname || msg.user_id }} changed the room name to: {{ msg.content.name }}
                         </span>