From aa3c9c7bd0736bca1b3626c87535192b89431583 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Aug 2015 10:57:47 +0100
Subject: [PATCH 01/34] Don't allow people to register user ids which only
 differ by case to an existing one

---
 synapse/handlers/register.py    |  4 ++--
 synapse/storage/registration.py | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 39392d9fd..86390a367 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -57,8 +57,8 @@ class RegistrationHandler(BaseHandler):
 
         yield self.check_user_id_is_valid(user_id)
 
-        u = yield self.store.get_user_by_id(user_id)
-        if u:
+        users = yield self.store.get_users_by_id_case_insensitive(user_id)
+        if users:
             raise SynapseError(
                 400,
                 "User ID already taken.",
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index bf803f2c6..25adecaf6 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -98,6 +98,17 @@ class RegistrationStore(SQLBaseStore):
             allow_none=True,
         )
 
+    def get_users_by_id_case_insensitive(self, user_id):
+        def f(txn):
+            sql = (
+                "SELECT name, password_hash FROM users"
+                " WHERE name = lower(?)"
+            )
+            txn.execute(sql, (user_id,))
+            return self.cursor_to_dict(txn)
+
+        return self.runInteraction("get_users_by_id_case_insensitive", f)
+
     @defer.inlineCallbacks
     def user_set_password_hash(self, user_id, password_hash):
         """

From 42f12ad92f5bc372569f15ffc81e9cf8146d2ac6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Aug 2015 11:34:43 +0100
Subject: [PATCH 02/34] When logging in fetch user by user_id case
 insensitively, *unless* there are multiple case insensitive matches, in which
 case require the exact user_id

---
 synapse/handlers/auth.py        | 31 +++++++++++++++++++++++--------
 synapse/rest/client/v1/login.py |  5 +++--
 synapse/storage/registration.py |  7 +++++--
 3 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index ff2c66f44..058a0f416 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -162,7 +162,8 @@ class AuthHandler(BaseHandler):
         if not user_id.startswith('@'):
             user_id = UserID.create(user_id, self.hs.hostname).to_string()
 
-        yield self._check_password(user_id, password)
+        user_id, password_hash = yield self._find_user_id_and_pwd_hash(user_id)
+        self._check_password(user_id, password, password_hash)
         defer.returnValue(user_id)
 
     @defer.inlineCallbacks
@@ -283,23 +284,37 @@ class AuthHandler(BaseHandler):
             StoreError if there was a problem storing the token.
             LoginError if there was an authentication problem.
         """
-        yield self._check_password(user_id, password)
+        user_id, password_hash = yield self._find_user_id_and_pwd_hash(user_id)
+        self._check_password(user_id, password, password_hash)
 
         reg_handler = self.hs.get_handlers().registration_handler
         access_token = reg_handler.generate_token(user_id)
         logger.info("Logging in user %s", user_id)
         yield self.store.add_access_token_to_user(user_id, access_token)
-        defer.returnValue(access_token)
+        defer.returnValue((user_id, access_token))
 
     @defer.inlineCallbacks
-    def _check_password(self, user_id, password):
-        """Checks that user_id has passed password, raises LoginError if not."""
-        user_info = yield self.store.get_user_by_id(user_id=user_id)
-        if not user_info:
+    def _find_user_id_and_pwd_hash(self, user_id):
+        user_infos = yield self.store.get_users_by_id_case_insensitive(user_id)
+        if not user_infos:
             logger.warn("Attempted to login as %s but they do not exist", user_id)
             raise LoginError(403, "", errcode=Codes.FORBIDDEN)
 
-        stored_hash = user_info["password_hash"]
+        if len(user_infos) > 1:
+            if user_id not in user_infos:
+                logger.warn(
+                    "Attempted to login as %s but it matches more than one user "
+                    "inexactly: %r",
+                    user_id, user_infos.keys()
+                )
+                raise LoginError(403, "", errcode=Codes.FORBIDDEN)
+
+            defer.returnValue((user_id, user_infos[user_id]))
+        else:
+            defer.returnValue(user_infos.popitem())
+
+    def _check_password(self, user_id, password, stored_hash):
+        """Checks that user_id has passed password, raises LoginError if not."""
         if not bcrypt.checkpw(password, stored_hash):
             logger.warn("Failed password login for user %s", user_id)
             raise LoginError(403, "", errcode=Codes.FORBIDDEN)
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index 0d5eafd0f..2444f2736 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -83,9 +83,10 @@ class LoginRestServlet(ClientV1RestServlet):
 
         if not user_id.startswith('@'):
             user_id = UserID.create(
-                user_id, self.hs.hostname).to_string()
+                user_id, self.hs.hostname
+            ).to_string()
 
-        token = yield self.handlers.auth_handler.login_with_password(
+        user_id, token = yield self.handlers.auth_handler.login_with_password(
             user_id=user_id,
             password=login_submission["password"])
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 25adecaf6..586628579 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -99,13 +99,16 @@ class RegistrationStore(SQLBaseStore):
         )
 
     def get_users_by_id_case_insensitive(self, user_id):
+        """Gets users that match user_id case insensitively.
+        Returns a mapping of user_id -> password_hash.
+        """
         def f(txn):
             sql = (
                 "SELECT name, password_hash FROM users"
-                " WHERE name = lower(?)"
+                " WHERE lower(name) = lower(?)"
             )
             txn.execute(sql, (user_id,))
-            return self.cursor_to_dict(txn)
+            return dict(txn.fetchall())
 
         return self.runInteraction("get_users_by_id_case_insensitive", f)
 

From fd5ad0f00ec963e9722d9f5bbe526dc84038e408 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Aug 2015 11:45:43 +0100
Subject: [PATCH 03/34] Doc string

---
 synapse/handlers/auth.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 058a0f416..602c5bcd8 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -295,6 +295,12 @@ class AuthHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _find_user_id_and_pwd_hash(self, user_id):
+        """Checks to see if a user with the given id exists. Will check case
+        insensitively, but will throw if there are multiple inexact matches.
+
+        Returns:
+            tuple: A 2-tuple of `(canonical_user_id, password_hash)`
+        """
         user_infos = yield self.store.get_users_by_id_case_insensitive(user_id)
         if not user_infos:
             logger.warn("Attempted to login as %s but they do not exist", user_id)

From b9490e8cbb384a6ec1612a26305c2700a0365046 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Aug 2015 13:07:37 +0100
Subject: [PATCH 04/34] Upate changelog

---
 CHANGES.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index e6d1a3730..8eca8fde0 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -10,6 +10,9 @@ General:
   (PR #208)
 * Add support for logging in with email address (PR #234)
 * Add support for new ``m.room.canonical_alias`` event. (PR #233)
+* Change synapse to treat user IDs case insensitively during registration and
+  login. (If two users already exist with case insensitive matching user ids,
+  synapse will continue to require them to specify their user ids exactly.)
 * Error if a user tries to register with an email already in use. (PR #211)
 * Add extra and improve existing caches  (PR #212, #219, #226, #228)
 * Batch various storage request (PR #226, #228)

From 0f6a25f670cd7a2c319910e3a5e847f1d50bae1a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Aug 2015 13:07:56 +0100
Subject: [PATCH 05/34] Upate changelog

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 8eca8fde0..287b04c3f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,4 +1,4 @@
-Changes in synapse v0.10.0-rc1 (2015-08-20)
+Changes in synapse v0.10.0-rc1 (2015-08-21)
 ===========================================
 
 Also see v0.9.4-rc1 changelog, which has been amalgamated into this release.

From 1bd1a43073f2b1f657f4c9a5bab8326c1aa19c53 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Aug 2015 14:30:34 +0100
Subject: [PATCH 06/34] Actually check if event_id isn't returned by
 _get_state_groups

---
 synapse/storage/state.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index c9110e630..9630efcfc 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -403,8 +403,15 @@ class StateStore(SQLBaseStore):
                 state_dict = results[group]
 
             for event_id in state_ids:
-                state_event = state_events[event_id]
-                state_dict[(state_event.type, state_event.state_key)] = state_event
+                try:
+                    state_event = state_events[event_id]
+                    state_dict[(state_event.type, state_event.state_key)] = state_event
+                except KeyError:
+                    # Hmm. So we do don't have that state event? Interesting.
+                    logger.warn(
+                        "Can't find state event %r for state group %r",
+                        event_id, group,
+                    )
 
             self._state_group_cache.update(
                 cache_seq_num,

From 457970c724b7d31f9ec6c153a6011a57b21ef157 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Sun, 23 Aug 2015 13:44:23 +0100
Subject: [PATCH 07/34] Don't insert events into 'event_*_extremeties' tables
 if they're outliers

---
 synapse/storage/event_federation.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 25cc84eb9..dda3027b6 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -331,7 +331,10 @@ class EventFederationStore(SQLBaseStore):
 
         txn.executemany(
             query,
-            [(ev.event_id, ev.room_id, ev.event_id) for ev in events]
+            [
+                (ev.event_id, ev.room_id, ev.event_id) for ev in events
+                if not ev.internal_metadata.is_outlier()
+            ]
         )
 
         query = (
@@ -358,7 +361,10 @@ class EventFederationStore(SQLBaseStore):
         )
         txn.executemany(
             query,
-            [(ev.event_id, ev.room_id) for ev in events]
+            [
+                (ev.event_id, ev.room_id) for ev in events
+                if not ev.internal_metadata.is_outlier()
+            ]
         )
 
         for room_id in events_by_room:

From f8f3d72e2b6e5b38bb6ab8e057ede454d77d114f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 24 Aug 2015 16:19:43 +0100
Subject: [PATCH 08/34] Don't make pushers handle presence/typing events

---
 synapse/handlers/events.py | 10 ++++++++--
 synapse/notifier.py        |  7 ++++++-
 synapse/push/__init__.py   |  8 +++++---
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index f9ca2f863..891502c04 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -49,7 +49,12 @@ class EventStreamHandler(BaseHandler):
     @defer.inlineCallbacks
     @log_function
     def get_stream(self, auth_user_id, pagin_config, timeout=0,
-                   as_client_event=True, affect_presence=True):
+                   as_client_event=True, affect_presence=True,
+                   only_room_events=False):
+        """Fetches the events stream for a given user.
+
+        If `only_room_events` is `True` only room events will be returned.
+        """
         auth_user = UserID.from_string(auth_user_id)
 
         try:
@@ -89,7 +94,8 @@ class EventStreamHandler(BaseHandler):
                 timeout = random.randint(int(timeout*0.9), int(timeout*1.1))
 
             events, tokens = yield self.notifier.get_events_for(
-                auth_user, room_ids, pagin_config, timeout
+                auth_user, room_ids, pagin_config, timeout,
+                only_room_events=only_room_events
             )
 
             time_now = self.clock.time_msec()
diff --git a/synapse/notifier.py b/synapse/notifier.py
index dbd8efe9f..f998fc83b 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -328,10 +328,13 @@ class Notifier(object):
         defer.returnValue(result)
 
     @defer.inlineCallbacks
-    def get_events_for(self, user, rooms, pagination_config, timeout):
+    def get_events_for(self, user, rooms, pagination_config, timeout,
+                       only_room_events=False):
         """ For the given user and rooms, return any new events for them. If
         there are no new events wait for up to `timeout` milliseconds for any
         new events to happen before returning.
+
+        If `only_room_events` is `True` only room events will be returned.
         """
         from_token = pagination_config.from_token
         if not from_token:
@@ -352,6 +355,8 @@ class Notifier(object):
                 after_id = getattr(after_token, keyname)
                 if before_id == after_id:
                     continue
+                if only_room_events and name != "room":
+                    continue
                 new_events, new_key = yield source.get_new_events_for_user(
                     user, getattr(from_token, keyname), limit,
                 )
diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py
index 13002e0db..f1952b5a0 100644
--- a/synapse/push/__init__.py
+++ b/synapse/push/__init__.py
@@ -249,7 +249,9 @@ class Pusher(object):
             # we fail to dispatch the push)
             config = PaginationConfig(from_token=None, limit='1')
             chunk = yield self.evStreamHandler.get_stream(
-                self.user_name, config, timeout=0)
+                self.user_name, config, timeout=0, affect_presence=False,
+                only_room_events=True
+            )
             self.last_token = chunk['end']
             self.store.update_pusher_last_token(
                 self.app_id, self.pushkey, self.user_name, self.last_token
@@ -280,8 +282,8 @@ class Pusher(object):
         config = PaginationConfig(from_token=from_tok, limit='1')
         timeout = (300 + random.randint(-60, 60)) * 1000
         chunk = yield self.evStreamHandler.get_stream(
-            self.user_name, config,
-            timeout=timeout, affect_presence=False
+            self.user_name, config, timeout=timeout, affect_presence=False,
+            only_room_events=True
         )
 
         # limiting to 1 may get 1 event plus 1 presence event, so

From 51c53369a318262ecc3adc3777be8b838509d19d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 24 Aug 2015 16:38:20 +0100
Subject: [PATCH 09/34] Do auth checks *before* persisting the event

---
 synapse/handlers/_base.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index e91f1129d..cb992143f 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -107,6 +107,22 @@ class BaseHandler(object):
         if not suppress_auth:
             self.auth.check(event, auth_events=context.current_state)
 
+        if event.type == EventTypes.CanonicalAlias:
+            # Check the alias is acually valid (at this time at least)
+            room_alias_str = event.content.get("alias", None)
+            if room_alias_str:
+                room_alias = RoomAlias.from_string(room_alias_str)
+                directory_handler = self.hs.get_handlers().directory_handler
+                mapping = yield directory_handler.get_association(room_alias)
+
+                if mapping["room_id"] != event.room_id:
+                    raise SynapseError(
+                        400,
+                        "Room alias %s does not point to the room" % (
+                            room_alias_str,
+                        )
+                    )
+
         (event_stream_id, max_stream_id) = yield self.store.persist_event(
             event, context=context
         )
@@ -130,22 +146,6 @@ class BaseHandler(object):
                         returned_invite.signatures
                     )
 
-        if event.type == EventTypes.CanonicalAlias:
-            # Check the alias is acually valid (at this time at least)
-            room_alias_str = event.content.get("alias", None)
-            if room_alias_str:
-                room_alias = RoomAlias.from_string(room_alias_str)
-                directory_handler = self.hs.get_handlers().directory_handler
-                mapping = yield directory_handler.get_association(room_alias)
-
-                if mapping["room_id"] != event.room_id:
-                    raise SynapseError(
-                        400,
-                        "Room alias %s does not point to the room" % (
-                            room_alias_str,
-                        )
-                    )
-
         destinations = set(extra_destinations)
         for k, s in context.current_state.items():
             try:

From 571ac105e6bf9c0de328bb22b095b97fbd09e5ae Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 24 Aug 2015 17:10:45 +0100
Subject: [PATCH 10/34] Bump version and changelog

---
 CHANGES.rst         | 10 ++++++++++
 synapse/__init__.py |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 287b04c3f..f4946c6df 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,13 @@
+Changes in synapse v0.10.0-rc2 (2015-08-24)
+===========================================
+
+* Fix bug where we incorrectly populated the ``event_forward_extremities``
+  table, resulting in problems joining large remote rooms (e.g.
+  ``#matrix:matrix.org``)
+* Reduce the number of times we wake up pushers by not listening for presence
+  or typing events, reducing the CPU cost of each pusher.
+
+
 Changes in synapse v0.10.0-rc1 (2015-08-21)
 ===========================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 5853165a2..b4eab89d4 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.10.0-rc1"
+__version__ = "0.10.0-rc2"

From 86cef6a91b330d99b86cf2eacf75a446ce2e2955 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 12:01:23 +0100
Subject: [PATCH 11/34] Allow specifying a directory to host a web client from

---
 synapse/app/homeserver.py | 8 +++++---
 synapse/config/server.py  | 1 +
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index f04493f92..2c59457cd 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -97,9 +97,11 @@ class SynapseHomeServer(HomeServer):
         return JsonResource(self)
 
     def build_resource_for_web_client(self):
-        import syweb
-        syweb_path = os.path.dirname(syweb.__file__)
-        webclient_path = os.path.join(syweb_path, "webclient")
+        webclient_path = self.get_config().web_client_location
+        if not webclient_path:
+            import syweb
+            syweb_path = os.path.dirname(syweb.__file__)
+            webclient_path = os.path.join(syweb_path, "webclient")
         # GZip is disabled here due to
         # https://twistedmatrix.com/trac/ticket/7678
         # (It can stay enabled for the API resources: they call
diff --git a/synapse/config/server.py b/synapse/config/server.py
index f9a3b5f15..a03e55c22 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -22,6 +22,7 @@ class ServerConfig(Config):
         self.server_name = config["server_name"]
         self.pid_file = self.abspath(config.get("pid_file"))
         self.web_client = config["web_client"]
+        self.web_client_location = config.get("web_client_location", None)
         self.soft_file_limit = config["soft_file_limit"]
         self.daemonize = config.get("daemonize")
         self.print_pidfile = config.get("print_pidfile")

From d9088c923f32fe1d2b6027320bb8fd5e46e1a428 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 13:34:50 +0100
Subject: [PATCH 12/34] Remove dependency on matrix-angular-sdk

---
 synapse/app/homeserver.py      | 9 ++++++++-
 synapse/python_dependencies.py | 6 +-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 2c59457cd..6255e9676 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -99,7 +99,14 @@ class SynapseHomeServer(HomeServer):
     def build_resource_for_web_client(self):
         webclient_path = self.get_config().web_client_location
         if not webclient_path:
-            import syweb
+            try:
+                import syweb
+            except ImportError:
+                quit_with_error(
+                    "Could not find a webclient. Please either install syweb\n"
+                    "or configure the location of the source to server via\n"
+                    "the config option `web_client_location`"
+                )
             syweb_path = os.path.dirname(syweb.__file__)
             webclient_path = os.path.join(syweb_path, "webclient")
         # GZip is disabled here due to
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index fa06480ad..61a362bb1 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -34,11 +34,7 @@ REQUIREMENTS = {
     "blist": ["blist"],
     "pysaml2": ["saml2"],
 }
-CONDITIONAL_REQUIREMENTS = {
-    "web_client": {
-        "matrix_angular_sdk>=0.6.6": ["syweb>=0.6.6"],
-    }
-}
+CONDITIONAL_REQUIREMENTS = {}
 
 
 def requirements(config=None, include_conditional=False):

From 8b52fe48b5f222bb444c93307c9681028693db2b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 14:10:31 +0100
Subject: [PATCH 13/34] Revert previous commit. Instead, always download
 matrix-angular-sdk as a requirement, but don't complain (when we do
 check_requirements) if we don't have it when we start synapse.

---
 synapse/python_dependencies.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 61a362bb1..82e28133e 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -34,13 +34,17 @@ REQUIREMENTS = {
     "blist": ["blist"],
     "pysaml2": ["saml2"],
 }
-CONDITIONAL_REQUIREMENTS = {}
+CONDITIONAL_REQUIREMENTS = {
+    "web_client": {
+        "matrix_angular_sdk>=0.6.6": ["syweb>=0.6.6"],
+    }
+}
 
 
 def requirements(config=None, include_conditional=False):
     reqs = REQUIREMENTS.copy()
-    for key, req in CONDITIONAL_REQUIREMENTS.items():
-        if (config and getattr(config, key)) or include_conditional:
+    if include_conditional:
+        for _, req in CONDITIONAL_REQUIREMENTS.items():
             reqs.update(req)
     return reqs
 

From 37403ab06cd8c82c2910d44f1b78f2338c2dab9b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 14:19:09 +0100
Subject: [PATCH 14/34] Update the log message

---
 synapse/app/homeserver.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 6255e9676..7f3e44172 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -103,9 +103,15 @@ class SynapseHomeServer(HomeServer):
                 import syweb
             except ImportError:
                 quit_with_error(
-                    "Could not find a webclient. Please either install syweb\n"
-                    "or configure the location of the source to server via\n"
-                    "the config option `web_client_location`"
+                    "Could not find a webclient.\n\n"
+                    "Please either install the matrix-angular-sdk or configure\n"
+                    "the location of the source to serve via the configuration\n"
+                    "option `web_client_location`\n\n"
+                    "To install the `matrix-angular-sdk` via pip, run:\n\n"
+                    "    pip install 'matrix-angular-sdk'\n"
+                    "\n"
+                    "You can also disable hosting of the webclient via the\n"
+                    "configuration option `web_client`\n"
                 )
             syweb_path = os.path.dirname(syweb.__file__)
             webclient_path = os.path.join(syweb_path, "webclient")
@@ -271,8 +277,7 @@ def quit_with_error(error_string):
     line_length = max([len(l) for l in message_lines]) + 2
     sys.stderr.write("*" * line_length + '\n')
     for line in message_lines:
-        if line.strip():
-            sys.stderr.write(" %s\n" % (line.strip(),))
+        sys.stderr.write(" %s\n" % (line.rstrip(),))
     sys.stderr.write("*" * line_length + '\n')
     sys.exit(1)
 

From f63208a1c073a0c1b9de276e8173a3ebd94b9a75 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 15:13:35 +0100
Subject: [PATCH 15/34] Add utility to parse config and print out a key

Usage:

```
$ python -m synapse.config read server_name -c homeserver.yaml
localhost
```
---
 synapse/config/__main__.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 synapse/config/__main__.py

diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py
new file mode 100644
index 000000000..725983a71
--- /dev/null
+++ b/synapse/config/__main__.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if __name__ == "__main__":
+    import sys
+    from homeserver import HomeServerConfig
+
+    action = sys.argv[1]
+
+    if action == "read":
+        key = sys.argv[2]
+        config = HomeServerConfig.load_config("", sys.argv[3:])
+
+        print getattr(config, key)
+        sys.exit(0)
+    else:
+        sys.stderr.write("Unknown command %r", action)
+        sys.exit(1)

From d33f31d741f703758c092e311bae263ff3c3ef64 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 15:33:23 +0100
Subject: [PATCH 16/34] Print the correct pip install line when failing due to
 lack of matrix-angular-sdk

---
 synapse/app/homeserver.py      |  7 ++++---
 synapse/python_dependencies.py | 10 +++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 7f3e44172..ff7807c2e 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -16,7 +16,7 @@
 
 import sys
 sys.dont_write_bytecode = True
-from synapse.python_dependencies import check_requirements
+from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS
 
 if __name__ == '__main__':
     check_requirements()
@@ -108,10 +108,11 @@ class SynapseHomeServer(HomeServer):
                     "the location of the source to serve via the configuration\n"
                     "option `web_client_location`\n\n"
                     "To install the `matrix-angular-sdk` via pip, run:\n\n"
-                    "    pip install 'matrix-angular-sdk'\n"
+                    "    pip install '%(dep)s'\n"
                     "\n"
                     "You can also disable hosting of the webclient via the\n"
                     "configuration option `web_client`\n"
+                    % {"dep": DEPENDENCY_LINKS["matrix-angular-sdk"]}
                 )
             syweb_path = os.path.dirname(syweb.__file__)
             webclient_path = os.path.join(syweb_path, "webclient")
@@ -274,7 +275,7 @@ class SynapseHomeServer(HomeServer):
 
 def quit_with_error(error_string):
     message_lines = error_string.split("\n")
-    line_length = max([len(l) for l in message_lines]) + 2
+    line_length = max([len(l) for l in message_lines if len(l) < 80]) + 2
     sys.stderr.write("*" * line_length + '\n')
     for line in message_lines:
         sys.stderr.write(" %s\n" % (line.rstrip(),))
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 82e28133e..d7e3a686f 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -52,18 +52,18 @@ def requirements(config=None, include_conditional=False):
 def github_link(project, version, egg):
     return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)
 
-DEPENDENCY_LINKS = [
-    github_link(
+DEPENDENCY_LINKS = {
+    "syutil": github_link(
         project="matrix-org/syutil",
         version="v0.0.7",
         egg="syutil-0.0.7",
     ),
-    github_link(
+    "matrix-angular-sdk": github_link(
         project="matrix-org/matrix-angular-sdk",
         version="v0.6.6",
         egg="matrix_angular_sdk-0.6.6",
     ),
-]
+}
 
 
 class MissingRequirementError(Exception):
@@ -131,7 +131,7 @@ def check_requirements(config=None):
 def list_requirements():
     result = []
     linked = []
-    for link in DEPENDENCY_LINKS:
+    for link in DEPENDENCY_LINKS.values():
         egg = link.split("#egg=")[1]
         linked.append(egg.split('-')[0])
         result.append(link)

From 1d1c303b9b3790256d5ebf2d2e93528a23e8270a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 15:39:16 +0100
Subject: [PATCH 17/34] Fix typo when using sys.stderr.write

---
 synapse/config/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py
index 725983a71..f822d1203 100644
--- a/synapse/config/__main__.py
+++ b/synapse/config/__main__.py
@@ -26,5 +26,5 @@ if __name__ == "__main__":
         print getattr(config, key)
         sys.exit(0)
     else:
-        sys.stderr.write("Unknown command %r", action)
+        sys.stderr.write("Unknown command %r\n" % (action,))
         sys.exit(1)

From bfb66773a438da2f8fb7192b7383f6efe65d87ec Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 16:25:54 +0100
Subject: [PATCH 18/34] Allow specifying directories as config files

---
 synapse/config/_base.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 73f695995..fd5080a3c 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -131,7 +131,8 @@ class Config(object):
             "-c", "--config-path",
             action="append",
             metavar="CONFIG_FILE",
-            help="Specify config file"
+            help="Specify config file. Can be given multiple times and"
+                 " may specify directories containing *.yaml files."
         )
         config_parser.add_argument(
             "--generate-config",
@@ -151,14 +152,31 @@ class Config(object):
 
         generate_keys = config_args.generate_keys
 
+        config_files = []
+        if config_args.config_path:
+            for config_path in config_args.config_path:
+                if os.path.isdir(config_path):
+                    # We accept specifying directories as config paths, we search
+                    # inside that directory for all files matching *.yaml, and then
+                    # we apply them in *sorted* order.
+                    config_files.extend(sorted(
+                        os.path.join(config_path, entry)
+                        for entry in os.listdir(config_path)
+                        if entry.endswith(".yaml") and os.path.isfile(
+                            os.path.join(config_path, entry)
+                        )
+                    ))
+                else:
+                    config_files.append(config_path)
+
         if config_args.generate_config:
-            if not config_args.config_path:
+            if not config_files:
                 config_parser.error(
                     "Must supply a config file.\nA config file can be automatically"
                     " generated using \"--generate-config -H SERVER_NAME"
                     " -c CONFIG-FILE\""
                 )
-            (config_path,) = config_args.config_path
+            (config_path,) = config_files
             if not os.path.exists(config_path):
                 config_dir_path = os.path.dirname(config_path)
                 config_dir_path = os.path.abspath(config_dir_path)
@@ -202,7 +220,7 @@ class Config(object):
         obj.invoke_all("add_arguments", parser)
         args = parser.parse_args(remaining_args)
 
-        if not config_args.config_path:
+        if not config_files:
             config_parser.error(
                 "Must supply a config file.\nA config file can be automatically"
                 " generated using \"--generate-config -H SERVER_NAME"
@@ -213,8 +231,8 @@ class Config(object):
         config_dir_path = os.path.abspath(config_dir_path)
 
         specified_config = {}
-        for config_path in config_args.config_path:
-            yaml_config = cls.read_config_file(config_path)
+        for config_file in config_files:
+            yaml_config = cls.read_config_file(config_file)
             specified_config.update(yaml_config)
 
         server_name = specified_config["server_name"]

From af7c1397d1ba402bb58482c2204484d23d33a403 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 16:58:01 +0100
Subject: [PATCH 19/34] Add config option to specify where generated files
 should be dumped

---
 synapse/config/_base.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index fd5080a3c..1bc2e61ee 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -144,6 +144,12 @@ class Config(object):
             action="store_true",
             help="Generate any missing key files then exit"
         )
+        config_parser.add_argument(
+            "--generated-directory",
+            metavar="DIRECTORY",
+            help="Used with 'generate-*' options to specify where generated"
+                 " files (such as certs and signing keys) should be stored."
+        )
         config_parser.add_argument(
             "-H", "--server-name",
             help="The server name to generate a config file for"
@@ -178,7 +184,10 @@ class Config(object):
                 )
             (config_path,) = config_files
             if not os.path.exists(config_path):
-                config_dir_path = os.path.dirname(config_path)
+                if config_args.generated_directory:
+                    config_dir_path = config_args.generated_directory
+                else:
+                    config_dir_path = os.path.dirname(config_path)
                 config_dir_path = os.path.abspath(config_dir_path)
 
                 server_name = config_args.server_name
@@ -227,7 +236,10 @@ class Config(object):
                 " -c CONFIG-FILE\""
             )
 
-        config_dir_path = os.path.dirname(config_args.config_path[-1])
+        if config_args.generated_directory:
+            config_dir_path = config_args.generated_directory
+        else:
+            config_dir_path = os.path.dirname(config_args.config_path[-1])
         config_dir_path = os.path.abspath(config_dir_path)
 
         specified_config = {}

From 3e1029fe8050edd8f6aed57bdf6507a12ffb0a0c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 17:08:23 +0100
Subject: [PATCH 20/34] Warn if we encounter unexpected files in config
 directories

---
 synapse/config/_base.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index fd5080a3c..2f218d4a7 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -159,13 +159,23 @@ class Config(object):
                     # We accept specifying directories as config paths, we search
                     # inside that directory for all files matching *.yaml, and then
                     # we apply them in *sorted* order.
-                    config_files.extend(sorted(
-                        os.path.join(config_path, entry)
-                        for entry in os.listdir(config_path)
-                        if entry.endswith(".yaml") and os.path.isfile(
-                            os.path.join(config_path, entry)
-                        )
-                    ))
+                    files = []
+                    for entry in os.listdir(config_path):
+                        entry_path = os.path.join(config_path, entry)
+                        if not os.path.isfile(entry_path):
+                            print (
+                                "Found subdirectory in config directory: %r. IGNORING."
+                            ) % (entry_path, )
+                            continue
+
+                        if not entry.endswith(".yaml"):
+                            print (
+                                "Found file in config directory that does not"
+                                " end in '.yaml': %r. IGNORING."
+                            ) % (entry_path, )
+                            continue
+
+                    config_files.extend(sorted(files))
                 else:
                     config_files.append(config_path)
 

From 82145912c330528ec85276467f4ea38362e796b8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 17:31:22 +0100
Subject: [PATCH 21/34] s/--generated-directory/--keys-directory/

---
 synapse/config/_base.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 1bc2e61ee..93433c075 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -145,10 +145,10 @@ class Config(object):
             help="Generate any missing key files then exit"
         )
         config_parser.add_argument(
-            "--generated-directory",
+            "--keys-directory",
             metavar="DIRECTORY",
-            help="Used with 'generate-*' options to specify where generated"
-                 " files (such as certs and signing keys) should be stored."
+            help="Used with 'generate-*' options to specify where files such as"
+                 " certs and signing keys should be stored in."
         )
         config_parser.add_argument(
             "-H", "--server-name",
@@ -184,8 +184,8 @@ class Config(object):
                 )
             (config_path,) = config_files
             if not os.path.exists(config_path):
-                if config_args.generated_directory:
-                    config_dir_path = config_args.generated_directory
+                if config_args.keys_directory:
+                    config_dir_path = config_args.keys_directory
                 else:
                     config_dir_path = os.path.dirname(config_path)
                 config_dir_path = os.path.abspath(config_dir_path)
@@ -236,8 +236,8 @@ class Config(object):
                 " -c CONFIG-FILE\""
             )
 
-        if config_args.generated_directory:
-            config_dir_path = config_args.generated_directory
+        if config_args.keys_directory:
+            config_dir_path = config_args.keys_directory
         else:
             config_dir_path = os.path.dirname(config_args.config_path[-1])
         config_dir_path = os.path.abspath(config_dir_path)

From 3f6f74686a564face040bb0b4daf57a3112161e2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 17:37:21 +0100
Subject: [PATCH 22/34] Update config doc

---
 synapse/config/_base.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 93433c075..e92a6c779 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -148,7 +148,8 @@ class Config(object):
             "--keys-directory",
             metavar="DIRECTORY",
             help="Used with 'generate-*' options to specify where files such as"
-                 " certs and signing keys should be stored in."
+                 " certs and signing keys should be stored in, unless explicitly"
+                 " specified in the config."
         )
         config_parser.add_argument(
             "-H", "--server-name",

From 90fde4b8d7f2f74822df1008dadb4c31c2cde56c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Aug 2015 17:49:58 +0100
Subject: [PATCH 23/34] Bump changelog and version

---
 CHANGES.rst         | 17 +++++++++++++++++
 synapse/__init__.py |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index f4946c6df..5f16e4485 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,20 @@
+Changes in synapse v0.10.0-rc3 (2015-08-25)
+===========================================
+
+* Add ``--keys-directory`` config option to specify where files such as
+  certs and signing keys should be stored in, when using ``--generate-config``
+  or ``--generate-keys``.
+* Allow ``--config-path`` to specify a directory, causing synapse to use all
+  \*.yaml files in the directory as config files.
+* Add ``web_client_location`` config option to specify static files to be
+  hosted by synapse under ``/_matrix/client``.
+* Add helper utility to synapse to read and parse the config files and extract
+  the value of a given key. For example::
+
+    $ python -m synapse.config read server_name -c homeserver.yaml
+    localhost
+
+
 Changes in synapse v0.10.0-rc2 (2015-08-24)
 ===========================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index b4eab89d4..41e3283f0 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.10.0-rc2"
+__version__ = "0.10.0-rc3"

From f4d552589e3cb815144dea646140db66d845a237 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 26 Aug 2015 10:51:08 +0100
Subject: [PATCH 24/34] Don't loop over all rooms ever in
 typing.get_new_events_for_user

---
 synapse/handlers/typing.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 026bd2b9d..1ed220d87 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -260,8 +260,8 @@ class TypingNotificationEventSource(object):
         )
 
         events = []
-        for room_id in handler._room_serials:
-            if room_id not in joined_room_ids:
+        for room_id in joined_room_ids:
+            if room_id not in handler._room_serials:
                 continue
             if handler._room_serials[room_id] <= from_key:
                 continue

From da51acf0e752badb73c036f4b1cf0ec943b6dcb1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 26 Aug 2015 11:08:23 +0100
Subject: [PATCH 25/34] Remove needless existence checks

---
 synapse/handlers/typing.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 1ed220d87..d7096aab8 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -204,15 +204,11 @@ class TypingNotificationHandler(BaseHandler):
             )
 
     def _push_update_local(self, room_id, user, typing):
-        if room_id not in self._room_serials:
-            self._room_serials[room_id] = 0
-            self._room_typing[room_id] = set()
-
-        room_set = self._room_typing[room_id]
+        room_set = self._room_typing.setdefault(room_id, set())
         if typing:
             room_set.add(user)
-        elif user in room_set:
-            room_set.remove(user)
+        else:
+            room_set.discard(user)
 
         self._latest_room_serial += 1
         self._room_serials[room_id] = self._latest_room_serial

From e85c7873dc885c18705c2a77d8487517379d64fb Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 26 Aug 2015 16:26:37 +0100
Subject: [PATCH 26/34] Allow non-ascii filenames for attachments

---
 synapse/rest/media/v1/base_resource.py   | 17 +++++++++++++----
 synapse/rest/media/v1/upload_resource.py |  6 ++----
 synapse/util/stringutils.py              |  2 ++
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index 4e21527c3..24297b20f 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -33,6 +33,7 @@ import os
 
 import cgi
 import logging
+import urllib
 
 logger = logging.getLogger(__name__)
 
@@ -181,10 +182,18 @@ class BaseMediaResource(Resource):
         if os.path.isfile(file_path):
             request.setHeader(b"Content-Type", media_type.encode("UTF-8"))
             if upload_name:
-                request.setHeader(
-                    b"Content-Disposition",
-                    b"inline; filename=%s" % (upload_name.encode("utf-8"),),
-                )
+                if is_ascii(upload_name):
+                    request.setHeader(
+                        b"Content-Disposition",
+                        b"inline; filename=%s" % (upload_name.encode("utf-8"),),
+                    )
+                else:
+                    request.setHeader(
+                        b"Content-Disposition",
+                        b"inline; filename*=utf-8''%s" % (
+                            urllib.quote(upload_name.encode("utf-8")),
+                        ),
+                    )
 
             # cache for at least a day.
             # XXX: we might want to turn this off for data we don't want to
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index cdd1d44e0..21d8fb9ce 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -15,7 +15,7 @@
 
 from synapse.http.server import respond_with_json, request_handler
 
-from synapse.util.stringutils import random_string, is_ascii
+from synapse.util.stringutils import random_string
 from synapse.api.errors import SynapseError
 
 from twisted.web.server import NOT_DONE_YET
@@ -86,9 +86,7 @@ class UploadResource(BaseMediaResource):
 
         upload_name = request.args.get("filename", None)
         if upload_name:
-            upload_name = upload_name[0]
-            if upload_name and not is_ascii(upload_name):
-                raise SynapseError(400, "filename must be ascii")
+            upload_name = upload_name[0].decode('UTF-8')
 
         headers = request.requestHeaders
 
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 7a1e96af3..f3a36340e 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -38,6 +38,8 @@ def random_string_with_symbols(length):
 def is_ascii(s):
     try:
         s.encode("ascii")
+    except UnicodeEncodeError:
+        return False
     except UnicodeDecodeError:
         return False
     else:

From 5a9e0c36824ffc8bb365cdb30a273d427f997bd9 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 26 Aug 2015 17:08:47 +0100
Subject: [PATCH 27/34] Handle unicode filenames given when downloading or
 received over federation

---
 synapse/rest/media/v1/base_resource.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index 24297b20f..ad2c9d4e7 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -34,6 +34,7 @@ import os
 import cgi
 import logging
 import urllib
+import urlparse
 
 logger = logging.getLogger(__name__)
 
@@ -43,10 +44,13 @@ def parse_media_id(request):
         # This allows users to append e.g. /test.png to the URL. Useful for
         # clients that parse the URL to see content type.
         server_name, media_id = request.postpath[:2]
-        if len(request.postpath) > 2 and is_ascii(request.postpath[-1]):
-            return server_name, media_id, request.postpath[-1]
-        else:
-            return server_name, media_id, None
+        file_name = None
+        if len(request.postpath) > 2:
+            try:
+                file_name = urlparse.unquote(request.postpath[-1]).decode("utf-8")
+            except UnicodeDecodeError:
+                pass
+        return server_name, media_id, file_name
     except:
         raise SynapseError(
             404,
@@ -144,6 +148,16 @@ class BaseMediaResource(Resource):
                 upload_name = params.get("filename", None)
                 if upload_name and not is_ascii(upload_name):
                     upload_name = None
+                else:
+                    upload_name_utf8 = params.get("filename*", None)
+                    if upload_name_utf8.lower().startswith("utf-8''"):
+                        upload_name = upload_name_utf8[7:]
+                if upload_name:
+                    upload_name = urlparse.unquote(upload_name)
+                    try:
+                        upload_name = upload_name.decode("utf-8");
+                    except UnicodeDecodeError:
+                        upload_name = None
             else:
                 upload_name = None
 
@@ -185,7 +199,9 @@ class BaseMediaResource(Resource):
                 if is_ascii(upload_name):
                     request.setHeader(
                         b"Content-Disposition",
-                        b"inline; filename=%s" % (upload_name.encode("utf-8"),),
+                        b"inline; filename=%s" % (
+                            urllib.quote(upload_name.encode("utf-8")),
+                        ),
                     )
                 else:
                     request.setHeader(

From c9cb354b58972b9e0e91cd6d6398e9bb02f7b967 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 26 Aug 2015 17:27:23 +0100
Subject: [PATCH 28/34] Give a sensible error message if the filename is
 invalid UTF-8

---
 synapse/rest/media/v1/base_resource.py   | 2 +-
 synapse/rest/media/v1/upload_resource.py | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index ad2c9d4e7..60751da1d 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -155,7 +155,7 @@ class BaseMediaResource(Resource):
                 if upload_name:
                     upload_name = urlparse.unquote(upload_name)
                     try:
-                        upload_name = upload_name.decode("utf-8");
+                        upload_name = upload_name.decode("utf-8")
                     except UnicodeDecodeError:
                         upload_name = None
             else:
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index 21d8fb9ce..031bfa80f 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -86,7 +86,13 @@ class UploadResource(BaseMediaResource):
 
         upload_name = request.args.get("filename", None)
         if upload_name:
-            upload_name = upload_name[0].decode('UTF-8')
+            try:
+                upload_name = upload_name[0].decode('UTF-8')
+            except UnicodeDecodeError:
+                raise SynapseError(
+                    msg="Invalid UTF-8 filename parameter: %r" % (upload_name),
+                    code=400,
+                )
 
         headers = request.requestHeaders
 

From 25b32b63aee6cc2feb4ffdd4c247475f63a241dc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 27 Aug 2015 10:09:32 +0100
Subject: [PATCH 29/34] Bump changelog and version

---
 CHANGES.rst         | 13 ++++++++++---
 synapse/__init__.py |  2 +-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 5f16e4485..22921668e 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,19 +1,26 @@
+Changes in synapse v0.10.0-rc4 (2015-08-27)
+===========================================
+
+* Allow UTF-8 filenames for upload. (PR #259)
+
 Changes in synapse v0.10.0-rc3 (2015-08-25)
 ===========================================
 
 * Add ``--keys-directory`` config option to specify where files such as
   certs and signing keys should be stored in, when using ``--generate-config``
-  or ``--generate-keys``.
+  or ``--generate-keys``. (PR #250)
 * Allow ``--config-path`` to specify a directory, causing synapse to use all
-  \*.yaml files in the directory as config files.
+  \*.yaml files in the directory as config files. (PR #249)
 * Add ``web_client_location`` config option to specify static files to be
-  hosted by synapse under ``/_matrix/client``.
+  hosted by synapse under ``/_matrix/client``. (PR #245)
 * Add helper utility to synapse to read and parse the config files and extract
   the value of a given key. For example::
 
     $ python -m synapse.config read server_name -c homeserver.yaml
     localhost
 
+  (PR #246)
+
 
 Changes in synapse v0.10.0-rc2 (2015-08-24)
 ===========================================
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 41e3283f0..635f53af1 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.10.0-rc3"
+__version__ = "0.10.0-rc4"

From f02532baadc4fbd95bec6cb7f45019d2c46c1324 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 27 Aug 2015 10:37:02 +0100
Subject: [PATCH 30/34] Check for None

---
 synapse/rest/media/v1/base_resource.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index 60751da1d..b0e997b47 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -150,7 +150,7 @@ class BaseMediaResource(Resource):
                     upload_name = None
                 else:
                     upload_name_utf8 = params.get("filename*", None)
-                    if upload_name_utf8.lower().startswith("utf-8''"):
+                    if upload_name and upload_name_utf8.lower().startswith("utf-8''"):
                         upload_name = upload_name_utf8[7:]
                 if upload_name:
                     upload_name = urlparse.unquote(upload_name)

From 53c2eed862c2c2fc90ee4b51bed624be5fcec9f3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 27 Aug 2015 10:38:22 +0100
Subject: [PATCH 31/34] None check the correct variable

---
 synapse/rest/media/v1/base_resource.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index b0e997b47..610cb3ef8 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -150,8 +150,9 @@ class BaseMediaResource(Resource):
                     upload_name = None
                 else:
                     upload_name_utf8 = params.get("filename*", None)
-                    if upload_name and upload_name_utf8.lower().startswith("utf-8''"):
-                        upload_name = upload_name_utf8[7:]
+                    if upload_name_utf8:
+                        if upload_name_utf8.lower().startswith("utf-8''"):
+                            upload_name = upload_name_utf8[7:]
                 if upload_name:
                     upload_name = urlparse.unquote(upload_name)
                     try:

From 66ec6cf9b892cd22dd75d9b66f10b120ebe233ed Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 27 Aug 2015 10:48:58 +0100
Subject: [PATCH 32/34] Check for an internationalised filename first

---
 synapse/rest/media/v1/base_resource.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index 610cb3ef8..03ebbbefe 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -145,14 +145,20 @@ class BaseMediaResource(Resource):
             content_disposition = headers.get("Content-Disposition", None)
             if content_disposition:
                 _, params = cgi.parse_header(content_disposition[0],)
-                upload_name = params.get("filename", None)
-                if upload_name and not is_ascii(upload_name):
-                    upload_name = None
-                else:
-                    upload_name_utf8 = params.get("filename*", None)
-                    if upload_name_utf8:
-                        if upload_name_utf8.lower().startswith("utf-8''"):
-                            upload_name = upload_name_utf8[7:]
+                upload_name = None
+
+                # First check if there is a valid UTF-8 filename
+                upload_name_utf8 = params.get("filename*", None)
+                if upload_name_utf8:
+                    if upload_name_utf8.lower().startswith("utf-8''"):
+                        upload_name = upload_name_utf8[7:]
+
+                # If there isn't check for an ascii name.
+                if not upload_name:
+                    upload_name = params.get("filename", None)
+                    if upload_name and not is_ascii(upload_name):
+                        upload_name = None
+
                 if upload_name:
                     upload_name = urlparse.unquote(upload_name)
                     try:

From ddf4d2bd981cbc4079b2bff0a2bba500b1aad208 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 27 Aug 2015 10:50:49 +0100
Subject: [PATCH 33/34] Consistency

---
 synapse/rest/media/v1/base_resource.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index 03ebbbefe..b2aeb8c90 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -155,9 +155,9 @@ class BaseMediaResource(Resource):
 
                 # If there isn't check for an ascii name.
                 if not upload_name:
-                    upload_name = params.get("filename", None)
-                    if upload_name and not is_ascii(upload_name):
-                        upload_name = None
+                    upload_name_ascii = params.get("filename", None)
+                    if upload_name_ascii and is_ascii(upload_name_ascii):
+                        upload_name = upload_name_ascii
 
                 if upload_name:
                     upload_name = urlparse.unquote(upload_name)

From 5371c2a1f74dcd9c79c2cd5166e5b33f7e02afef Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 27 Aug 2015 11:20:36 +0100
Subject: [PATCH 34/34] Bump version and changelog

---
 CHANGES.rst         | 5 +++++
 synapse/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 22921668e..8b9916c96 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,8 @@
+Changes in synapse v0.10.0-rc5 (2015-08-27)
+===========================================
+
+* Fix bug that broke downloading files with ascii filenames across federation.
+
 Changes in synapse v0.10.0-rc4 (2015-08-27)
 ===========================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 635f53af1..57b8304d3 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.10.0-rc4"
+__version__ = "0.10.0-rc5"