Merge remote-tracking branch 'origin/develop' into store_event_actions

CHANGES.rst

@@ -1,6 +1,7 @@
-Changes in synapse v0.12.0-rc4 (unreleased)
+Changes in synapse v0.12.0 (2016-01-04)
-===========================================
+=======================================
-* Fix C-S API to expose ``/login`` under ``r0`` (PR #459)
+
+* Expose ``/login`` under ``r0`` (PR #459)
 
 Changes in synapse v0.12.0-rc3 (2015-12-23)
 ===========================================
@@ -14,17 +15,15 @@ Changes in synapse v0.12.0-rc3 (2015-12-23)
 * Add a ``display_name`` to third-party invites (PR #449)
 * Send more information to the identity server for third-party invites so that
   it can send richer messages to the invitee (PR #446)
-
+* Cache the responses to ``/initialSync`` for 5 minutes. If a client
-* Cache the responses to ``/intialSync`` for 5 minutes. If a client
   retries a request to ``/initialSync`` before the a response was computed
   to the first request then the same response is used for both requests
   (PR #457)
-
 * Fix a bug where synapse would always request the signing keys of
   remote servers even when the key was cached locally (PR #452)
 * Fix 500 when pagination search results (PR #447)
 * Fix a bug where synapse was leaking raw email address in third-party invites
-   (PR #448)
+  (PR #448)
 
 Changes in synapse v0.12.0-rc2 (2015-12-14)
 ===========================================

README.rst

@@ -146,6 +146,10 @@ Note that these packages do not include  a client - choose one from
 https://matrix.org/blog/try-matrix-now/ (or build your own with 
 https://github.com/matrix-org/matrix-js-sdk/). 
 
+Finally, Martin Giess has created an auto-deployment process with vagrant/ansible, 
+tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy 
+for details.
+
 To set up your homeserver, run (in your virtualenv, as before)::
 
     cd ~/.synapse

synapse/__init__.py

@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.12.0-rc3"
+__version__ = "0.12.0"

synapse/api/auth.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -583,7 +583,7 @@ class Auth(object):
             AuthError if no user by that token exists or the token is invalid.
         """
         try:
-            ret = yield self._get_user_from_macaroon(token)
+            ret = yield self.get_user_from_macaroon(token)
         except AuthError:
             # TODO(daniel): Remove this fallback when all existing access tokens
             # have been re-issued as macaroons.
@@ -591,7 +591,7 @@ class Auth(object):
         defer.returnValue(ret)
 
     @defer.inlineCallbacks
-    def _get_user_from_macaroon(self, macaroon_str):
+    def get_user_from_macaroon(self, macaroon_str):
         try:
             macaroon = pymacaroons.Macaroon.deserialize(macaroon_str)
             self.validate_macaroon(macaroon, "access", False)

synapse/handlers/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -55,8 +55,7 @@ class BaseHandler(object):
         self.event_builder_factory = hs.get_event_builder_factory()
 
     @defer.inlineCallbacks
-    def _filter_events_for_client(self, user_id, events, is_guest=False,
+    def _filter_events_for_client(self, user_id, events, is_guest=False):
-                                  require_all_visible_for_guests=True):
         # Assumes that user has at some point joined the room if not is_guest.
 
         def allowed(event, membership, visibility):
@@ -117,17 +116,6 @@ class BaseHandler(object):
             if should_include:
                 events_to_return.append(event)
 
-        if (require_all_visible_for_guests
-                and is_guest
-                and len(events_to_return) < len(events)):
-            # This indicates that some events in the requested range were not
-            # visible to guest users. To be safe, we reject the entire request,
-            # so that we don't have to worry about interpreting visibility
-            # boundaries.
-            raise AuthError(403, "User %s does not have permission" % (
-                user_id
-            ))
-
         defer.returnValue(events_to_return)
 
     def ratelimit(self, user_id):

synapse/handlers/auth.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -408,7 +408,7 @@ class AuthHandler(BaseHandler):
             macaroon = pymacaroons.Macaroon.deserialize(login_token)
             auth_api = self.hs.get_auth()
             auth_api.validate_macaroon(macaroon, "login", True)
-            return self._get_user_from_macaroon(macaroon)
+            return self.get_user_from_macaroon(macaroon)
         except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError):
             raise AuthError(401, "Invalid token", errcode=Codes.UNKNOWN_TOKEN)
 
@@ -421,7 +421,7 @@ class AuthHandler(BaseHandler):
         macaroon.add_first_party_caveat("user_id = %s" % (user_id,))
         return macaroon
 
-    def _get_user_from_macaroon(self, macaroon):
+    def get_user_from_macaroon(self, macaroon):
         user_prefix = "user_id = "
         for caveat in macaroon.caveats:
             if caveat.caveat_id.startswith(user_prefix):

synapse/handlers/message.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -685,7 +685,7 @@ class MessageHandler(BaseHandler):
         ).addErrback(unwrapFirstError)
 
         messages = yield self._filter_events_for_client(
-            user_id, messages, is_guest=is_guest, require_all_visible_for_guests=False
+            user_id, messages, is_guest=is_guest,
         )
 
         start_token = now_token.copy_and_replace("room_key", token[0])

synapse/handlers/register.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -40,12 +40,13 @@ class RegistrationHandler(BaseHandler):
     def __init__(self, hs):
         super(RegistrationHandler, self).__init__(hs)
 
+        self.auth = hs.get_auth()
         self.distributor = hs.get_distributor()
         self.distributor.declare("registered_user")
         self.captcha_client = CaptchaServerHttpClient(hs)
 
     @defer.inlineCallbacks
-    def check_username(self, localpart):
+    def check_username(self, localpart, guest_access_token=None):
         yield run_on_reactor()
 
         if urllib.quote(localpart) != localpart:
@@ -62,14 +63,29 @@ class RegistrationHandler(BaseHandler):
 
         users = yield self.store.get_users_by_id_case_insensitive(user_id)
         if users:
-            raise SynapseError(
+            if not guest_access_token:
-                400,
+                raise SynapseError(
-                "User ID already taken.",
+                    400,
-                errcode=Codes.USER_IN_USE,
+                    "User ID already taken.",
-            )
+                    errcode=Codes.USER_IN_USE,
+                )
+            user_data = yield self.auth.get_user_from_macaroon(guest_access_token)
+            if not user_data["is_guest"] or user_data["user"].localpart != localpart:
+                raise AuthError(
+                    403,
+                    "Cannot register taken user ID without valid guest "
+                    "credentials for that user.",
+                    errcode=Codes.FORBIDDEN,
+                )
 
     @defer.inlineCallbacks
-    def register(self, localpart=None, password=None, generate_token=True):
+    def register(
+        self,
+        localpart=None,
+        password=None,
+        generate_token=True,
+        guest_access_token=None
+    ):
         """Registers a new client on the server.
 
         Args:
@@ -89,7 +105,7 @@ class RegistrationHandler(BaseHandler):
             password_hash = self.auth_handler().hash(password)
 
         if localpart:
-            yield self.check_username(localpart)
+            yield self.check_username(localpart, guest_access_token=guest_access_token)
 
             user = UserID(localpart, self.hs.hostname)
             user_id = user.to_string()
@@ -100,7 +116,8 @@ class RegistrationHandler(BaseHandler):
             yield self.store.register(
                 user_id=user_id,
                 token=token,
-                password_hash=password_hash
+                password_hash=password_hash,
+                was_guest=guest_access_token is not None,
             )
 
             yield registered_user(self.distributor, user)

synapse/handlers/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -115,6 +115,8 @@ class RoomCreationHandler(BaseHandler):
             except:
                 raise SynapseError(400, "Invalid user_id: %s" % (i,))
 
+        invite_3pid_list = config.get("invite_3pid", [])
+
         is_public = config.get("visibility", None) == "public"
 
         if room_id:
@@ -220,6 +222,20 @@ class RoomCreationHandler(BaseHandler):
                 "content": {"membership": Membership.INVITE},
             }, ratelimit=False)
 
+        for invite_3pid in invite_3pid_list:
+            id_server = invite_3pid["id_server"]
+            address = invite_3pid["address"]
+            medium = invite_3pid["medium"]
+            yield self.hs.get_handlers().room_member_handler.do_3pid_invite(
+                room_id,
+                user,
+                medium,
+                address,
+                id_server,
+                token_id=None,
+                txn_id=None,
+            )
+
         result = {"room_id": room_id}
 
         if room_alias:
@@ -879,14 +895,12 @@ class RoomContextHandler(BaseHandler):
             user.to_string(),
             results["events_before"],
             is_guest=is_guest,
-            require_all_visible_for_guests=False
         )
 
         results["events_after"] = yield self._filter_events_for_client(
             user.to_string(),
             results["events_after"],
             is_guest=is_guest,
-            require_all_visible_for_guests=False
         )
 
         if results["events_after"]:

synapse/handlers/sync.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015 OpenMarket Ltd
+# Copyright 2015 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -118,7 +118,7 @@ class SyncResult(collections.namedtuple("SyncResult", [
         events.
         """
         return bool(
-            self.presence or self.joined or self.invited
+            self.presence or self.joined or self.invited or self.archived
         )
 
 GuestRoom = collections.namedtuple("GuestRoom", ("room_id", "membership"))
@@ -690,7 +690,6 @@ class SyncHandler(BaseHandler):
                 sync_config.user.to_string(),
                 loaded_recents,
                 is_guest=sync_config.is_guest,
-                require_all_visible_for_guests=False
             )
             loaded_recents.extend(recents)
             recents = loaded_recents

synapse/notifier.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -386,7 +386,6 @@ class Notifier(object):
                         user.to_string(),
                         new_events,
                         is_guest=is_guest,
-                        require_all_visible_for_guests=False
                     )
 
                 events.extend(new_events)

synapse/rest/client/v2_alpha/register.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015 OpenMarket Ltd
+# Copyright 2015 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -119,8 +119,13 @@ class RegisterRestServlet(RestServlet):
         if self.hs.config.disable_registration:
             raise SynapseError(403, "Registration has been disabled")
 
+        guest_access_token = body.get("guest_access_token", None)
+
         if desired_username is not None:
-            yield self.registration_handler.check_username(desired_username)
+            yield self.registration_handler.check_username(
+                desired_username,
+                guest_access_token=guest_access_token
+            )
 
         if self.hs.config.enable_registration_captcha:
             flows = [
@@ -150,7 +155,8 @@ class RegisterRestServlet(RestServlet):
 
         (user_id, token) = yield self.registration_handler.register(
             localpart=desired_username,
-            password=new_password
+            password=new_password,
+            guest_access_token=guest_access_token,
         )
 
         if result and LoginType.EMAIL_IDENTITY in result:

synapse/rest/media/v1/thumbnail_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -248,6 +248,7 @@ class ThumbnailResource(BaseMediaResource):
 
         if desired_method.lower() == "crop":
             info_list = []
+            info_list2 = []
             for info in thumbnail_infos:
                 t_w = info["thumbnail_width"]
                 t_h = info["thumbnail_height"]
@@ -258,12 +259,20 @@ class ThumbnailResource(BaseMediaResource):
                     size_quality = abs((d_w - t_w) * (d_h - t_h))
                     type_quality = desired_type != info["thumbnail_type"]
                     length_quality = info["thumbnail_length"]
-                    info_list.append((
+                    if t_w >= d_w or t_h >= d_h:
-                        aspect_quality, min_quality, size_quality, type_quality,
+                        info_list.append((
-                        length_quality, info
+                            aspect_quality, min_quality, size_quality, type_quality,
-                    ))
+                            length_quality, info
+                        ))
+                    else:
+                        info_list2.append((
+                            aspect_quality, min_quality, size_quality, type_quality,
+                            length_quality, info
+                        ))
             if info_list:
                 return min(info_list)[-1]
+            else:
+                return min(info_list2)[-1]
         else:
             info_list = []
             info_list2 = []

synapse/storage/prepare_database.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 27
+SCHEMA_VERSION = 28
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 

synapse/storage/registration.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -73,30 +73,39 @@ class RegistrationStore(SQLBaseStore):
         )
 
     @defer.inlineCallbacks
-    def register(self, user_id, token, password_hash):
+    def register(self, user_id, token, password_hash, was_guest=False):
         """Attempts to register an account.
 
         Args:
             user_id (str): The desired user ID to register.
             token (str): The desired access token to use for this user.
             password_hash (str): Optional. The password hash for this user.
+            was_guest (bool): Optional. Whether this is a guest account being
+                upgraded to a non-guest account.
         Raises:
             StoreError if the user_id could not be registered.
         """
         yield self.runInteraction(
             "register",
-            self._register, user_id, token, password_hash
+            self._register, user_id, token, password_hash, was_guest
         )
 
-    def _register(self, txn, user_id, token, password_hash):
+    def _register(self, txn, user_id, token, password_hash, was_guest):
         now = int(self.clock.time())
 
         next_id = self._access_tokens_id_gen.get_next_txn(txn)
 
         try:
-            txn.execute("INSERT INTO users(name, password_hash, creation_ts) "
+            if was_guest:
-                        "VALUES (?,?,?)",
+                txn.execute("UPDATE users SET"
-                        [user_id, password_hash, now])
+                            " password_hash = ?,"
+                            " upgrade_ts = ?"
+                            " WHERE name = ?",
+                            [password_hash, now, user_id])
+            else:
+                txn.execute("INSERT INTO users(name, password_hash, creation_ts) "
+                            "VALUES (?,?,?)",
+                            [user_id, password_hash, now])
         except self.database_engine.module.IntegrityError:
             raise StoreError(
                 400, "User ID already taken.", errcode=Codes.USER_IN_USE

synapse/storage/schema/delta/28/upgrade_times.sql

@@ -0,0 +1,21 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Stores the timestamp when a user upgraded from a guest to a full user, if
+ * that happened.
+ */
+
+ALTER TABLE users ADD COLUMN upgrade_ts BIGINT;

tests/api/test_auth.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015 OpenMarket Ltd
+# Copyright 2015 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -154,7 +154,7 @@ class AuthTestCase(unittest.TestCase):
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user_id,))
-        user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize())
+        user_info = yield self.auth.get_user_from_macaroon(macaroon.serialize())
         user = user_info["user"]
         self.assertEqual(UserID.from_string(user_id), user)
 
@@ -171,7 +171,7 @@ class AuthTestCase(unittest.TestCase):
         macaroon.add_first_party_caveat("guest = true")
         serialized = macaroon.serialize()
 
-        user_info = yield self.auth._get_user_from_macaroon(serialized)
+        user_info = yield self.auth.get_user_from_macaroon(serialized)
         user = user_info["user"]
         is_guest = user_info["is_guest"]
         self.assertEqual(UserID.from_string(user_id), user)
@@ -192,7 +192,7 @@ class AuthTestCase(unittest.TestCase):
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user,))
         with self.assertRaises(AuthError) as cm:
-            yield self.auth._get_user_from_macaroon(macaroon.serialize())
+            yield self.auth.get_user_from_macaroon(macaroon.serialize())
         self.assertEqual(401, cm.exception.code)
         self.assertIn("User mismatch", cm.exception.msg)
 
@@ -212,7 +212,7 @@ class AuthTestCase(unittest.TestCase):
         macaroon.add_first_party_caveat("type = access")
 
         with self.assertRaises(AuthError) as cm:
-            yield self.auth._get_user_from_macaroon(macaroon.serialize())
+            yield self.auth.get_user_from_macaroon(macaroon.serialize())
         self.assertEqual(401, cm.exception.code)
         self.assertIn("No user caveat", cm.exception.msg)
 
@@ -234,7 +234,7 @@ class AuthTestCase(unittest.TestCase):
         macaroon.add_first_party_caveat("user_id = %s" % (user,))
 
         with self.assertRaises(AuthError) as cm:
-            yield self.auth._get_user_from_macaroon(macaroon.serialize())
+            yield self.auth.get_user_from_macaroon(macaroon.serialize())
         self.assertEqual(401, cm.exception.code)
         self.assertIn("Invalid macaroon", cm.exception.msg)
 
@@ -257,7 +257,7 @@ class AuthTestCase(unittest.TestCase):
         macaroon.add_first_party_caveat("cunning > fox")
 
         with self.assertRaises(AuthError) as cm:
-            yield self.auth._get_user_from_macaroon(macaroon.serialize())
+            yield self.auth.get_user_from_macaroon(macaroon.serialize())
         self.assertEqual(401, cm.exception.code)
         self.assertIn("Invalid macaroon", cm.exception.msg)
 
@@ -285,11 +285,11 @@ class AuthTestCase(unittest.TestCase):
 
         self.hs.clock.now = 5000 # seconds
 
-        yield self.auth._get_user_from_macaroon(macaroon.serialize())
+        yield self.auth.get_user_from_macaroon(macaroon.serialize())
         # TODO(daniel): Turn on the check that we validate expiration, when we
         # validate expiration (and remove the above line, which will start
         # throwing).
         # with self.assertRaises(AuthError) as cm:
-        #     yield self.auth._get_user_from_macaroon(macaroon.serialize())
+        #     yield self.auth.get_user_from_macaroon(macaroon.serialize())
         # self.assertEqual(401, cm.exception.code)
         # self.assertIn("Invalid macaroon", cm.exception.msg)