Merge branch 'develop' into rav/no_more_refresh_tokens

README.rst

@@ -11,7 +11,7 @@ VoIP.  The basics you need to know to get up and running are:
   like ``#matrix:matrix.org`` or ``#test:localhost:8448``.
 
 - Matrix user IDs look like ``@matthew:matrix.org`` (although in the future
-  you will normally refer to yourself and others using a third party identifier 
+  you will normally refer to yourself and others using a third party identifier
   (3PID): email address, phone number, etc rather than manipulating Matrix user IDs)
 
 The overall architecture is::
@@ -156,8 +156,8 @@ In case of problems, please see the _Troubleshooting section below.
 Alternatively, Silvio Fricke has contributed a Dockerfile to automate the
 above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/.
 
-Also, Martin Giess has created an auto-deployment process with vagrant/ansible, 
+Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
-tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy 
+tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
 for details.
 
 To set up your homeserver, run (in your virtualenv, as before)::
@@ -427,7 +427,7 @@ to install using pip and a virtualenv::
 
     virtualenv env
     source env/bin/activate
-    python synapse/python_dependencies.py | xargs -n1 pip install
+    python synapse/python_dependencies.py | xargs pip install
     pip install setuptools_trial mock
 
 This will run a process of downloading and installing all the needed
@@ -605,8 +605,8 @@ First calculate the hash of the new password:
 
     $ source ~/.synapse/bin/activate
     $ ./scripts/hash_password
-    Password: 
+    Password:
-    Confirm password: 
+    Confirm password:
     $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
 Then update the `users` table in the database:
@@ -650,4 +650,3 @@ matrix.org on.  The default setting is currently 0.1, which is probably
 around a ~700MB footprint.  You can dial it down further to 0.02 if
 desired, which targets roughly ~512MB.  Conversely you can dial it up if
 you need performance for lots of users and have a box with a lot of RAM.
-

jenkins/prepare_synapse.sh

@@ -15,6 +15,6 @@ tox -e py27 --notest -v
 
 TOX_BIN=$TOX_DIR/py27/bin
 $TOX_BIN/pip install setuptools
-python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install
+{ python synapse/python_dependencies.py
-$TOX_BIN/pip install lxml
+  echo lxml psycopg2
-$TOX_BIN/pip install psycopg2
+} | xargs $TOX_BIN/pip install

synapse/api/auth.py

@@ -39,6 +39,9 @@ AuthEventTypes = (
     EventTypes.ThirdPartyInvite,
 )
 
+# guests always get this device id.
+GUEST_DEVICE_ID = "guest_device"
+
 
 class Auth(object):
     """
@@ -717,7 +720,8 @@ class Auth(object):
                     "user": user,
                     "is_guest": True,
                     "token_id": None,
-                    "device_id": None,
+                    # all guests get the same device id
+                    "device_id": GUEST_DEVICE_ID,
                 }
             elif rights == "delete_pusher":
                 # We don't store these tokens in the database
@@ -790,9 +794,6 @@ class Auth(object):
             type_string(str): The kind of token required (e.g. "access", "refresh",
                               "delete_pusher")
             verify_expiry(bool): Whether to verify whether the macaroon has expired.
-                This should really always be True, but there exist access tokens
-                in the wild which expire when they should not, so we can't
-                enforce expiry yet.
             user_id (str): The user_id required
         """
         v = pymacaroons.Verifier()
@@ -805,11 +806,24 @@ class Auth(object):
         v.satisfy_exact("type = " + type_string)
         v.satisfy_exact("user_id = %s" % user_id)
         v.satisfy_exact("guest = true")
+
+        # verify_expiry should really always be True, but there exist access
+        # tokens in the wild which expire when they should not, so we can't
+        # enforce expiry yet (so we have to allow any caveat starting with
+        # 'time < ' in access tokens).
+        #
+        # On the other hand, short-term login tokens (as used by CAS login, for
+        # example) have an expiry time which we do want to enforce.
+
         if verify_expiry:
             v.satisfy_general(self._verify_expiry)
         else:
             v.satisfy_general(lambda c: c.startswith("time < "))
 
+        # access_tokens and refresh_tokens include a nonce for uniqueness: any
+        # value is acceptable
+        v.satisfy_general(lambda c: c.startswith("nonce = "))
+
         v.verify(macaroon, self.hs.config.macaroon_secret_key)
 
     def _verify_expiry(self, caveat):

synapse/config/registration.py

@@ -32,7 +32,6 @@ class RegistrationConfig(Config):
             )
 
         self.registration_shared_secret = config.get("registration_shared_secret")
-        self.user_creation_max_duration = int(config["user_creation_max_duration"])
 
         self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
         self.trusted_third_party_id_servers = config["trusted_third_party_id_servers"]
@@ -55,11 +54,6 @@ class RegistrationConfig(Config):
         # secret, even if registration is otherwise disabled.
         registration_shared_secret: "%(registration_shared_secret)s"
 
-        # Sets the expiry for the short term user creation in
-        # milliseconds. For instance the bellow duration is two weeks
-        # in milliseconds.
-        user_creation_max_duration: 1209600000
-
         # Set the number of bcrypt rounds used to generate password hash.
         # Larger numbers increase the work factor needed to generate the hash.
         # The default number of rounds is 12.

synapse/handlers/auth.py

@@ -526,14 +526,15 @@ class AuthHandler(BaseHandler):
                                                   device_id)
         defer.returnValue(access_token)
 
-    def generate_access_token(self, user_id, extra_caveats=None,
+    def generate_access_token(self, user_id, extra_caveats=None):
-                              duration_in_ms=(60 * 60 * 1000)):
         extra_caveats = extra_caveats or []
         macaroon = self._generate_base_macaroon(user_id)
         macaroon.add_first_party_caveat("type = access")
-        now = self.hs.get_clock().time_msec()
+        # Include a nonce, to make sure that each login gets a different
-        expiry = now + duration_in_ms
+        # access token.
-        macaroon.add_first_party_caveat("time < %d" % (expiry,))
+        macaroon.add_first_party_caveat("nonce = %s" % (
+            stringutils.random_string_with_symbols(16),
+        ))
         for caveat in extra_caveats:
             macaroon.add_first_party_caveat(caveat)
         return macaroon.serialize()

synapse/handlers/register.py

@@ -369,7 +369,7 @@ class RegistrationHandler(BaseHandler):
         defer.returnValue(data)
 
     @defer.inlineCallbacks
-    def get_or_create_user(self, requester, localpart, displayname, duration_in_ms,
+    def get_or_create_user(self, requester, localpart, displayname,
                            password_hash=None):
         """Creates a new user if the user does not exist,
         else revokes all previous access tokens and generates a new one.
@@ -399,8 +399,7 @@ class RegistrationHandler(BaseHandler):
 
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
-        token = self.auth_handler().generate_access_token(
+        token = self.auth_handler().generate_access_token(user_id)
-            user_id, None, duration_in_ms)
 
         if need_register:
             yield self.store.register(

synapse/http/matrixfederationclient.py

@@ -33,6 +33,7 @@ from synapse.api.errors import (
 
 from signedjson.sign import sign_json
 
+import cgi
 import simplejson as json
 import logging
 import random
@@ -292,12 +293,7 @@ class MatrixFederationHttpClient(object):
 
         if 200 <= response.code < 300:
             # We need to update the transactions table to say it was sent?
-            c_type = response.headers.getRawHeaders("Content-Type")
+            check_content_type_is_json(response.headers)
-
-            if "application/json" not in c_type:
-                raise RuntimeError(
-                    "Content-Type not application/json"
-                )
 
         body = yield preserve_context_over_fn(readBody, response)
         defer.returnValue(json.loads(body))
@@ -342,12 +338,7 @@ class MatrixFederationHttpClient(object):
 
         if 200 <= response.code < 300:
             # We need to update the transactions table to say it was sent?
-            c_type = response.headers.getRawHeaders("Content-Type")
+            check_content_type_is_json(response.headers)
-
-            if "application/json" not in c_type:
-                raise RuntimeError(
-                    "Content-Type not application/json"
-                )
 
         body = yield preserve_context_over_fn(readBody, response)
 
@@ -400,12 +391,7 @@ class MatrixFederationHttpClient(object):
 
         if 200 <= response.code < 300:
             # We need to update the transactions table to say it was sent?
-            c_type = response.headers.getRawHeaders("Content-Type")
+            check_content_type_is_json(response.headers)
-
-            if "application/json" not in c_type:
-                raise RuntimeError(
-                    "Content-Type not application/json"
-                )
 
         body = yield preserve_context_over_fn(readBody, response)
 
@@ -525,3 +511,29 @@ def _flatten_response_never_received(e):
         )
     else:
         return "%s: %s" % (type(e).__name__, e.message,)
+
+
+def check_content_type_is_json(headers):
+    """
+    Check that a set of HTTP headers have a Content-Type header, and that it
+    is application/json.
+
+    Args:
+        headers (twisted.web.http_headers.Headers): headers to check
+
+    Raises:
+        RuntimeError if the
+
+    """
+    c_type = headers.getRawHeaders("Content-Type")
+    if c_type is None:
+        raise RuntimeError(
+            "No Content-Type header"
+        )
+
+    c_type = c_type[0]  # only the first header
+    val, options = cgi.parse_header(c_type)
+    if val != "application/json":
+        raise RuntimeError(
+            "Content-Type not application/json: was '%s'" % c_type
+        )

synapse/rest/client/v1/register.py

@@ -384,7 +384,6 @@ class CreateUserRestServlet(ClientV1RestServlet):
     def __init__(self, hs):
         super(CreateUserRestServlet, self).__init__(hs)
         self.store = hs.get_datastore()
-        self.direct_user_creation_max_duration = hs.config.user_creation_max_duration
         self.handlers = hs.get_handlers()
 
     @defer.inlineCallbacks
@@ -418,18 +417,8 @@ class CreateUserRestServlet(ClientV1RestServlet):
         if "displayname" not in user_json:
             raise SynapseError(400, "Expected 'displayname' key.")
 
-        if "duration_seconds" not in user_json:
-            raise SynapseError(400, "Expected 'duration_seconds' key.")
-
         localpart = user_json["localpart"].encode("utf-8")
         displayname = user_json["displayname"].encode("utf-8")
-        duration_seconds = 0
-        try:
-            duration_seconds = int(user_json["duration_seconds"])
-        except ValueError:
-            raise SynapseError(400, "Failed to parse 'duration_seconds'")
-        if duration_seconds > self.direct_user_creation_max_duration:
-            duration_seconds = self.direct_user_creation_max_duration
         password_hash = user_json["password_hash"].encode("utf-8") \
             if user_json.get("password_hash") else None
 
@@ -438,7 +427,6 @@ class CreateUserRestServlet(ClientV1RestServlet):
             requester=requester,
             localpart=localpart,
             displayname=displayname,
-            duration_in_ms=(duration_seconds * 1000),
             password_hash=password_hash
         )
 

synapse/rest/client/v2_alpha/devices.py

@@ -39,7 +39,7 @@ class DevicesRestServlet(servlet.RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         devices = yield self.device_handler.get_devices_by_user(
             requester.user.to_string()
         )
@@ -63,7 +63,7 @@ class DeviceRestServlet(servlet.RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, device_id):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         device = yield self.device_handler.get_device(
             requester.user.to_string(),
             device_id,
@@ -99,7 +99,7 @@ class DeviceRestServlet(servlet.RestServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, request, device_id):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
 
         body = servlet.parse_json_object_from_request(request)
         yield self.device_handler.update_device(

synapse/rest/client/v2_alpha/keys.py

@@ -65,7 +65,7 @@ class KeyUploadServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, device_id):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         user_id = requester.user.to_string()
         body = parse_json_object_from_request(request)
 
@@ -150,7 +150,7 @@ class KeyQueryServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, user_id, device_id):
-        yield self.auth.get_user_by_req(request)
+        yield self.auth.get_user_by_req(request, allow_guest=True)
         timeout = parse_integer(request, "timeout", 10 * 1000)
         body = parse_json_object_from_request(request)
         result = yield self.e2e_keys_handler.query_devices(body, timeout)
@@ -158,7 +158,7 @@ class KeyQueryServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id, device_id):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         timeout = parse_integer(request, "timeout", 10 * 1000)
         auth_user_id = requester.user.to_string()
         user_id = user_id if user_id else auth_user_id
@@ -204,7 +204,7 @@ class OneTimeKeyServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id, device_id, algorithm):
-        yield self.auth.get_user_by_req(request)
+        yield self.auth.get_user_by_req(request, allow_guest=True)
         timeout = parse_integer(request, "timeout", 10 * 1000)
         result = yield self.e2e_keys_handler.claim_one_time_keys(
             {"one_time_keys": {user_id: {device_id: algorithm}}},
@@ -214,7 +214,7 @@ class OneTimeKeyServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, user_id, device_id, algorithm):
-        yield self.auth.get_user_by_req(request)
+        yield self.auth.get_user_by_req(request, allow_guest=True)
         timeout = parse_integer(request, "timeout", 10 * 1000)
         body = parse_json_object_from_request(request)
         result = yield self.e2e_keys_handler.claim_one_time_keys(

synapse/rest/client/v2_alpha/register.py

@@ -15,6 +15,7 @@
 
 from twisted.internet import defer
 
+import synapse
 from synapse.api.auth import get_access_token_from_request, has_access_token
 from synapse.api.constants import LoginType
 from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError
@@ -100,12 +101,14 @@ class RegisterRestServlet(RestServlet):
     def on_POST(self, request):
         yield run_on_reactor()
 
+        body = parse_json_object_from_request(request)
+
         kind = "user"
         if "kind" in request.args:
             kind = request.args["kind"][0]
 
         if kind == "guest":
-            ret = yield self._do_guest_registration()
+            ret = yield self._do_guest_registration(body)
             defer.returnValue(ret)
             return
         elif kind != "user":
@@ -113,8 +116,6 @@ class RegisterRestServlet(RestServlet):
                 "Do not understand membership kind: %s" % (kind,)
             )
 
-        body = parse_json_object_from_request(request)
-
         # we do basic sanity checks here because the auth layer will store these
         # in sessions. Pull out the username/password provided to us.
         desired_password = None
@@ -420,13 +421,22 @@ class RegisterRestServlet(RestServlet):
         )
 
     @defer.inlineCallbacks
-    def _do_guest_registration(self):
+    def _do_guest_registration(self, params):
         if not self.hs.config.allow_guest_access:
             defer.returnValue((403, "Guest access is disabled"))
         user_id, _ = yield self.registration_handler.register(
             generate_token=False,
             make_guest=True
         )
+
+        # we don't allow guests to specify their own device_id, because
+        # we have nowhere to store it.
+        device_id = synapse.api.auth.GUEST_DEVICE_ID
+        initial_display_name = params.get("initial_device_display_name")
+        self.device_handler.check_device_registered(
+            user_id, device_id, initial_display_name
+        )
+
         access_token = self.auth_handler.generate_access_token(
             user_id, ["guest = true"]
         )
@@ -434,6 +444,7 @@ class RegisterRestServlet(RestServlet):
         # so long as we don't return a refresh_token here.
         defer.returnValue((200, {
             "user_id": user_id,
+            "device_id": device_id,
             "access_token": access_token,
             "home_server": self.hs.hostname,
         }))

synapse/rest/client/v2_alpha/sendtodevice.py

@@ -50,7 +50,7 @@ class SendToDeviceRestServlet(servlet.RestServlet):
 
     @defer.inlineCallbacks
     def _put(self, request, message_type, txn_id):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
 
         content = parse_json_object_from_request(request)
 

tests/handlers/test_auth.py

@@ -61,14 +61,14 @@ class AuthTestCase(unittest.TestCase):
         def verify_type(caveat):
             return caveat == "type = access"
 
-        def verify_expiry(caveat):
+        def verify_nonce(caveat):
-            return caveat == "time < 8600000"
+            return caveat.startswith("nonce =")
 
         v = pymacaroons.Verifier()
         v.satisfy_general(verify_gen)
         v.satisfy_general(verify_user)
         v.satisfy_general(verify_type)
-        v.satisfy_general(verify_expiry)
+        v.satisfy_general(verify_nonce)
         v.verify(macaroon, self.hs.config.macaroon_secret_key)
 
     def test_short_term_login_token_gives_user_id(self):

tests/handlers/test_register.py

@@ -53,13 +53,12 @@ class RegistrationTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_user_is_created_and_logged_in_if_doesnt_exist(self):
-        duration_ms = 200
         local_part = "someone"
         display_name = "someone"
         user_id = "@someone:test"
         requester = create_requester("@as:test")
         result_user_id, result_token = yield self.handler.get_or_create_user(
-            requester, local_part, display_name, duration_ms)
+            requester, local_part, display_name)
         self.assertEquals(result_user_id, user_id)
         self.assertEquals(result_token, 'secret')
 
@@ -71,12 +70,11 @@ class RegistrationTestCase(unittest.TestCase):
             user_id=frank.to_string(),
             token="jkv;g498752-43gj['eamb!-5",
             password_hash=None)
-        duration_ms = 200
         local_part = "frank"
         display_name = "Frank"
         user_id = "@frank:test"
         requester = create_requester("@as:test")
         result_user_id, result_token = yield self.handler.get_or_create_user(
-            requester, local_part, display_name, duration_ms)
+            requester, local_part, display_name)
         self.assertEquals(result_user_id, user_id)
         self.assertEquals(result_token, 'secret')