Merge branch 'develop' of github.com:matrix-org/synapse into develop

Conflicts:
	synapse/http/client.py

UPGRADE.rst

@@ -1,11 +1,6 @@
 Upgrading to v0.2.0
 ===================
 
-To upgrade the database schema, run::
-
-    ./database-prepare-for-0.2.0.sh "<database>.db"
-
-
 The home server now requires setting up of SSL config before it can run. To
 automatically generate default config use::
 

cmdclient/console.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -88,6 +88,8 @@ class SynapseCmd(cmd.Cmd):
         return False
 
     def _domain(self):
+        if "user" not in self.config or not self.config["user"]:
+            return None
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
@@ -191,8 +193,10 @@ class SynapseCmd(cmd.Cmd):
                 p = getpass.getpass("Enter your password: ")
                 user = args["user_id"]
                 if self._is_on("complete_usernames") and not user.startswith("@"):
-                    user = "@" + user + ":" + self._domain()
+                    domain = self._domain()
-
+                    if domain:
+                        user = "@" + user + ":" + domain
+                
                 reactor.callFromThread(self._do_login, user, p)
                 #print " got %s " % p
         except Exception as e:
@@ -312,7 +316,7 @@ class SynapseCmd(cmd.Cmd):
         try:
             args = self._parse(line, ["roomname"], force_keys=True)
             path = "/join/%s" % urllib.quote(args["roomname"])
-            reactor.callFromThread(self._run_and_pprint, "PUT", path, {})
+            reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
             print e
 
@@ -700,7 +704,7 @@ def main(server_url, identity_server_url, username, token, config_path):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser("Starts a synapse client.")
     parser.add_argument(
-        "-s", "--server", dest="server", default="http://localhost:8080",
+        "-s", "--server", dest="server", default="http://localhost:8008",
         help="The URL of the home server to talk to.")
     parser.add_argument(
         "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",

cmdclient/http.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

database-prepare-for-0.2.0.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# This is will prepare a synapse database for running with v0.2.0 of synapse. 
-
-set -e
-
-cp "$1" "$1.bak"
-
-sqlite3 "$1" < "synapse/storage/schema/im.sql" 
-sqlite3 "$1" <<< "PRAGMA user_version = 2;"

docs/client-server/OLD_specification.rst

@@ -2,6 +2,20 @@
 Matrix Client-Server API
 ========================
 
+
+.. WARNING::
+  This specification is old. Please see /docs/specification.rst instead.
+
+
+
+
+
+
+
+
+
+
+
 The following specification outlines how a client can send and receive data from 
 a home server.
 

docs/client-server/swagger_matrix/api-docs-directory

@@ -1,7 +1,7 @@
 {
   "apiVersion": "1.0.0",
   "swaggerVersion": "1.2",
-  "basePath": "http://localhost:8080/_matrix/client/api/v1",
+  "basePath": "http://localhost:8008/_matrix/client/api/v1",
   "resourcePath": "/directory",
   "produces": [
     "application/json"
@@ -13,6 +13,7 @@
         {
           "method": "GET",
           "summary": "Get the room ID corresponding to this room alias.",
+          "notes": "Volatile: This API is likely to change.",
           "type": "DirectoryResponse",
           "nickname": "get_room_id_for_alias",
           "parameters": [
@@ -28,6 +29,7 @@
         {
           "method": "PUT",
           "summary": "Create a new mapping from room alias to room ID.",
+          "notes": "Volatile: This API is likely to change.",
           "type": "void",
           "nickname": "add_room_alias",
           "parameters": [

docs/client-server/swagger_matrix/api-docs-events

@@ -1,7 +1,7 @@
 {
   "apiVersion": "1.0.0",
   "swaggerVersion": "1.2",
-  "basePath": "http://localhost:8080/_matrix/client/api/v1",
+  "basePath": "http://localhost:8008/_matrix/client/api/v1",
   "resourcePath": "/events",
   "produces": [
     "application/json"

docs/client-server/swagger_matrix/api-docs-login

@@ -8,7 +8,7 @@
           "nickname": "get_login_info", 
           "notes": "All login stages MUST be mentioned if there is >1 login type.", 
           "summary": "Get the login mechanism to use when logging in.", 
-          "type": "LoginInfo"
+          "type": "LoginFlows"
         }, 
         {
           "method": "POST", 
@@ -40,17 +40,31 @@
       "path": "/login"
     }
   ], 
-  "basePath": "http://localhost:8080/_matrix/client/api/v1", 
+  "basePath": "http://localhost:8008/_matrix/client/api/v1", 
   "consumes": [
     "application/json"
   ], 
   "models": {
+    "LoginFlows": {
+      "id": "LoginFlows",
+      "properties": {
+        "flows": {
+          "description": "A list of valid login flows.",
+          "type": "array",
+          "items": {
+            "$ref": "LoginInfo"
+          }
+        }
+      }
+    },
     "LoginInfo": {
       "id": "LoginInfo", 
       "properties": {
         "stages": {
           "description": "Multi-stage login only: An array of all the login types required to login.", 
-          "format": "string", 
+          "items": {
+            "$ref": "string"
+          }, 
           "type": "array"
         }, 
         "type": {
@@ -65,6 +79,10 @@
         "access_token": {
           "description": "The access token for this user's login if this is the final stage of the login process.", 
           "type": "string"
+        },
+        "user_id": {
+          "description": "The user's fully-qualified user ID.",
+          "type": "string"
         }, 
         "next": {
           "description": "Multi-stage login only: The next login type to submit.", 

docs/client-server/swagger_matrix/api-docs-presence

@@ -1,7 +1,7 @@
 {
   "apiVersion": "1.0.0",
   "swaggerVersion": "1.2",
-  "basePath": "http://localhost:8080/_matrix/client/api/v1",
+  "basePath": "http://localhost:8008/_matrix/client/api/v1",
   "resourcePath": "/presence",
   "produces": [
     "application/json"
@@ -106,7 +106,7 @@
     "PresenceUpdate": {
       "id": "PresenceUpdate",
       "properties": {
-        "state": {
+        "presence": {
           "type": "string",
           "description": "Enum: The presence state.",
           "enum": [
@@ -128,10 +128,10 @@
     "Presence": {
       "id": "Presence",
       "properties": {
-        "mtime_age": {
+        "last_active_ago": {
           "type": "integer",
           "format": "int64",
-          "description": "The last time this user's presence state changed, in milliseconds."
+          "description": "The last time this user performed an action on their home server."
         },
         "user_id": {
           "type": "string",

docs/client-server/swagger_matrix/api-docs-profile

@@ -1,7 +1,7 @@
 {
   "apiVersion": "1.0.0",
   "swaggerVersion": "1.2",
-  "basePath": "http://localhost:8080/_matrix/client/api/v1",
+  "basePath": "http://localhost:8008/_matrix/client/api/v1",
   "resourcePath": "/profile",
   "produces": [
     "application/json"

docs/client-server/swagger_matrix/api-docs-registration

@@ -37,7 +37,7 @@
       "path": "/register"
     }
   ], 
-  "basePath": "http://localhost:8080/_matrix/client/api/v1", 
+  "basePath": "http://localhost:8008/_matrix/client/api/v1", 
   "consumes": [
     "application/json"
   ], 
@@ -52,6 +52,10 @@
         "user_id": {
           "description": "The fully-qualified user ID.", 
           "type": "string"
+        },
+        "home_server": {
+          "description": "The name of the home server.",
+          "type": "string"
         }
       }
     }, 

docs/client-server/swagger_matrix/api-docs-rooms

@@ -1,7 +1,7 @@
 {
   "apiVersion": "1.0.0",
   "swaggerVersion": "1.2",
-  "basePath": "http://localhost:8080/_matrix/client/api/v1", 
+  "basePath": "http://localhost:8008/_matrix/client/api/v1", 
   "resourcePath": "/rooms",
   "produces": [
     "application/json"
@@ -180,6 +180,59 @@
         }
       ]
     },
+    {
+      "path": "/rooms/{roomId}/state/m.room.name",
+      "operations": [
+        {
+          "method": "PUT",
+          "summary": "Set the name of this room.",
+          "notes": "Set the name of this room.",
+          "type": "void",
+          "nickname": "set_room_name",
+          "consumes": [
+            "application/json"
+          ],
+          "parameters": [
+            {
+              "name": "body",
+              "description": "The name contents",
+              "required": true,
+              "type": "RoomName",
+              "paramType": "body"
+            },
+            {
+              "name": "roomId",
+              "description": "The room to set the name of.",
+              "required": true,
+              "type": "string",
+              "paramType": "path"
+            }
+          ]
+        },
+        {
+          "method": "GET",
+          "summary": "Get the room's name.",
+          "notes": "",
+          "type": "RoomName",
+          "nickname": "get_room_name",
+          "parameters": [
+            {
+              "name": "roomId",
+              "description": "The room to get the name of.",
+              "required": true,
+              "type": "string",
+              "paramType": "path"
+            }
+          ],
+          "responseMessages": [
+            {
+              "code": 404,
+              "message": "Name not found."
+            }
+          ]
+        }
+      ]
+    },
     {
       "path": "/rooms/{roomId}/send/m.room.message.feedback",
       "operations": [
@@ -267,6 +320,12 @@
               "required": true,
               "type": "string",
               "paramType": "path"
+            },
+            {
+              "name": "body",
+              "required": true,
+              "type": "JoinRequest",
+              "paramType": "body"
             }
           ]
         }  
@@ -291,6 +350,12 @@
               "required": true,
               "type": "string",
               "paramType": "path"
+            },
+            {
+              "name": "body",
+              "required": true,
+              "type": "LeaveRequest",
+              "paramType": "body"
             }
           ]
         }  
@@ -424,10 +489,10 @@
       "path": "/join/{roomAliasOrId}",
       "operations": [
         {
-          "method": "PUT",
+          "method": "POST",
           "summary": "Join a room via a room alias or room ID.",
           "notes": "Join a room via a room alias or room ID.",
-          "type": "RoomInfo",
+          "type": "JoinRoomInfo",
           "nickname": "join",
           "consumes": [
             "application/json"
@@ -574,7 +639,7 @@
         {
           "method": "GET",
           "summary": "Get a list of all the current state events for this room.",
-          "notes": "Get a list of all the current state events for this room.",
+          "notes": "NOT YET IMPLEMENTED.",
           "type": "array",
           "items": {
             "$ref": "Event"
@@ -598,7 +663,7 @@
         {
           "method": "GET",
           "summary": "Get all the current information for this room, including messages and state events.",
-          "notes": "Get all the current information for this room, including messages and state events.",
+          "notes": "NOT YET IMPLEMENTED.",
           "type": "InitialSyncRoomData",
           "nickname": "get_room_sync_data",
           "parameters": [
@@ -624,6 +689,15 @@
         }
       }
     },
+    "RoomName": {
+      "id": "RoomName",
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "The human-readable name for the room. Can contain spaces."
+        }
+      }
+    },
     "Message": {
       "id": "Message",
       "properties": {
@@ -640,6 +714,16 @@
     "Feedback": {
       "id": "Feedback",
       "properties": {
+        "target_event_id": {
+          "type": "string",
+          "description": "The event ID being acknowledged.",
+          "required": true
+        },
+        "type": {
+          "type": "string",
+          "description": "The type of feedback. Either 'delivered' or 'read'.",
+          "required": true
+        }
       }
     },
     "Member": {
@@ -652,7 +736,7 @@
             "invite",
             "join",
             "leave",
-            "knock"
+            "ban"
           ]
         }
       }
@@ -672,6 +756,16 @@
         }
       }
     },
+    "JoinRoomInfo": {
+      "id": "JoinRoomInfo",
+      "properties": {
+        "room_id": {
+          "type": "string",
+          "description": "The room ID joined, if joined via a room alias only.",
+          "required": true
+        }
+      }
+    },
     "RoomConfig": {
       "id": "RoomConfig",
       "properties": {
@@ -830,6 +924,14 @@
         }
       }
     },
+    "JoinRequest": {
+      "id": "JoinRequest",
+      "properties": {}
+    },
+    "LeaveRequest": {
+      "id": "LeaveRequest",
+      "properties": {}
+    },
     "BanRequest": {
       "id": "BanRequest",
       "properties": {

docs/server-server/security-threat-model.rst

@@ -1,141 +0,0 @@
-Overview
-========
-
-Scope
------
-
-This document considers threats specific to the server to server federation 
-synapse protocol.
-
-
-Attacker
---------
-
-It is assumed that the attacker can see and manipulate all network traffic 
-between any of the servers and may be in control of one or more homeservers 
-participating in the federation protocol.
-
-Threat Model
-============
-
-Denial of Service
------------------
-
-The attacker could attempt to prevent delivery of messages to or from the 
-victim in order to:
-
-    * Disrupt service or marketing campaign of a commercial competitor.
-    * Censor a discussion or censor a participant in a discussion.
-    * Perform general vandalism.
-
-Threat: Resource Exhaustion
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could cause the victims server to exhaust a particular resource 
-(e.g. open TCP connections, CPU, memory, disk storage)
-
-Threat: Unrecoverable Consistency Violations
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could send messages which created an unrecoverable "split-brain"
-state in the cluster such that the victim's servers could no longer dervive a
-consistent view of the chatroom state.
-
-Threat: Bad History
-~~~~~~~~~~~~~~~~~~~
-
-An attacker could convince the victim to accept invalid messages which the 
-victim would then include in their view of the chatroom history. Other servers
-in the chatroom would reject the invalid messages and potentially reject the
-victims messages as well since they depended on the invalid messages.
-
-Threat: Block Network Traffic
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could try to firewall traffic between the victim's server and some
-or all of the other servers in the chatroom.
-
-Threat: High Volume of Messages
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could send large volumes of messages to a chatroom with the victim
-making the chatroom unusable.
-
-Threat: Banning users without necessary authorisation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could attempt to ban a user from a chatroom with the necessary
-authorisation.
-
-Spoofing
---------
-
-An attacker could try to send a message claiming to be from the victim without 
-the victim having sent the message in order to:
-
-    * Impersonate the victim while performing illict activity.
-    * Obtain privileges of the victim.
-
-Threat: Altering Message Contents
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could try to alter the contents of an existing message from the 
-victim.
-
-Threat: Fake Message "origin" Field
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could try to send a new message purporting to be from the victim
-with a phony "origin" field.
-
-Spamming
---------
-
-The attacker could try to send a high volume of solicicted or unsolicted 
-messages to the victim in order to:
-    
-    * Find victims for scams.
-    * Market unwanted products.
-
-Threat: Unsoliticted Messages
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could try to send messages to victims who do not wish to receive 
-them.
-
-Threat: Abusive Messages
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could send abusive or threatening messages to the victim
-
-Spying
-------
-
-The attacker could try to access message contents or metadata for messages sent
-by the victim or to the victim that were not intended to reach the attacker in
-order to:
-
-    * Gain sensitive personal or commercial information.
-    * Impersonate the victim using credentials contained in the messages.
-      (e.g. password reset messages)
-    * Discover who the victim was talking to and when.
-
-Threat: Disclosure during Transmission
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could try to expose the message contents or metadata during 
-transmission between the servers.
-
-Threat: Disclosure to Servers Outside Chatroom
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could try to convince servers within a chatroom to send messages to
-a server it controls that was not authorised to be within the chatroom.
-
-Threat: Disclosure to Servers Within Chatroom
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-An attacker could take control of a server within a chatroom to expose message
-contents or metadata for messages in that room.
-
-

docs/specification.rst

@@ -30,48 +30,52 @@ ecosystem to communicate with one another.
 
 The principles that Matrix attempts to follow are:
 
- - Pragmatic Web-friendly APIs (i.e. JSON over REST)
+- Pragmatic Web-friendly APIs (i.e. JSON over REST)
- - Keep It Simple & Stupid
+- Keep It Simple & Stupid
 
-   + provide a simple architecture with minimal third-party dependencies.
+  + provide a simple architecture with minimal third-party dependencies.
 
- - Fully open:
+- Fully open:
 
-   + Fully open federation - anyone should be able to participate in the global Matrix network
+  + Fully open federation - anyone should be able to participate in the global
-   + Fully open standard - publicly documented standard with no IP or patent licensing encumbrances
+    Matrix network
-   + Fully open source reference implementation - liberally-licensed example implementations with no
+  + Fully open standard - publicly documented standard with no IP or patent
-     IP or patent licensing encumbrances
+    licensing encumbrances
+  + Fully open source reference implementation - liberally-licensed example
+    implementations with no IP or patent licensing encumbrances
 
- - Empowering the end-user
+- Empowering the end-user
 
-   + The user should be able to choose the server and clients they use
+  + The user should be able to choose the server and clients they use
-   + The user should be control how private their communication is
+  + The user should be control how private their communication is
-   + The user should know precisely where their data is stored
+  + The user should know precisely where their data is stored
 
- - Fully decentralised - no single points of control over conversations or the network as a whole
+- Fully decentralised - no single points of control over conversations or the
- - Learning from history to avoid repeating it
+  network as a whole
+- Learning from history to avoid repeating it
 
-   + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP whilst trying to avoid their failings
+  + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP
+    whilst trying to avoid their failings
 
 The functionality that Matrix provides includes:
 
- - Creation and management of fully distributed chat rooms with no
+- Creation and management of fully distributed chat rooms with no
-   single points of control or failure
+  single points of control or failure
- - Eventually-consistent cryptographically secure synchronisation of room 
+- Eventually-consistent cryptographically secure synchronisation of room
-   state across a global open network of federated servers and services
+  state across a global open network of federated servers and services
- - Sending and receiving extensible messages in a room with (optional)
+- Sending and receiving extensible messages in a room with (optional)
-   end-to-end encryption
+  end-to-end encryption
- - Extensible user management (inviting, joining, leaving, kicking, banning)
+- Extensible user management (inviting, joining, leaving, kicking, banning)
-   mediated by a power-level based user privilege system.
+  mediated by a power-level based user privilege system.
- - Extensible room state management (room naming, aliasing, topics, bans)
+- Extensible room state management (room naming, aliasing, topics, bans)
- - Extensible user profile management (avatars, displaynames, etc)
+- Extensible user profile management (avatars, displaynames, etc)
- - Managing user accounts (registration, login, logout)
+- Managing user accounts (registration, login, logout)
- - Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
+- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
-   Facebook accounts to authenticate, identify and discover users on Matrix.
+  Facebook accounts to authenticate, identify and discover users on Matrix.
- - Trusted federation of Identity servers for:
+- Trusted federation of Identity servers for:
 
-   + Publishing user public keys for PKI
+  + Publishing user public keys for PKI
-   + Mapping of 3PIDs to Matrix IDs
+  + Mapping of 3PIDs to Matrix IDs
 
 The end goal of Matrix is to be a ubiquitous messaging layer for synchronising
 arbitrary data between sets of people, devices and services - be that for instant
@@ -428,9 +432,10 @@ event also has a ``creator`` key which contains the user ID of the room
 creator. It will also generate several other events in order to manage
 permissions in this room. This includes:
 
- - ``m.room.power_levels`` : Sets the authority of the room creator.
+ - ``m.room.power_levels`` : Sets the power levels of users.
  - ``m.room.join_rules`` : Whether the room is "invite-only" or not.
- - ``m.room.add_state_level``
+ - ``m.room.add_state_level``: The power level required in order to
+   add new state to the room (as opposed to updating exisiting state)
  - ``m.room.send_event_level`` : The power level required in order to
    send a message in this room.
  - ``m.room.ops_level`` : The power level required in order to kick or
@@ -463,6 +468,27 @@ Permissions
       Link through to respective sections where necessary. How does this tie in with permissions, e.g.
       give example of creating a read-only room.
 
+Permissions for rooms are done via the concept of power levels - to do any
+action in a room a user must have a suitable power level. 
+
+Power levels for users are defined in ``m.room.power_levels``, where both
+a default and specific users' power levels can be set. By default all users
+have a power level of 0.
+
+State events may contain a ``required_power_level`` key, which indicates the
+minimum power a user must have before they can update that state key. The only
+exception to this is when a user leaves a room.
+
+To perform certain actions there are additional power level requirements
+defined in the following state events:
+
+- ``m.room.send_event_level`` defines the minimum level for sending non-state 
+  events. Defaults to 5.
+- ``m.room.add_state_level`` defines the minimum level for adding new state,
+  rather than updating existing state. Defaults to 5.
+- ``m.room.ops_level`` defines the minimum levels to ban and kick other users.
+  This defaults to a kick and ban levels of 5 each.
+
 
 Joining rooms
 -------------
@@ -797,89 +823,88 @@ prefixed with ``m.``
     to force this state change directly will fail. See the `Rooms`_ section for how to 
     use the membership APIs.
 
-``m.room.config``
+``m.room.create``
   Summary:
-    The room config.
+    The first event in the room.
   Type: 
     State event
   JSON format:
-    TODO
+    ``{ "creator": "string"}``
   Example:
-    TODO
+    ``{ "creator": "@user:example.com" }``
   Description:
-    TODO : What it represents, What are the valid keys / values and what they represent, When is this event emitted and by what
+    This is the first event in a room and cannot be changed. It acts as the 
+    root of all other events.
 
-``m.room.invite_join``
-  Summary:
-    TODO.
-  Type: 
-    State event
-  JSON format:
-    TODO
-  Example:
-    TODO
-  Description:
-    TODO : What it represents, What are the valid keys / values and what they represent, When is this event emitted and by what
-    
 ``m.room.join_rules``
   Summary:
-    TODO.
+    Descripes how/if people are allowed to join.
   Type: 
     State event
   JSON format:
-    TODO
+    ``{ "join_rule": "enum [ public|knock|invite|private ]" }``
   Example:
-    TODO
+    ``{ "join_rule": "public" }``
   Description:
-    TODO : What it represents, What are the valid keys / values and what they represent, When is this event emitted and by what
+    TODO : Use docs/models/rooms.rst
-    
+   
 ``m.room.power_levels``
   Summary:
-    TODO.
+    Defines the power levels of users in the room.
   Type: 
     State event
   JSON format:
-    TODO
+    ``{ "<user_id>": <int>, ..., "default": <int>}``
   Example:
-    TODO
+    ``{ "@user:example.com": 5, "@user2:example.com": 10, "default": 0 }`` 
   Description:
-    TODO : What it represents, What are the valid keys / values and what they represent, When is this event emitted and by what
+    If a user is in the list, then they have the associated power level. 
-    
+    Otherwise they have the default level. If not ``default`` key is supplied,
+    it is assumed to be 0.
+
 ``m.room.add_state_level``
   Summary:
-    TODO.
+    Defines the minimum power level a user needs to add state.
   Type: 
     State event
   JSON format:
-    TODO
+    ``{ "level": <int> }``
   Example:
-    TODO
+    ``{ "level": 5 }``
   Description:
-    TODO : What it represents, What are the valid keys / values and what they represent, When is this event emitted and by what
+    To add a new piece of state to the room a user must have the given power 
+    level. This does not apply to updating current state, which is goverened
+    by the ``required_power_level`` event key.
     
 ``m.room.send_event_level``
   Summary:
-    TODO.
+    Defines the minimum power level a user needs to send an event.
   Type: 
     State event
   JSON format:
-    TODO
+    ``{ "level": <int> }``
   Example:
-    TODO
+    ``{ "level": 0 }``
   Description:
-    TODO : What it represents, What are the valid keys / values and what they represent, When is this event emitted and by what
+    To send a new event into the room a user must have at least this power 
-    
+    level. This allows ops to make the room read only by increasing this level,
+    or muting individual users by lowering their power level below this
+    threshold.
+
 ``m.room.ops_levels``
   Summary:
-    TODO.
+    Defines the minimum power levels that a user must have before they can 
+    kick and/or ban other users.
   Type: 
     State event
   JSON format:
-    TODO
+    ``{ "ban_level": <int>, "kick_level": <int> }``
   Example:
-    TODO
+    ``{ "ban_level": 5, "kick_level": 5 }``
   Description:
-    TODO : What it represents, What are the valid keys / values and what they represent, When is this event emitted and by what
+    This defines who can ban and/or kick people in the room. Most of the time
+    ``ban_level`` will be greater than or equal to ``kick_level`` since 
+    banning is more severe than kicking.
 
 ``m.room.message``
   Summary:
@@ -1485,6 +1510,31 @@ Each transaction has:
  - An origin and destination server name.
  - A list of "previous IDs".
  - A list of PDUs and EDUs - the actual message payload that the Transaction carries.
+ 
+``origin``
+  Type: 
+    String
+  Description:
+    DNS name of homeserver making this transaction.
+    
+``ts``
+  Type: 
+    Integer
+  Description:
+    Timestamp in milliseconds on originating homeserver when this transaction 
+    started.
+    
+``previous_ids``
+  Type:
+    List of strings
+  Description:
+    List of transactions that were sent immediately prior to this transaction.
+    
+``pdus``
+  Type:
+    List of Objects.
+  Description:
+    List of updates contained in this transaction.
 
 ::
 
@@ -1526,8 +1576,98 @@ All PDUs have:
  - A list of other PDU IDs that have been seen recently on that context (regardless of which origin
    sent them)
 
-[[TODO(paul): Update this structure so that 'pdu_id' is a two-element
+``context``
-[origin,ref] pair like the prev_pdus are]]
+  Type:
+    String
+  Description:
+    Event context identifier
+    
+``origin``
+  Type:
+    String
+  Description:
+    DNS name of homeserver that created this PDU.
+    
+``pdu_id``
+  Type:
+    String
+  Description:
+    Unique identifier for PDU within the context for the originating homeserver
+
+``ts``
+  Type:
+    Integer
+  Description:
+    Timestamp in milliseconds on originating homeserver when this PDU was created.
+
+``pdu_type``
+  Type:
+    String
+  Description:
+    PDU event type.
+
+``prev_pdus``
+  Type:
+    List of pairs of strings
+  Description:
+    The originating homeserver and PDU ids of the most recent PDUs the 
+    homeserver was aware of for this context when it made this PDU.
+
+``depth``
+  Type:
+    Integer
+  Description:
+    The maximum depth of the previous PDUs plus one.
+
+
+.. TODO paul
+  [[TODO(paul): Update this structure so that 'pdu_id' is a two-element
+  [origin,ref] pair like the prev_pdus are]]
+  
+
+For state updates:
+
+``is_state``
+  Type:
+    Boolean
+  Description:
+    True if this PDU is updating state.
+    
+``state_key``
+  Type:
+    String
+  Description:
+    Optional key identifying the updated state within the context.
+    
+``power_level``
+  Type:
+    Integer
+  Description:
+    The asserted power level of the user performing the update.
+    
+``min_update``
+  Type:
+    Integer
+  Description:
+    The required power level needed to replace this update.
+
+``prev_state_id``
+  Type:
+    String
+  Description:
+    PDU event type.
+    
+``prev_state_origin``
+  Type:
+    String
+  Description:
+    The PDU id of the update this replaces.
+    
+``user``
+  Type:
+    String
+  Description:
+    The user updating the state.
 
 ::
 
@@ -1568,12 +1708,13 @@ keys exist to support this:
   "prev_state_id":TODO
   "prev_state_origin":TODO}
 
-[[TODO(paul): At this point we should probably have a long description of how
+.. TODO paul
-State management works, with descriptions of clobbering rules, power levels, etc
+  [[TODO(paul): At this point we should probably have a long description of how
-etc... But some of that detail is rather up-in-the-air, on the whiteboard, and
+  State management works, with descriptions of clobbering rules, power levels, etc
-so on. This part needs refining. And writing in its own document as the details
+  etc... But some of that detail is rather up-in-the-air, on the whiteboard, and
-relate to the server/system as a whole, not specifically to server-server
+  so on. This part needs refining. And writing in its own document as the details
-federation.]]
+  relate to the server/system as a whole, not specifically to server-server
+  federation.]]
 
 EDUs, by comparison to PDUs, do not have an ID, a context, or a list of
 "previous" IDs. The only mandatory fields for these are the type, origin and
@@ -1585,6 +1726,79 @@ destination home server names, and the actual nested content.
   "origin":"blue",
   "destination":"orange",
   "content":...}
+  
+  
+Protocol URLs
+=============
+.. WARNING::
+  This section may be misleading or inaccurate.
+
+All these URLs are namespaced within a prefix of::
+
+  /_matrix/federation/v1/...
+
+For active pushing of messages representing live activity "as it happens"::
+
+  PUT .../send/:transaction_id/
+    Body: JSON encoding of a single Transaction
+    Response: TODO
+
+The transaction_id path argument will override any ID given in the JSON body.
+The destination name will be set to that of the receiving server itself. Each
+embedded PDU in the transaction body will be processed.
+
+
+To fetch a particular PDU::
+
+  GET .../pdu/:origin/:pdu_id/
+    Response: JSON encoding of a single Transaction containing one PDU
+
+Retrieves a given PDU from the server. The response will contain a single new
+Transaction, inside which will be the requested PDU.
+  
+
+To fetch all the state of a given context::
+
+  GET .../state/:context/
+    Response: JSON encoding of a single Transaction containing multiple PDUs
+
+Retrieves a snapshot of the entire current state of the given context. The
+response will contain a single Transaction, inside which will be a list of
+PDUs that encode the state.
+
+To backfill events on a given context::
+
+  GET .../backfill/:context/
+    Query args: v, limit
+    Response: JSON encoding of a single Transaction containing multiple PDUs
+
+Retrieves a sliding-window history of previous PDUs that occurred on the
+given context. Starting from the PDU ID(s) given in the "v" argument, the
+PDUs that preceeded it are retrieved, up to a total number given by the
+"limit" argument. These are then returned in a new Transaction containing all
+off the PDUs.
+
+
+To stream events all the events::
+
+  GET .../pull/
+    Query args: origin, v
+    Response: JSON encoding of a single Transaction consisting of multiple PDUs
+
+Retrieves all of the transactions later than any version given by the "v"
+arguments.
+
+
+To make a query::
+
+  GET .../query/:query_type
+    Query args: as specified by the individual query types
+    Response: JSON encoding of a response object
+
+Performs a single query request on the receiving home server. The Query Type
+part of the path specifies the kind of query being made, and its query
+arguments have a meaning specific to that kind of query. The response is a
+JSON-encoded object whose meaning also depends on the kind of query.
 
 Backfilling
 -----------
@@ -1604,9 +1818,133 @@ SRV Records
 
 Security
 ========
+
 .. NOTE::
   This section is a work in progress.
 
+Threat Model
+------------
+
+Denial of Service
+~~~~~~~~~~~~~~~~~
+
+The attacker could attempt to prevent delivery of messages to or from the
+victim in order to:
+
+* Disrupt service or marketing campaign of a commercial competitor.
+* Censor a discussion or censor a participant in a discussion.
+* Perform general vandalism.
+
+Threat: Resource Exhaustion
++++++++++++++++++++++++++++
+
+An attacker could cause the victims server to exhaust a particular resource
+(e.g. open TCP connections, CPU, memory, disk storage)
+
+Threat: Unrecoverable Consistency Violations
+++++++++++++++++++++++++++++++++++++++++++++
+
+An attacker could send messages which created an unrecoverable "split-brain"
+state in the cluster such that the victim's servers could no longer dervive a
+consistent view of the chatroom state.
+
+Threat: Bad History
++++++++++++++++++++
+
+An attacker could convince the victim to accept invalid messages which the
+victim would then include in their view of the chatroom history. Other servers
+in the chatroom would reject the invalid messages and potentially reject the
+victims messages as well since they depended on the invalid messages.
+
+Threat: Block Network Traffic
++++++++++++++++++++++++++++++
+
+An attacker could try to firewall traffic between the victim's server and some
+or all of the other servers in the chatroom.
+
+Threat: High Volume of Messages
++++++++++++++++++++++++++++++++
+
+An attacker could send large volumes of messages to a chatroom with the victim
+making the chatroom unusable.
+
+Threat: Banning users without necessary authorisation
++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+An attacker could attempt to ban a user from a chatroom with the necessary
+authorisation.
+
+Spoofing
+~~~~~~~~
+
+An attacker could try to send a message claiming to be from the victim without
+the victim having sent the message in order to:
+
+* Impersonate the victim while performing illict activity.
+* Obtain privileges of the victim.
+
+Threat: Altering Message Contents
++++++++++++++++++++++++++++++++++
+
+An attacker could try to alter the contents of an existing message from the
+victim.
+
+Threat: Fake Message "origin" Field
++++++++++++++++++++++++++++++++++++
+
+An attacker could try to send a new message purporting to be from the victim
+with a phony "origin" field.
+
+Spamming
+~~~~~~~~
+
+The attacker could try to send a high volume of solicicted or unsolicted
+messages to the victim in order to:
+
+* Find victims for scams.
+* Market unwanted products.
+
+Threat: Unsoliticted Messages
++++++++++++++++++++++++++++++
+
+An attacker could try to send messages to victims who do not wish to receive
+them.
+
+Threat: Abusive Messages
+++++++++++++++++++++++++
+
+An attacker could send abusive or threatening messages to the victim
+
+Spying
+~~~~~~
+
+The attacker could try to access message contents or metadata for messages sent
+by the victim or to the victim that were not intended to reach the attacker in
+order to:
+
+* Gain sensitive personal or commercial information.
+* Impersonate the victim using credentials contained in the messages.
+  (e.g. password reset messages)
+* Discover who the victim was talking to and when.
+
+Threat: Disclosure during Transmission
+++++++++++++++++++++++++++++++++++++++
+
+An attacker could try to expose the message contents or metadata during
+transmission between the servers.
+
+Threat: Disclosure to Servers Outside Chatroom
+++++++++++++++++++++++++++++++++++++++++++++++
+
+An attacker could try to convince servers within a chatroom to send messages to
+a server it controls that was not authorised to be within the chatroom.
+
+Threat: Disclosure to Servers Within Chatroom
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+An attacker could take control of a server within a chatroom to expose message
+contents or metadata for messages in that room.
+
 Rate limiting
 -------------
 Home servers SHOULD implement rate limiting to reduce the risk of being overloaded. If a
@@ -1660,57 +1998,121 @@ Glossary
 .. NOTE::
   This section is a work in progress.
 
-.. TODO
+Backfilling:
-  - domain specific words/acronyms with definitions
+  The process of synchronising historic state from one home server to another,
+  to backfill the event storage so that scrollback can be presented to the
+  client(s). Not to be confused with pagination.
+
+Context:
+  A single human-level entity of interest (currently, a chat room)
+
+EDU (Ephemeral Data Unit):
+  A message that relates directly to a given pair of home servers that are
+  exchanging it. EDUs are short-lived messages that related only to one single
+  pair of servers; they are not persisted for a long time and are not forwarded
+  on to other servers. Because of this, they have no internal ID nor previous
+  EDUs reference chain.
+
+Event:
+  A record of activity that records a single thing that happened on to a context
+  (currently, a chat room). These are the "chat messages" that Synapse makes
+  available.
+
+PDU (Persistent Data Unit):
+  A message that relates to a single context, irrespective of the server that
+  is communicating it. PDUs either encode a single Event, or a single State
+  change. A PDU is referred to by its PDU ID; the pair of its origin server
+  and local reference from that server.
+
+PDU ID:
+  The pair of PDU Origin and PDU Reference, that together globally uniquely
+  refers to a specific PDU.
+
+PDU Origin:
+  The name of the origin server that generated a given PDU. This may not be the
+  server from which it has been received, due to the way they are copied around
+  from server to server. The origin always records the original server that
+  created it.
+
+PDU Reference:
+  A local ID used to refer to a specific PDU from a given origin server. These
+  references are opaque at the protocol level, but may optionally have some
+  structured meaning within a given origin server or implementation.
+
+Presence:
+  The concept of whether a user is currently online, how available they declare
+  they are, and so on. See also: doc/model/presence
+
+Profile:
+  A set of metadata about a user, such as a display name, provided for the
+  benefit of other users. See also: doc/model/profiles
+
+Room ID:
+  An opaque string (of as-yet undecided format) that identifies a particular
+  room and used in PDUs referring to it.
+
+Room Alias:
+  A human-readable string of the form #name:some.domain that users can use as a
+  pointer to identify a room; a Directory Server will map this to its Room ID
+
+State:
+  A set of metadata maintained about a Context, which is replicated among the
+  servers in addition to the history of Events.
 
 User ID:
-  An opaque ID which identifies an end-user, which consists of some opaque 
+  A string of the form @localpart:domain.name that identifies a user for
-  localpart combined with the domain name of their home server. 
+  wire-protocol purposes. The localpart is meaningless outside of a particular
+  home server. This takes a human-readable form that end-users can use directly
+  if they so wish, avoiding the 3PIDs.
+
+Transaction:
+  A message which relates to the communication between a given pair of servers.
+  A transaction contains possibly-empty lists of PDUs and EDUs.
 
 
 .. Links through the external API docs are below
 .. =============================================
 
 .. |createRoom| replace:: ``/createRoom``
-.. _createRoom: /-rooms/create_room
+.. _createRoom: /docs/api/client-server/#!/-rooms/create_room
 
 .. |initialSync| replace:: ``/initialSync``
-.. _initialSync: /-events/initial_sync
+.. _initialSync: /docs/api/client-server/#!/-events/initial_sync
 
 .. |/rooms/<room_id>/initialSync| replace:: ``/rooms/<room_id>/initialSync``
-.. _/rooms/<room_id>/initialSync: /-rooms/get_room_sync_data
+.. _/rooms/<room_id>/initialSync: /docs/api/client-server/#!/-rooms/get_room_sync_data
 
 .. |login| replace:: ``/login``
-.. _login: /-login
+.. _login: /docs/api/client-server/#!/-login
 
 .. |/rooms/<room_id>/messages| replace:: ``/rooms/<room_id>/messages``
-.. _/rooms/<room_id>/messages: /-rooms/get_messages
+.. _/rooms/<room_id>/messages: /docs/api/client-server/#!/-rooms/get_messages
 
 .. |/rooms/<room_id>/members| replace:: ``/rooms/<room_id>/members``
-.. _/rooms/<room_id>/members: /-rooms/get_members
+.. _/rooms/<room_id>/members: /docs/api/client-server/#!/-rooms/get_members
 
 .. |/rooms/<room_id>/state| replace:: ``/rooms/<room_id>/state``
-.. _/rooms/<room_id>/state: /-rooms/get_state_events
+.. _/rooms/<room_id>/state: /docs/api/client-server/#!/-rooms/get_state_events
 
 .. |/rooms/<room_id>/send/<event_type>| replace:: ``/rooms/<room_id>/send/<event_type>``
-.. _/rooms/<room_id>/send/<event_type>: /-rooms/send_non_state_event
+.. _/rooms/<room_id>/send/<event_type>: /docs/api/client-server/#!/-rooms/send_non_state_event
 
 .. |/rooms/<room_id>/state/<event_type>/<state_key>| replace:: ``/rooms/<room_id>/state/<event_type>/<state_key>``
-.. _/rooms/<room_id>/state/<event_type>/<state_key>: /-rooms/send_state_event
+.. _/rooms/<room_id>/state/<event_type>/<state_key>: /docs/api/client-server/#!/-rooms/send_state_event
 
 .. |/rooms/<room_id>/invite| replace:: ``/rooms/<room_id>/invite``
-.. _/rooms/<room_id>/invite: /-rooms/invite
+.. _/rooms/<room_id>/invite: /docs/api/client-server/#!/-rooms/invite
 
 .. |/rooms/<room_id>/join| replace:: ``/rooms/<room_id>/join``
-.. _/rooms/<room_id>/join: /-rooms/join_room
+.. _/rooms/<room_id>/join: /docs/api/client-server/#!/-rooms/join_room
 
 .. |/rooms/<room_id>/leave| replace:: ``/rooms/<room_id>/leave``
-.. _/rooms/<room_id>/leave: /-rooms/leave
+.. _/rooms/<room_id>/leave: /docs/api/client-server/#!/-rooms/leave
 
 .. |/rooms/<room_id>/ban| replace:: ``/rooms/<room_id>/ban``
-.. _/rooms/<room_id>/ban: /-rooms/ban
+.. _/rooms/<room_id>/ban: /docs/api/client-server/#!/-rooms/ban
 
 .. |/join/<room_alias_or_id>| replace:: ``/join/<room_alias_or_id>``
-.. _/join/<room_alias_or_id>: /-rooms/join
+.. _/join/<room_alias_or_id>: /docs/api/client-server/#!/-rooms/join
 
-.. _`Event Stream`: /-events/get_event_stream
+.. _`Event Stream`: /docs/api/client-server/#!/-events/get_event_stream

experiments/cursesio.py

@@ -1,4 +1,4 @@
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

experiments/test_messaging.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

graph/graph.py

@@ -1,4 +1,4 @@
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

scripts/copyrighter.pl

@@ -1,5 +1,5 @@
 #!/usr/bin/perl -pi
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 $copyright = <<EOT;
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

scripts/gendoc.sh

@@ -9,4 +9,6 @@ perl -pi -e 's#<head>#<head><link rel="stylesheet" href="/site.css">#' $MATRIXDO
 
 perl -pi -e 's#<body>#<body><div id="header"><div id="headerContent">&nbsp;</div></div><div id="page"><div id="wrapper"><div style="text-align: center; padding: 40px;"><img src="/matrix.png" width="305" height="130" alt="[matrix]"/></div>#' $MATRIXDOTORG/docs/spec/index.html $MATRIXDOTORG/docs/howtos/client-server.html
 
-perl -pi -e 's#</body>#</div></div><div id="footer"><div id="footerContent">&copy 2014 Matrix.org</div></div></body>#' $MATRIXDOTORG/docs/spec/index.html $MATRIXDOTORG/docs/howtos/client-server.html
+perl -pi -e 's#</body>#</div></div><div id="footer"><div id="footerContent">&copy 2014 Matrix.org</div></div></body>#' $MATRIXDOTORG/docs/spec/index.html $MATRIXDOTORG/docs/howtos/client-server.html
+
+scp -r $MATRIXDOTORG/docs matrix@ldc-prd-matrix-001:/sites/matrix-beta

setup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/auth.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/constants.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/errors.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/events/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/events/factory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/events/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -103,8 +103,7 @@ class FeedbackEvent(SynapseEvent):
     def get_content_template(self):
         return {
             "type": u"string",
-            "target_event_id": u"string",
+            "target_event_id": u"string"
-            "msg_sender_id": u"string"
         }
 
 

synapse/api/ratelimiting.py

@@ -1,4 +1,4 @@
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/urls.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/app/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/app/homeserver.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,8 @@ from twisted.enterprise import adbapi
 from twisted.web.resource import Resource
 from twisted.web.static import File
 from twisted.web.server import Site
-from synapse.http.server import JsonResource, RootRedirect, ContentRepoResource
+from synapse.http.server import JsonResource, RootRedirect
+from synapse.http.content_repository import ContentRepoResource
 from synapse.http.client import TwistedHttpClient
 from synapse.api.urls import (
     CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX
@@ -74,7 +75,9 @@ class SynapseHomeServer(HomeServer):
         return File("webclient")  # TODO configurable?
 
     def build_resource_for_content_repo(self):
-        return ContentRepoResource(self, self.upload_dir, self.auth)
+        return ContentRepoResource(
+            self, self.upload_dir, self.auth, self.content_addr
+        )
 
     def build_db_pool(self):
         """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we
@@ -90,20 +93,28 @@ class SynapseHomeServer(HomeServer):
             if row and row[0]:
                 user_version = row[0]
 
-                if user_version < SCHEMA_VERSION:
+                if user_version > SCHEMA_VERSION:
-                    # TODO(paul): add some kind of intelligent fixup here
+                    raise ValueError("Cannot use this database as it is too " +
-                    raise ValueError("Cannot use this database as the " +
+                        "new for the server to understand"
-                        "schema version (%d) does not match (%d)" %
-                        (user_version, SCHEMA_VERSION)
                     )
+                elif user_version < SCHEMA_VERSION:
+                    logging.info("Upgrading database from version %d",
+                        user_version
+                    )
+
+                    # Run every version since after the current version.
+                    for v in range(user_version + 1, SCHEMA_VERSION + 1):
+                        sql_script = read_schema("delta/v%d" % (v))
+                        c.executescript(sql_script)
+
+                    db_conn.commit()
 
             else:
                 for sql_loc in SCHEMAS:
                     sql_script = read_schema(sql_loc)
 
                     c.executescript(sql_script)
-                    db_conn.commit()
+                db_conn.commit()
-
                 c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION)
 
             c.close()
@@ -248,6 +259,7 @@ def setup():
         db_name=config.database_path,
         tls_context_factory=tls_context_factory,
         config=config,
+        content_addr=config.content_addr,
     )
 
     hs.register_servlets()

synapse/config/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/database.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/homeserver.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,9 +18,10 @@ from .server import ServerConfig
 from .logger import LoggingConfig
 from .database import DatabaseConfig
 from .ratelimiting import RatelimitConfig
+from .repository import ContentRepositoryConfig
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
-                       RatelimitConfig):
+                       RatelimitConfig, ContentRepositoryConfig):
     pass
 
 if __name__=='__main__':

synapse/config/logger.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/ratelimiting.py

@@ -1,4 +1,4 @@
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/repository.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 matrix.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import Config
+import os
+
+class ContentRepositoryConfig(Config):
+    def __init__(self, args):
+        super(ContentRepositoryConfig, self).__init__(args)
+        self.max_upload_size = self.parse_size(args.max_upload_size)
+
+    def parse_size(self, string):
+        sizes = {"K": 1024, "M": 1024 * 1024}
+        size = 1
+        suffix = string[-1]
+        if suffix in sizes:
+            string = string[:-1]
+            size = sizes[suffix]
+        return int(string) * size
+
+    @classmethod
+    def add_arguments(cls, parser):
+        super(ContentRepositoryConfig, cls).add_arguments(parser)
+        db_group = parser.add_argument_group("content_repository")
+        db_group.add_argument(
+            "--max-upload-size", default="1M"
+        )

synapse/config/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,6 +32,14 @@ class ServerConfig(Config):
         self.webclient = True
         self.manhole = args.manhole
 
+        if not args.content_addr:
+            host = args.server_name
+            if ':' not in host:
+                host  = "%s:%d" % (host, args.bind_port)
+            args.content_addr = "https://%s" % (host,)
+
+        self.content_addr = args.content_addr
+
     @classmethod
     def add_arguments(cls, parser):
         super(ServerConfig, cls).add_arguments(parser)
@@ -50,13 +58,16 @@ class ServerConfig(Config):
                                   help="Local interface to listen on")
         server_group.add_argument("-D", "--daemonize", action='store_true',
                                   help="Daemonize the home server")
-        server_group.add_argument('--pid-file', default="hs.pid",
+        server_group.add_argument('--pid-file', default="homeserver.pid",
                                   help="When running as a daemon, the file to"
                                   " store the pid in")
         server_group.add_argument("--manhole", metavar="PORT", dest="manhole",
                                   type=int,
                                   help="Turn on the twisted telnet manhole"
                                   " service on the given port.")
+        server_group.add_argument("--content-addr", default=None,
+                                  help="The host and scheme to use for the "
+                                  "content repository")
 
     def read_signing_key(self, signing_key_path):
         signing_key_base64 = self.read_file(signing_key_path, "signing_key")
@@ -77,3 +88,4 @@ class ServerConfig(Config):
             with open(args.signing_key_path, "w") as signing_key_file:
                 key = nacl.signing.SigningKey.generate()
                 signing_key_file.write(encode_base64(key.encode()))
+

synapse/config/tls.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/context_factory.py

@@ -1,4 +1,4 @@
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/keyclient.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/keyserver.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/resource/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/resource/key.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/pdu_codec.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/persistence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/replication.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -628,7 +628,6 @@ class _TransactionQueue(object):
 
             for deferred in deferreds:
                 deferred.errback(e)
-                yield deferred
 
         finally:
             # We want to be *very* sure we delete this after we stop processing

synapse/federation/transport.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/units.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/directory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ from twisted.internet import defer
 from ._base import BaseHandler
 
 from synapse.api.errors import SynapseError
+from synapse.http.client import HttpClient
 
 import logging
 
@@ -36,7 +37,7 @@ class DirectoryHandler(BaseHandler):
         )
 
     @defer.inlineCallbacks
-    def create_association(self, room_alias, room_id, servers):
+    def create_association(self, room_alias, room_id, servers=None):
         # TODO(erikj): Do auth.
 
         if not room_alias.is_mine:
@@ -47,6 +48,12 @@ class DirectoryHandler(BaseHandler):
 
         # TODO(erikj): Check if there is a current association.
 
+        if not servers:
+            servers = yield self.store.get_joined_hosts_for_room(room_id)
+
+        if not servers:
+            raise SynapseError(400, "Failed to get server list")
+
         yield self.store.create_room_alias_association(
             room_alias,
             room_id,
@@ -68,7 +75,10 @@ class DirectoryHandler(BaseHandler):
             result = yield self.federation.make_query(
                 destination=room_alias.domain,
                 query_type="directory",
-                args={"room_alias": room_alias.to_string()},
+                args={
+                    "room_alias": room_alias.to_string(),
+                    HttpClient.RETRY_DNS_LOOKUP_FAILURES: False
+                }
             )
 
             if result and "room_id" in result and "servers" in result:
@@ -79,6 +89,9 @@ class DirectoryHandler(BaseHandler):
             defer.returnValue({})
             return
 
+        extra_servers = yield self.store.get_joined_hosts_for_room(room_id)
+        servers = list(set(extra_servers) | set(servers))
+
         defer.returnValue({
             "room_id": room_id,
             "servers": servers,

synapse/handlers/events.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -126,5 +126,7 @@ class EventHandler(BaseHandler):
             defer.returnValue(None)
             return
 
-        yield self.auth.check(event, raises=True)
+        if hasattr(event, "room_id"):
+            yield self.auth.check_joined_room(event.room_id, user.to_string())
+
         defer.returnValue(event)

synapse/handlers/federation.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/login.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/message.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -142,7 +142,12 @@ class MessageHandler(BaseRoomHandler):
             SynapseError if something went wrong.
         """
 
-        snapshot = yield self.store.snapshot_room(event.room_id, event.user_id)
+        snapshot = yield self.store.snapshot_room(
+            event.room_id,
+            event.user_id,
+            state_type=event.type,
+            state_key=event.state_key,
+        )
 
         yield self.auth.check(event, snapshot, raises=True)
 

synapse/handlers/presence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -155,19 +155,18 @@ class PresenceHandler(BaseHandler):
         if observer_user == observed_user:
             defer.returnValue(True)
 
-        allowed_by_subscription = yield self.store.is_presence_visible(
+        if (yield self.store.user_rooms_intersect(
-            observed_localpart=observed_user.localpart,
+            [u.to_string() for u in observer_user, observed_user]
-            observer_userid=observer_user.to_string(),
+        )):
-        )
-
-        if allowed_by_subscription:
             defer.returnValue(True)
 
-        share_room = yield self.store.do_users_share_a_room(
+        if (yield self.store.is_presence_visible(
-            [observer_user, observed_user]
+            observed_localpart=observed_user.localpart,
-        )
+            observer_userid=observer_user.to_string(),
+        )):
+            defer.returnValue(True)
 
-        defer.returnValue(share_room)
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def get_state(self, target_user, auth_user):
@@ -181,7 +180,7 @@ class PresenceHandler(BaseHandler):
             state = yield self.store.get_presence_state(target_user.localpart)
             if "mtime" in state:
                 del state["mtime"]
-            state["presence"] = state["state"]
+            state["presence"] = state.pop("state")
 
             if target_user in self._user_cachemap:
                 state["last_active"] = (
@@ -208,21 +207,17 @@ class PresenceHandler(BaseHandler):
             raise SynapseError(400, "User is not hosted on this Home Server")
 
         if target_user != auth_user:
-            raise AuthError(400, "Cannot set another user's displayname")
+            raise AuthError(400, "Cannot set another user's presence")
 
         if "status_msg" not in state:
             state["status_msg"] = None
 
         for k in state.keys():
-            if k not in ("presence", "state", "status_msg"):
+            if k not in ("presence", "status_msg"):
                 raise SynapseError(
                     400, "Unexpected presence state key '%s'" % (k,)
                 )
 
-        # Handle legacy "state" key for now
-        if "state" in state:
-            state["presence"] = state.pop("state")
-
         if state["presence"] not in self.STATE_LEVELS:
             raise SynapseError(400, "'%s' is not a valid presence state" %
                 state["presence"]
@@ -601,7 +596,7 @@ class PresenceHandler(BaseHandler):
         if state is None:
             state = yield self.store.get_presence_state(user.localpart)
             del state["mtime"]
-            state["presence"] = state["state"]
+            state["presence"] = state.pop("state")
 
             if user in self._user_cachemap:
                 state["last_active"] = (
@@ -622,8 +617,6 @@ class PresenceHandler(BaseHandler):
             "user_id": user.to_string(),
         }
         user_state.update(**state)
-        if "state" in user_state and "presence" not in user_state:
-            user_state["presence"] = user_state["state"]
 
         yield self.federation.send_edu(
             destination=destination,
@@ -655,21 +648,12 @@ class PresenceHandler(BaseHandler):
             state = dict(push)
             del state["user_id"]
 
-            if "presence" in state:
+            if "presence" not in state:
-                # all is OK
-                pass
-            elif "state" in state:
-                # Legacy handling
-                state["presence"] = state["state"]
-            else:
                 logger.warning("Received a presence 'push' EDU from %s without"
-                    + " either a 'presence' or 'state' key", origin
+                    + " a 'presence' key", origin
                 )
                 continue
 
-            if "state" in state:
-                del state["state"]
-
             if "last_active_ago" in state:
                 state["last_active"] = int(
                     self.clock.time_msec() - state.pop("last_active_ago")
@@ -773,15 +757,52 @@ class PresenceEventSource(object):
         self.hs = hs
         self.clock = hs.get_clock()
 
+    @defer.inlineCallbacks
+    def is_visible(self, observer_user, observed_user):
+        if observer_user == observed_user:
+            defer.returnValue(True)
+
+        presence = self.hs.get_handlers().presence_handler
+
+        if (yield presence.store.user_rooms_intersect(
+            [u.to_string() for u in observer_user, observed_user]
+        )):
+            defer.returnValue(True)
+
+        if observed_user.is_mine:
+            pushmap = presence._local_pushmap
+
+            defer.returnValue(
+                observed_user.localpart in pushmap and
+                observer_user in pushmap[observed_user.localpart]
+            )
+        else:
+            recvmap = presence._remote_recvmap
+
+            defer.returnValue(
+                observed_user in recvmap and
+                observer_user in recvmap[observed_user]
+            )
+
+    @defer.inlineCallbacks
     def get_new_events_for_user(self, user, from_key, limit):
         from_key = int(from_key)
 
+        observer_user = user
+
         presence = self.hs.get_handlers().presence_handler
         cachemap = presence._user_cachemap
 
-        # TODO(paul): limit, and filter by visibility
+        updates = []
-        updates = [(k, cachemap[k]) for k in cachemap
+        # TODO(paul): use a DeferredList ? How to limit concurrency.
-                   if from_key < cachemap[k].serial]
+        for observed_user in cachemap.keys():
+            if not (from_key < cachemap[observed_user].serial):
+                continue
+
+            if (yield self.is_visible(observer_user, observed_user)):
+                updates.append((observed_user, cachemap[observed_user]))
+
+        # TODO(paul): limit
 
         if updates:
             clock = self.clock
@@ -789,20 +810,23 @@ class PresenceEventSource(object):
             latest_serial = max([x[1].serial for x in updates])
             data = [x[1].make_event(user=x[0], clock=clock) for x in updates]
 
-            return ((data, latest_serial))
+            defer.returnValue((data, latest_serial))
         else:
-            return (([], presence._user_cachemap_latest_serial))
+            defer.returnValue(([], presence._user_cachemap_latest_serial))
 
     def get_current_key(self):
         presence = self.hs.get_handlers().presence_handler
         return presence._user_cachemap_latest_serial
 
+    @defer.inlineCallbacks
     def get_pagination_rows(self, user, pagination_config, key):
         # TODO (erikj): Does this make sense? Ordering?
 
         from_token = pagination_config.from_token
         to_token = pagination_config.to_token
 
+        observer_user = user
+
         from_key = int(from_token.presence_key)
 
         if to_token:
@@ -813,7 +837,17 @@ class PresenceEventSource(object):
         presence = self.hs.get_handlers().presence_handler
         cachemap = presence._user_cachemap
 
-        # TODO(paul): limit, and filter by visibility
+        updates = []
+        # TODO(paul): use a DeferredList ? How to limit concurrency.
+        for observed_user in cachemap.keys():
+            if not (to_key < cachemap[observed_user].serial < from_key):
+                continue
+
+            if (yield self.is_visible(observer_user, observed_user)):
+                updates.append((observed_user, cachemap[observed_user]))
+
+        # TODO(paul): limit
+
         updates = [(k, cachemap[k]) for k in cachemap
                    if to_key < cachemap[k].serial < from_key]
 
@@ -831,13 +865,13 @@ class PresenceEventSource(object):
             next_token = next_token.copy_and_replace(
                 "presence_key", earliest_serial
             )
-            return ((data, next_token))
+            defer.returnValue((data, next_token))
         else:
             if not to_token:
                 to_token = from_token.copy_and_replace(
                     "presence_key", 0
                 )
-            return (([], to_token))
+            defer.returnValue(([], to_token))
 
 
 class UserPresenceCache(object):
@@ -851,7 +885,6 @@ class UserPresenceCache(object):
 
     def update(self, state, serial):
         assert("mtime_age" not in state)
-        assert("state" not in state)
 
         self.state.update(state)
         # Delete keys that are now 'None'
@@ -869,11 +902,6 @@ class UserPresenceCache(object):
     def get_state(self):
         # clone it so caller can't break our cache
         state = dict(self.state)
-
-        # Legacy handling
-        if "presence" in state:
-            state["state"] = state["presence"]
-
         return state
 
     def make_event(self, user, clock):

synapse/handlers/profile.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/register.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/typing.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/http/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/http/client.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 
 
 from twisted.internet import defer, reactor
+from twisted.internet.error import DNSLookupError
 from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer
 from twisted.web.http_headers import Headers
 
@@ -23,7 +24,7 @@ from synapse.util.async import sleep
 
 from syutil.jsonutil import encode_canonical_json
 
-from synapse.api.errors import CodeMessageException
+from synapse.api.errors import CodeMessageException, SynapseError
 
 from StringIO import StringIO
 
@@ -45,6 +46,7 @@ _destination_mappings = {
 class HttpClient(object):
     """ Interface for talking json over http
     """
+    RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
 
     def put_json(self, destination, path, data):
         """ Sends the specifed json data using PUT
@@ -144,13 +146,23 @@ class TwistedHttpClient(HttpClient):
             destination = _destination_mappings[destination]
 
         logger.debug("get_json args: %s", args)
+
+        retry_on_dns_fail = True
+        if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args:
+            # FIXME: This isn't ideal, but the interface exposed in get_json
+            # isn't comprehensive enough to give caller's any control over
+            # their connection mechanics.
+            retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES)
+
         query_bytes = urllib.urlencode(args, True)
+        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
 
         response = yield self._create_request(
             destination.encode("ascii"),
             "GET",
             path.encode("ascii"),
-            query_bytes=query_bytes
+            query_bytes=query_bytes,
+            retry_on_dns_fail=retry_on_dns_fail
         )
 
         body = yield readBody(response)
@@ -179,7 +191,8 @@ class TwistedHttpClient(HttpClient):
 
     @defer.inlineCallbacks
     def _create_request(self, destination, method, path_bytes, param_bytes=b"",
-                        query_bytes=b"", producer=None, headers_dict={}):
+                        query_bytes=b"", producer=None, headers_dict={},
+                        retry_on_dns_fail=True):
         """ Creates and sends a request to the given url
         """
         headers_dict[b"User-Agent"] = [b"Synapse"]
@@ -218,6 +231,11 @@ class TwistedHttpClient(HttpClient):
                 logger.debug("Got response to %s", method)
                 break
             except Exception as e:
+                if not retry_on_dns_fail and isinstance(e, DNSLookupError):
+                    logger.warn("DNS Lookup failed to %s with %s", destination,
+                                e)
+                    raise SynapseError(400, "Domain specified not found.")
+
                 logger.exception("Got error in _create_request")
                 _print_ex(e)
 

synapse/http/content_repository.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .server import respond_with_json_bytes
+
+from synapse.util.stringutils import random_string
+from synapse.api.errors import (
+    cs_exception, SynapseError, CodeMessageException, Codes, cs_error
+)
+
+from twisted.protocols.basic import FileSender
+from twisted.web import server, resource
+from twisted.internet import defer
+
+import base64
+import json
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+class ContentRepoResource(resource.Resource):
+    """Provides file uploading and downloading.
+
+    Uploads are POSTed to wherever this Resource is linked to. This resource
+    returns a "content token" which can be used to GET this content again. The
+    token is typically a path, but it may not be. Tokens can expire, be one-time
+    uses, etc.
+
+    In this case, the token is a path to the file and contains 3 interesting
+    sections:
+        - User ID base64d (for namespacing content to each user)
+        - random 24 char string
+        - Content type base64d (so we can return it when clients GET it)
+
+    """
+    isLeaf = True
+
+    def __init__(self, hs, directory, auth, external_addr):
+        resource.Resource.__init__(self)
+        self.hs = hs
+        self.directory = directory
+        self.auth = auth
+        self.external_addr = external_addr.rstrip('/')
+        self.max_upload_size = hs.config.max_upload_size
+
+        if not os.path.isdir(self.directory):
+            os.mkdir(self.directory)
+            logger.info("ContentRepoResource : Created %s directory.",
+                        self.directory)
+
+    @defer.inlineCallbacks
+    def map_request_to_name(self, request):
+        # auth the user
+        auth_user = yield self.auth.get_user_by_req(request)
+
+        # namespace all file uploads on the user
+        prefix = base64.urlsafe_b64encode(
+            auth_user.to_string()
+        ).replace('=', '')
+
+        # use a random string for the main portion
+        main_part = random_string(24)
+
+        # suffix with a file extension if we can make one. This is nice to
+        # provide a hint to clients on the file information. We will also reuse
+        # this info to spit back the content type to the client.
+        suffix = ""
+        if request.requestHeaders.hasHeader("Content-Type"):
+            content_type = request.requestHeaders.getRawHeaders(
+                "Content-Type")[0]
+            suffix = "." + base64.urlsafe_b64encode(content_type)
+            if (content_type.split("/")[0].lower() in
+                    ["image", "video", "audio"]):
+                file_ext = content_type.split("/")[-1]
+                # be a little paranoid and only allow a-z
+                file_ext = re.sub("[^a-z]", "", file_ext)
+                suffix += "." + file_ext
+
+        file_name = prefix + main_part + suffix
+        file_path = os.path.join(self.directory, file_name)
+        logger.info("User %s is uploading a file to path %s",
+                    auth_user.to_string(),
+                    file_path)
+
+        # keep trying to make a non-clashing file, with a sensible max attempts
+        attempts = 0
+        while os.path.exists(file_path):
+            main_part = random_string(24)
+            file_name = prefix + main_part + suffix
+            file_path = os.path.join(self.directory, file_name)
+            attempts += 1
+            if attempts > 25:  # really? Really?
+                raise SynapseError(500, "Unable to create file.")
+
+        defer.returnValue(file_path)
+
+    def render_GET(self, request):
+        # no auth here on purpose, to allow anyone to view, even across home
+        # servers.
+
+        # TODO: A little crude here, we could do this better.
+        filename = request.path.split('/')[-1]
+        # be paranoid
+        filename = re.sub("[^0-9A-z.-_]", "", filename)
+
+        file_path = self.directory + "/" + filename
+
+        logger.debug("Searching for %s", file_path)
+
+        if os.path.isfile(file_path):
+            # filename has the content type
+            base64_contentype = filename.split(".")[1]
+            content_type = base64.urlsafe_b64decode(base64_contentype)
+            logger.info("Sending file %s", file_path)
+            f = open(file_path, 'rb')
+            request.setHeader('Content-Type', content_type)
+            d = FileSender().beginFileTransfer(f, request)
+
+            # after the file has been sent, clean up and finish the request
+            def cbFinished(ignored):
+                f.close()
+                request.finish()
+            d.addCallback(cbFinished)
+        else:
+            respond_with_json_bytes(
+                request,
+                404,
+                json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)),
+                send_cors=True)
+
+        return server.NOT_DONE_YET
+
+    def render_POST(self, request):
+        self._async_render(request)
+        return server.NOT_DONE_YET
+
+    def render_OPTIONS(self, request):
+        respond_with_json_bytes(request, 200, {}, send_cors=True)
+        return server.NOT_DONE_YET
+
+    @defer.inlineCallbacks
+    def _async_render(self, request):
+        try:
+            # TODO: The checks here are a bit late. The content will have
+            # already been uploaded to a tmp file at this point
+            content_length = request.getHeader("Content-Length")
+            if content_length is None:
+                raise SynapseError(
+                    msg="Request must specify a Content-Length", code=400
+                )
+            if int(content_length) > self.max_upload_size:
+                raise SynapseError(
+                    msg="Upload request body is too large",
+                    code=413,
+                )
+
+            fname = yield self.map_request_to_name(request)
+
+            # TODO I have a suspcious feeling this is just going to block
+            with open(fname, "wb") as f:
+                f.write(request.content.read())
+
+
+            # FIXME (erikj): These should use constants.
+            file_name = os.path.basename(fname)
+            # FIXME: we can't assume what the public mounted path of the repo is
+            # ...plus self-signed SSL won't work to remote clients anyway
+            # ...and we can't assume that it's SSL anyway, as we might want to
+            # server it via the non-SSL listener...
+            url = "%s/_matrix/content/%s" % (
+                self.external_addr, file_name
+            )
+
+            respond_with_json_bytes(request, 200,
+                                    json.dumps({"content_token": url}),
+                                    send_cors=True)
+
+        except CodeMessageException as e:
+            logger.exception(e)
+            respond_with_json_bytes(request, e.code,
+                                    json.dumps(cs_exception(e)))
+        except Exception as e:
+            logger.error("Failed to store file: %s" % e)
+            respond_with_json_bytes(
+                request,
+                500,
+                json.dumps({"error": "Internal server error"}),
+                send_cors=True)
+
+
+

synapse/http/endpoint.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/http/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,22 +18,16 @@ from syutil.jsonutil import (
     encode_canonical_json, encode_pretty_printed_json
 )
 from synapse.api.errors import (
-    cs_exception, SynapseError, CodeMessageException, Codes, cs_error
+    cs_exception, SynapseError, CodeMessageException
 )
-from synapse.util.stringutils import random_string
 
 from twisted.internet import defer, reactor
-from twisted.protocols.basic import FileSender
 from twisted.web import server, resource
 from twisted.web.server import NOT_DONE_YET
 from twisted.web.util import redirectTo
 
-import base64
 import collections
-import json
 import logging
-import os
-import re
 
 logger = logging.getLogger(__name__)
 
@@ -198,161 +192,6 @@ class RootRedirect(resource.Resource):
         return resource.Resource.getChild(self, name, request)
 
 
-class ContentRepoResource(resource.Resource):
-    """Provides file uploading and downloading.
-
-    Uploads are POSTed to wherever this Resource is linked to. This resource
-    returns a "content token" which can be used to GET this content again. The
-    token is typically a path, but it may not be. Tokens can expire, be one-time
-    uses, etc.
-
-    In this case, the token is a path to the file and contains 3 interesting
-    sections:
-        - User ID base64d (for namespacing content to each user)
-        - random 24 char string
-        - Content type base64d (so we can return it when clients GET it)
-
-    """
-    isLeaf = True
-
-    def __init__(self, hs, directory, auth):
-        resource.Resource.__init__(self)
-        self.hs = hs
-        self.directory = directory
-        self.auth = auth
-
-        if not os.path.isdir(self.directory):
-            os.mkdir(self.directory)
-            logger.info("ContentRepoResource : Created %s directory.",
-                        self.directory)
-
-    @defer.inlineCallbacks
-    def map_request_to_name(self, request):
-        # auth the user
-        auth_user = yield self.auth.get_user_by_req(request)
-
-        # namespace all file uploads on the user
-        prefix = base64.urlsafe_b64encode(
-            auth_user.to_string()
-        ).replace('=', '')
-
-        # use a random string for the main portion
-        main_part = random_string(24)
-
-        # suffix with a file extension if we can make one. This is nice to
-        # provide a hint to clients on the file information. We will also reuse
-        # this info to spit back the content type to the client.
-        suffix = ""
-        if request.requestHeaders.hasHeader("Content-Type"):
-            content_type = request.requestHeaders.getRawHeaders(
-                "Content-Type")[0]
-            suffix = "." + base64.urlsafe_b64encode(content_type)
-            if (content_type.split("/")[0].lower() in
-                    ["image", "video", "audio"]):
-                file_ext = content_type.split("/")[-1]
-                # be a little paranoid and only allow a-z
-                file_ext = re.sub("[^a-z]", "", file_ext)
-                suffix += "." + file_ext
-
-        file_name = prefix + main_part + suffix
-        file_path = os.path.join(self.directory, file_name)
-        logger.info("User %s is uploading a file to path %s",
-                    auth_user.to_string(),
-                    file_path)
-
-        # keep trying to make a non-clashing file, with a sensible max attempts
-        attempts = 0
-        while os.path.exists(file_path):
-            main_part = random_string(24)
-            file_name = prefix + main_part + suffix
-            file_path = os.path.join(self.directory, file_name)
-            attempts += 1
-            if attempts > 25:  # really? Really?
-                raise SynapseError(500, "Unable to create file.")
-
-        defer.returnValue(file_path)
-
-    def render_GET(self, request):
-        # no auth here on purpose, to allow anyone to view, even across home
-        # servers.
-
-        # TODO: A little crude here, we could do this better.
-        filename = request.path.split('/')[-1]
-        # be paranoid
-        filename = re.sub("[^0-9A-z.-_]", "", filename)
-
-        file_path = self.directory + "/" + filename
-
-        logger.debug("Searching for %s", file_path)
-
-        if os.path.isfile(file_path):
-            # filename has the content type
-            base64_contentype = filename.split(".")[1]
-            content_type = base64.urlsafe_b64decode(base64_contentype)
-            logger.info("Sending file %s", file_path)
-            f = open(file_path, 'rb')
-            request.setHeader('Content-Type', content_type)
-            d = FileSender().beginFileTransfer(f, request)
-
-            # after the file has been sent, clean up and finish the request
-            def cbFinished(ignored):
-                f.close()
-                request.finish()
-            d.addCallback(cbFinished)
-        else:
-            respond_with_json_bytes(
-                request,
-                404,
-                json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)),
-                send_cors=True)
-
-        return server.NOT_DONE_YET
-
-    def render_POST(self, request):
-        self._async_render(request)
-        return server.NOT_DONE_YET
-
-    def render_OPTIONS(self, request):
-        respond_with_json_bytes(request, 200, {}, send_cors=True)
-        return server.NOT_DONE_YET
-
-    @defer.inlineCallbacks
-    def _async_render(self, request):
-        try:
-            fname = yield self.map_request_to_name(request)
-
-            # TODO I have a suspcious feeling this is just going to block
-            with open(fname, "wb") as f:
-                f.write(request.content.read())
-
-
-            # FIXME (erikj): These should use constants.
-            file_name = os.path.basename(fname)
-            # FIXME: we can't assume what the public mounted path of the repo is
-            # ...plus self-signed SSL won't work to remote clients anyway
-            # ...and we can't assume that it's SSL anyway, as we might want to
-            # server it via the non-SSL listener...
-            url = "https://%s/_matrix/content/%s" % (
-                self.hs.domain_with_port, file_name
-            )
-
-            respond_with_json_bytes(request, 200,
-                                    json.dumps({"content_token": url}),
-                                    send_cors=True)
-
-        except CodeMessageException as e:
-            logger.exception(e)
-            respond_with_json_bytes(request, e.code,
-                                    json.dumps(cs_exception(e)))
-        except Exception as e:
-            logger.error("Failed to store file: %s" % e)
-            respond_with_json_bytes(
-                request,
-                500,
-                json.dumps({"error": "Internal server error"}),
-                send_cors=True)
-
-
 def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
                             response_code_message=None):
     """Sends encoded JSON in response to the given request.

synapse/notifier.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -167,7 +167,12 @@ class Notifier(object):
                 )
 
         def eb(failure):
-            logger.exception("Failed to notify listener", failure)
+            logger.error("Failed to notify listener",
+                exc_info=(
+                    failure.type,
+                    failure.value,
+                    failure.getTracebackObject())
+            )
 
         yield defer.DeferredList(
             [notify(l).addErrback(eb) for l in listeners]

synapse/rest/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/directory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 
 from twisted.internet import defer
 
+from synapse.api.errors import SynapseError, Codes
 from base import RestServlet, client_path_pattern
 
 import json
@@ -44,8 +45,10 @@ class ClientDirectoryServer(RestServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, request, room_alias):
-        # TODO(erikj): Exceptions
+        content = _parse_json(request)
-        content = json.loads(request.content.read())
+        if not "room_id" in content:
+            raise SynapseError(400, "Missing room_id key",
+                               errcode=Codes.BAD_JSON)
 
         logger.debug("Got content: %s", content)
 
@@ -54,7 +57,7 @@ class ClientDirectoryServer(RestServlet):
         logger.debug("Got room name: %s", room_alias.to_string())
 
         room_id = content["room_id"]
-        servers = content["servers"]
+        servers = content["servers"] if "servers" in content else None
 
         logger.debug("Got room_id: %s", room_id)
         logger.debug("Got servers: %s", servers)
@@ -68,7 +71,20 @@ class ClientDirectoryServer(RestServlet):
             yield dir_handler.create_association(
                 room_alias, room_id, servers
             )
+        except SynapseError as e:
+            raise e
         except:
             logger.exception("Failed to create association")
 
         defer.returnValue((200, {}))
+
+
+def _parse_json(request):
+    try:
+        content = json.loads(request.content.read())
+        if type(content) != dict:
+            raise SynapseError(400, "Content must be a JSON object.",
+                               errcode=Codes.NOT_JSON)
+        return content
+    except ValueError:
+        raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON)

synapse/rest/events.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/initial_sync.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/login.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/presence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,11 +17,12 @@
 """
 from twisted.internet import defer
 
+from synapse.api.errors import SynapseError
 from base import RestServlet, client_path_pattern
 
 import json
 import logging
-
+import urllib
 
 logger = logging.getLogger(__name__)
 
@@ -32,6 +33,7 @@ class PresenceStatusRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
         auth_user = yield self.auth.get_user_by_req(request)
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         state = yield self.handlers.presence_handler.get_state(
@@ -42,25 +44,26 @@ class PresenceStatusRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
         auth_user = yield self.auth.get_user_by_req(request)
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         state = {}
         try:
             content = json.loads(request.content.read())
 
-            # Legacy handling
+            state["presence"] = content.pop("presence")
-            if "state" in content:
-                state["presence"] = content.pop("state")
-            else:
-                state["presence"] = content.pop("presence")
 
             if "status_msg" in content:
                 state["status_msg"] = content.pop("status_msg")
+                if not isinstance(state["status_msg"], basestring):
+                    raise SynapseError(400, "status_msg must be a string.")
 
             if content:
                 raise KeyError()
+        except SynapseError as e:
+            raise e
         except:
-            defer.returnValue((400, "Unable to parse state"))
+            raise SynapseError(400, "Unable to parse state")
 
         yield self.handlers.presence_handler.set_state(
             target_user=user, auth_user=auth_user, state=state)
@@ -77,13 +80,14 @@ class PresenceListRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
         auth_user = yield self.auth.get_user_by_req(request)
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         if not user.is_mine:
-            defer.returnValue((400, "User not hosted on this Home Server"))
+            raise SynapseError(400, "User not hosted on this Home Server")
 
         if auth_user != user:
-            defer.returnValue((400, "Cannot get another user's presence list"))
+            raise SynapseError(400, "Cannot get another user's presence list")
 
         presence = yield self.handlers.presence_handler.get_presence_list(
             observer_user=user, accepted=True)
@@ -97,31 +101,40 @@ class PresenceListRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_POST(self, request, user_id):
         auth_user = yield self.auth.get_user_by_req(request)
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         if not user.is_mine:
-            defer.returnValue((400, "User not hosted on this Home Server"))
+            raise SynapseError(400, "User not hosted on this Home Server")
 
         if auth_user != user:
-            defer.returnValue((
+            raise SynapseError(
-                400, "Cannot modify another user's presence list"))
+                400, "Cannot modify another user's presence list")
 
         try:
             content = json.loads(request.content.read())
         except:
             logger.exception("JSON parse error")
-            defer.returnValue((400, "Unable to parse content"))
+            raise SynapseError(400, "Unable to parse content")
 
         deferreds = []
 
         if "invite" in content:
             for u in content["invite"]:
+                if not isinstance(u, basestring):
+                    raise SynapseError(400, "Bad invite value.")
+                if len(u) == 0:
+                    continue
                 invited_user = self.hs.parse_userid(u)
                 deferreds.append(self.handlers.presence_handler.send_invite(
                     observer_user=user, observed_user=invited_user))
 
         if "drop" in content:
             for u in content["drop"]:
+                if not isinstance(u, basestring):
+                    raise SynapseError(400, "Bad drop value.")
+                if len(u) == 0:
+                    continue
                 dropped_user = self.hs.parse_userid(u)
                 deferreds.append(self.handlers.presence_handler.drop(
                     observer_user=user, observed_user=dropped_user))

synapse/rest/profile.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ from twisted.internet import defer
 from base import RestServlet, client_path_pattern
 
 import json
+import urllib
 
 
 class ProfileDisplaynameRestServlet(RestServlet):
@@ -26,6 +27,7 @@ class ProfileDisplaynameRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         displayname = yield self.handlers.profile_handler.get_displayname(
@@ -37,6 +39,7 @@ class ProfileDisplaynameRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
         auth_user = yield self.auth.get_user_by_req(request)
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         try:
@@ -59,6 +62,7 @@ class ProfileAvatarURLRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         avatar_url = yield self.handlers.profile_handler.get_avatar_url(
@@ -70,6 +74,7 @@ class ProfileAvatarURLRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
         auth_user = yield self.auth.get_user_by_req(request)
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         try:
@@ -92,6 +97,7 @@ class ProfileRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
+        user_id = urllib.unquote(user_id)
         user = self.hs.parse_userid(user_id)
 
         displayname = yield self.handlers.profile_handler.get_displayname(

synapse/rest/register.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -388,7 +388,7 @@ class RoomMembershipRestServlet(RestServlet):
     def register(self, http_server):
         # /rooms/$roomid/[invite|join|leave]
         PATTERN = ("/rooms/(?P<room_id>[^/]*)/" +
-            "(?P<membership_action>join|invite|leave|ban)")
+            "(?P<membership_action>join|invite|leave|ban|kick)")
         register_txn_path(self, PATTERN, http_server)
 
     @defer.inlineCallbacks
@@ -399,11 +399,14 @@ class RoomMembershipRestServlet(RestServlet):
 
         # target user is you unless it is an invite
         state_key = user.to_string()
-        if membership_action in ["invite", "ban"]:
+        if membership_action in ["invite", "ban", "kick"]:
             if "user_id" not in content:
                 raise SynapseError(400, "Missing user_id key.")
             state_key = content["user_id"]
 
+            if membership_action == "kick":
+                membership_action = "leave"
+
         event = self.event_factory.create_event(
             etype=RoomMemberEvent.TYPE,
             content={"membership": unicode(membership_action)},

synapse/rest/transactions.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/state.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -179,6 +179,18 @@ class StateHandler(object):
                 key=lambda x: x.depth
             )
 
+            if not hasattr(missing_prev, "prev_state_id"):
+                # FIXME Hmm
+                # temporary fallback
+                for algo in conflict_res:
+                    new_res, curr_res = algo(new_branch, current_branch)
+
+                    if new_res < curr_res:
+                        defer.returnValue(False)
+                    elif new_res > curr_res:
+                        defer.returnValue(True)
+                return
+
             pdu_id = missing_prev.prev_state_id
             origin = missing_prev.prev_state_origin
 

synapse/storage/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -79,19 +79,21 @@ class SQLBaseStore(object):
     # "Simple" SQL API methods that operate on a single table with no JOINs,
     # no complex WHERE clauses, just a dict of values for columns.
 
-    def _simple_insert(self, table, values):
+    def _simple_insert(self, table, values, or_replace=False):
         """Executes an INSERT query on the named table.
 
         Args:
             table : string giving the table name
             values : dict of new column names and values for them
+            or_replace : bool; if True performs an INSERT OR REPLACE
         """
         return self._db_pool.runInteraction(
-            self._simple_insert_txn, table, values,
+            self._simple_insert_txn, table, values, or_replace=or_replace
         )
 
-    def _simple_insert_txn(self, txn, table, values):
+    def _simple_insert_txn(self, txn, table, values, or_replace=False):
-        sql = "INSERT INTO %s (%s) VALUES(%s)" % (
+        sql = "%s INTO %s (%s) VALUES(%s)" % (
+            ("INSERT OR REPLACE" if or_replace else "INSERT"),
             table,
             ", ".join(k for k in values),
             ", ".join("?" for k in values)

synapse/storage/directory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/feedback.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/keys.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/pdu.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/presence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/profile.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/registration.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/roommember.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -165,7 +165,7 @@ class RoomMemberStore(SQLBaseStore):
         defer.returnValue(results)
 
     @defer.inlineCallbacks
-    def do_users_share_a_room(self, user_list):
+    def user_rooms_intersect(self, user_list):
         """ Checks whether a list of users share a room.
         """
         user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_list))

synapse/storage/schema/delta/v2.sql

@@ -0,0 +1,168 @@
+/* Copyright 2014 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS events(
+    stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
+    topological_ordering INTEGER NOT NULL,
+    event_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    content TEXT NOT NULL,
+    unrecognized_keys TEXT,
+    processed BOOL NOT NULL,
+    outlier BOOL NOT NULL,
+    CONSTRAINT ev_uniq UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id);
+CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
+CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
+CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
+
+CREATE TABLE IF NOT EXISTS state_events(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    prev_state TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id);
+CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
+CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
+CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
+
+
+CREATE TABLE IF NOT EXISTS current_state_events(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id);
+CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
+CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
+CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
+
+CREATE TABLE IF NOT EXISTS room_memberships(
+    event_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    sender TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    membership TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id);
+CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
+CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
+
+CREATE TABLE IF NOT EXISTS feedback(
+    event_id TEXT NOT NULL,
+    feedback_type TEXT,
+    target_event_id TEXT,
+    sender TEXT,
+    room_id TEXT
+);
+
+CREATE TABLE IF NOT EXISTS topics(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    topic TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS room_names(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    name TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS rooms(
+    room_id TEXT PRIMARY KEY NOT NULL,
+    is_public INTEGER,
+    creator TEXT
+);
+
+CREATE TABLE IF NOT EXISTS room_join_rules(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    join_rule TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS room_join_rules_event_id ON room_join_rules(event_id);
+CREATE INDEX IF NOT EXISTS room_join_rules_room_id ON room_join_rules(room_id);
+
+
+CREATE TABLE IF NOT EXISTS room_power_levels(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    level INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS room_power_levels_event_id ON room_power_levels(event_id);
+CREATE INDEX IF NOT EXISTS room_power_levels_room_id ON room_power_levels(room_id);
+CREATE INDEX IF NOT EXISTS room_power_levels_room_user ON room_power_levels(room_id, user_id);
+
+
+CREATE TABLE IF NOT EXISTS room_default_levels(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    level INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS room_default_levels_event_id ON room_default_levels(event_id);
+CREATE INDEX IF NOT EXISTS room_default_levels_room_id ON room_default_levels(room_id);
+
+
+CREATE TABLE IF NOT EXISTS room_add_state_levels(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    level INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS room_add_state_levels_event_id ON room_add_state_levels(event_id);
+CREATE INDEX IF NOT EXISTS room_add_state_levels_room_id ON room_add_state_levels(room_id);
+
+
+CREATE TABLE IF NOT EXISTS room_send_event_levels(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    level INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS room_send_event_levels_event_id ON room_send_event_levels(event_id);
+CREATE INDEX IF NOT EXISTS room_send_event_levels_room_id ON room_send_event_levels(room_id);
+
+
+CREATE TABLE IF NOT EXISTS room_ops_levels(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    ban_level INTEGER,
+    kick_level INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS room_ops_levels_event_id ON room_ops_levels(event_id);
+CREATE INDEX IF NOT EXISTS room_ops_levels_room_id ON room_ops_levels(room_id);
+
+
+CREATE TABLE IF NOT EXISTS room_hosts(
+    room_id TEXT NOT NULL,
+    host TEXT NOT NULL,
+    CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE
+);
+
+CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
+
+PRAGMA user_version = 2;

synapse/storage/schema/edge_pdus.sql

@@ -1,4 +1,4 @@
-/* Copyright 2014 matrix.org
+/* Copyright 2014 OpenMarket Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

synapse/storage/schema/im.sql

@@ -1,4 +1,4 @@
-/* Copyright 2014 matrix.org
+/* Copyright 2014 OpenMarket Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

synapse/storage/schema/keys.sql

@@ -1,4 +1,4 @@
-/* Copyright 2014 matrix.org
+/* Copyright 2014 OpenMarket Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

synapse/storage/schema/pdu.sql

@@ -1,4 +1,4 @@
-/* Copyright 2014 matrix.org
+/* Copyright 2014 OpenMarket Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.