diff --git a/.gitignore b/.gitignore index b91b52b61..339a99e0d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc .*.swp +.DS_Store _trial_temp/ logs/ dbs/ @@ -11,6 +12,14 @@ docs/build/ cmdclient_config.json homeserver*.db +homeserver*.log +homeserver*.pid +homeserver*.yaml + +*.signing.key +*.tls.crt +*.tls.dh +*.tls.key .coverage htmlcov @@ -24,7 +33,8 @@ graph/*.svg graph/*.png graph/*.dot -webclient/config.js -webclient/test/environment-protractor.js +**/webclient/config.js +**/webclient/test/coverage/ +**/webclient/test/environment-protractor.js uploads diff --git a/CHANGES.rst b/CHANGES.rst index 78c178baf..5a284c385 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,27 @@ +Changes in synapse 0.5.0 (2014-11-19) +===================================== +This release includes changes to the federation protocol and client-server API +that is not backwards compatible. + +This release also changes the internal database schemas and so requires servers to +drop their current history. See UPGRADES.rst for details. + +Homeserver: + * Add authentication and authorization to the federation protocol. Events are + now signed by their originating homeservers. + * Implement the new authorization model for rooms. + * Split out web client into a seperate repository: matrix-angular-sdk. + * Change the structure of PDUs. + * Fix bug where user could not join rooms via an alias containing 4-byte + UTF-8 characters. + * Merge concept of PDUs and Events internally. + * Improve logging by adding request ids to log lines. + * Implement a very basic room initial sync API. + * Implement the new invite/join federation APIs. + +Webclient: + * The webclient has been moved to a seperate repository. + Changes in synapse 0.4.2 (2014-10-31) ===================================== diff --git a/MANIFEST.in b/MANIFEST.in index 73e0eff6e..a1a77ff54 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ recursive-include docs * recursive-include tests *.py -recursive-include synapse/persistence/schema *.sql +recursive-include synapse/storage/schema *.sql +recursive-include syweb/webclient * diff --git a/README.rst b/README.rst index f40492b8a..542f19987 100644 --- a/README.rst +++ b/README.rst @@ -4,9 +4,9 @@ Introduction Matrix is an ambitious new ecosystem for open federated Instant Messaging and VoIP. The basics you need to know to get up and running are: -- Chatrooms are distributed and do not exist on any single server. Rooms - can be found using aliases like ``#matrix:matrix.org`` or - ``#test:localhost:8008`` or they can be ephemeral. +- Everything in Matrix happens in a room. Rooms are distributed and do not + exist on any single server. Rooms can be located using convenience aliases + like ``#matrix:matrix.org`` or ``#test:localhost:8008``. - Matrix user IDs look like ``@matthew:matrix.org`` (although in the future you will normally refer to yourself and others using a 3PID: email @@ -17,56 +17,12 @@ The overall architecture is:: client <----> homeserver <=====================> homeserver <----> client https://somewhere.org/_matrix https://elsewhere.net/_matrix -WARNING -======= - -**Synapse is currently in a state of rapid development, and not all features -are yet functional. Critically, some security features are still in -development, which means Synapse can *not* be considered secure or reliable at -this point.** For instance: - -- **SSL Certificates used by server-server federation are not yet validated.** -- **Room permissions are not yet enforced on traffic received via federation.** -- **Homeservers do not yet cryptographically sign their events to avoid - tampering** -- Default configuration provides open signup to the service from the internet - -Despite this, we believe Synapse is more than useful as a way for experimenting -and exploring Synapse, and the missing features will land shortly. **Until -then, please do *NOT* use Synapse for any remotely important or secure -communication.** - - -Quick Start -=========== - -System requirements: -- POSIX-compliant system (tested on Linux & OSX) -- Python 2.7 - -To get up and running: - -- To simply play with an **existing** homeserver you can - just go straight to http://matrix.org/alpha. - -- To run your own **private** homeserver on localhost:8008, generate a basic - config file: ``./synctl start`` will give you instructions on how to do this. - For this purpose, you can use 'localhost' or your hostname as a server name. - Once you've done so, running ``./synctl start`` again will start your private - home sserver. You will find a webclient running at http://localhost:8008. - Please use a recent Chrome or Firefox for now (or Safari if you don't need - VoIP support). - -- To run a **public** homeserver and let it exchange messages with other - homeservers and participate in the global Matrix federation, you must expose - port 8448 to the internet and edit homeserver.yaml to specify server_name - (the public DNS entry for this server) and then run ``synctl start``. If you - changed the server_name, you may need to move the old database - (homeserver.db) out of the way first. Then come join ``#matrix:matrix.org`` - and say hi! :) - -For more detailed setup instructions, please see further down this document. +``#matrix:matrix.org`` is the official support room for Matrix, and can be +accessed by the web client at http://matrix.org/alpha or via an IRC bridge at +irc://irc.freenode.net/matrix. +Synapse is currently in rapid development, but as of version 0.5 we believe it +is sufficiently stable to be run as an internet-facing service for real usage! About Matrix ============ @@ -76,10 +32,10 @@ which handle: - Creating and managing fully distributed chat rooms with no single points of control or failure -- Eventually-consistent cryptographically secure[1] synchronisation of room +- Eventually-consistent cryptographically secure synchronisation of room state across a global open network of federated servers and services - Sending and receiving extensible messages in a room with (optional) - end-to-end encryption[2] + end-to-end encryption[1] - Inviting, joining, leaving, kicking, banning room members - Managing user accounts (registration, login, logout) - Using 3rd Party IDs (3PIDs) such as email addresses, phone numbers, @@ -111,56 +67,121 @@ Synapse ships with two basic demo Matrix clients: webclient (a basic group chat web client demo implemented in AngularJS) and cmdclient (a basic Python command line utility which lets you easily see what the JSON APIs are up to). -We'd like to invite you to take a look at the Matrix spec, try to run a -homeserver, and join the existing Matrix chatrooms already out there, -experiment with the APIs and the demo clients, and let us know your thoughts at -https://github.com/matrix-org/synapse/issues or at matrix@matrix.org. +Meanwhile, iOS and Android SDKs and clients are currently in development and available from: -Thanks for trying Matrix! + * https://github.com/matrix-org/matrix-ios-sdk + * https://github.com/matrix-org/matrix-android-sdk -[1] Cryptographic signing of messages isn't turned on yet +We'd like to invite you to join #matrix:matrix.org (via http://matrix.org/alpha), run a homeserver, take a look at the Matrix spec at +http://matrix.org/docs/spec, experiment with the APIs and the demo +clients, and report any bugs via http://matrix.org/jira. -[2] End-to-end encryption is currently in development +Thanks for using Matrix! +[1] End-to-end encryption is currently in development Homeserver Installation ======================= -First, the dependencies need to be installed. Start by installing -'python2.7-dev' and the various tools of the compiler toolchain. +System requirements: +- POSIX-compliant system (tested on Linux & OSX) +- Python 2.7 -Installing prerequisites on Ubuntu:: +Synapse is written in python but some of the libraries is uses are written in +C. So before we can install synapse itself we need a working C compiler and the +header files for python C extensions. - $ sudo apt-get install build-essential python2.7-dev libffi-dev +Installing prerequisites on Ubuntu or Debian:: + + $ sudo apt-get install build-essential python2.7-dev libffi-dev \ + python-pip python-setuptools Installing prerequisites on Mac OS X:: $ xcode-select --install + +To install the synapse homeserver run:: + + $ pip install --user --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + +This installs synapse, along with the libraries it uses, into +``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. + +Troubleshooting Installation +---------------------------- + +Synapse requires pip 1.7 or later, so if your OS provides too old a version and +you get errors about ``error: no such option: --process-dependency-links`` you +may need to manually upgrade it:: + + $ sudo pip install --upgrade pip + +If pip crashes mid-installation for reason (e.g. lost terminal), pip may +refuse to run until you remove the temporary installation directory it +created. To reset the installation:: + + $ rm -rf /tmp/pip_install_matrix + +pip seems to leak *lots* of memory during installation. For instance, a Linux +host with 512MB of RAM may run out of memory whilst installing Twisted. If this +happens, you will have to individually install the dependencies which are +failing, e.g.: + + $ pip install --user twisted + +Running Your Homeserver +======================= + +To actually run your new homeserver, pick a working directory for Synapse to run +(e.g. ``~/.synapse``), and:: + + $ mkdir ~/.synapse + $ cd ~/.synapse + + $ # on Linux + $ ~/.local/bin/synctl start + + $ # on OSX + $ ~/Library/Python/2.7/bin/synctl start + +Troubleshooting Running +----------------------- + +If ``synctl`` fails with ``pkg_resources.DistributionNotFound`` errors you may +need a newer version of setuptools than that provided by your OS. + + $ sudo pip install setuptools --upgrade + +If synapse fails with ``missing "sodium.h"`` crypto errors, you may need +to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for +encryption and digital signatures. +Unfortunately PyNACL currently has a few issues +(https://github.com/pyca/pynacl/issues/53) and +(https://github.com/pyca/pynacl/issues/79) that mean it may not install +correctly, causing all tests to fail with errors about missing "sodium.h". To +fix try re-installing from PyPI or directly from +(https://github.com/pyca/pynacl):: + + $ # Install from PyPI + $ pip install --user --upgrade --force pynacl + $ # Install from github + $ pip install --user https://github.com/pyca/pynacl/tarball/master + + +Homeserver Development +====================== + +To check out a homeserver for development, clone the git repo into a working +directory of your choice: + + $ git clone https://github.com/matrix-org/synapse.git + $ cd synapse The homeserver has a number of external dependencies, that are easiest to install by making setup.py do so, in --user mode:: $ python setup.py develop --user -You'll need a version of setuptools new enough to know about git, so you -may need to also run:: - - $ sudo apt-get install python-pip - $ sudo pip install --upgrade setuptools - -If you don't have access to github, then you may need to install ``syutil`` -manually by checking it out and running ``python setup.py develop --user`` on -it too. - -If you get errors about ``sodium.h`` being missing, you may also need to -manually install a newer PyNaCl via pip as setuptools installs an old one. Or -you can check PyNaCl out of git directly (https://github.com/pyca/pynacl) and -installing it. Installing PyNaCl using pip may also work (remember to remove -any other versions installed by setuputils in, for example, ~/.local/lib). - -On OSX, if you encounter ``clang: error: unknown argument: '-mno-fused-madd'`` -you will need to ``export CFLAGS=-Qunused-arguments``. - This will run a process of downloading and installing into your user's .local/lib directory all of the required dependencies that are missing. @@ -180,8 +201,8 @@ This should end with a 'PASSED' result:: Upgrading an existing homeserver ================================ -Before upgrading an existing homeserver to a new version, please refer to -UPGRADE.rst for any additional instructions. +IMPORTANT: Before upgrading an existing homeserver to a new version, please +refer to UPGRADE.rst for any additional instructions. Setting up Federation @@ -202,18 +223,15 @@ IDs: domain name. For the first form, simply pass the required hostname (of the machine) as the ---host parameter:: +--server-name parameter:: - $ python synapse/app/homeserver.py \ + $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.config \ --generate-config - $ python synapse/app/homeserver.py --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.config -Alternatively, you can run synapse via synctl - running ``synctl start`` to -generate a homeserver.yaml config file, where you can then edit server-name to -specify machine.my.domain.name, and then set the actual server running again -with synctl start. +Alternatively, you can run ``synctl start`` to guide you through the process. For the second form, first create your SRV record and publish it in DNS. This needs to be named _matrix._tcp.YOURDOMAIN, and point at at least one hostname @@ -221,17 +239,19 @@ and port where the server is running. (At the current time synapse does not support clustering multiple servers into a single logical homeserver). The DNS record would then look something like:: + $ dig -t srv _matrix._tcp.machine.my.domaine.name _matrix._tcp IN SRV 10 0 8448 machine.my.domain.name. + At this point, you should then run the homeserver with the hostname of this SRV record, as that is the name other machines will expect it to have:: - $ python synapse/app/homeserver.py \ + $ python -m synapse.app.homeserver \ --server-name YOURDOMAIN \ --bind-port 8448 \ --config-path homeserver.config \ --generate-config - $ python synapse/app/homeserver.py --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.config You may additionally want to pass one or more "-v" options, in order to @@ -250,6 +270,8 @@ private federation (``localhost:8080``, ``localhost:8081`` and http://localhost:8080. Simply run:: $ demo/start.sh + +This is mainly useful just for development purposes. Running The Demo Web Client =========================== @@ -308,13 +330,14 @@ time. Where's the spec?! ================== -For now, please go spelunking in the ``docs/`` directory to find out. +The source of the matrix spec lives at https://github.com/matrix-org/matrix-doc. +A recent HTML snapshot of this lives at http://matrix.org/docs/spec Building Internal API Documentation =================================== -Before building internal API documentation install spinx and +Before building internal API documentation install sphinx and sphinxcontrib-napoleon:: $ pip install sphinx diff --git a/UPGRADE.rst b/UPGRADE.rst index 99ce1a2d3..2229470c3 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,33 @@ +Upgrading to v0.5.0 +=================== + +This release completely changes the database schema and so requires upgrading +it before starting the new version of the homeserver. + +The script "database-prepare-for-0.5.0.sh" should be used to upgrade the +database. This will save all user information, such as logins and profiles, +but will otherwise purge the database. This includes messages, which +rooms the home server was a member of and room alias mappings. + +If you would like to keep your history, please take a copy of your database +file and ask for help in #matrix:matrix.org. The upgrade process is, +unfortunately, non trivial and requires human intervention to resolve any +resutling conflicts during the upgrade process. + +Before running the command the homeserver should be first completely +shutdown. To run it, simply specify the location of the database, e.g.: + + ./database-prepare-for-0.5.0.sh "homeserver.db" + +Once this has successfully completed it will be safe to restart the +homeserver. You may notice that the homeserver takes a few seconds longer to +restart than usual as it reinitializes the database. + +On startup of the new version, users can either rejoin remote rooms using room +aliases or by being reinvited. Alternatively, if any other homeserver sends a +message to a room that the homeserver was previously in the local HS will +automatically rejoin the room. + Upgrading to v0.4.0 =================== diff --git a/VERSION b/VERSION index 2b7c5ae01..8f0916f76 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.2 +0.5.0 diff --git a/database-prepare-for-0.5.0.sh b/database-prepare-for-0.5.0.sh new file mode 100755 index 000000000..e824cb583 --- /dev/null +++ b/database-prepare-for-0.5.0.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This is will prepare a synapse database for running with v0.5.0 of synapse. +# It will store all the user information, but will *delete* all messages and +# room data. + +set -e + +cp "$1" "$1.bak" + +DUMP=$(sqlite3 "$1" << 'EOF' +.dump users +.dump access_tokens +.dump presence +.dump profiles +EOF +) + +rm "$1" + +sqlite3 "$1" <<< "$DUMP" diff --git a/demo/start.sh b/demo/start.sh index fc6cd6303..ce3e29248 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -32,7 +32,7 @@ for port in 8080 8081 8082; do -D --pid-file "$DIR/$port.pid" \ --manhole $((port + 1000)) \ --tls-dh-params-path "demo/demo.tls.dh" \ - $PARAMS + $PARAMS $SYNAPSE_PARAMS python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ @@ -40,7 +40,4 @@ for port in 8080 8081 8082; do done -echo "Starting webclient on port 8000..." -python "demo/webserver.py" -p 8000 -P "$DIR/webserver.pid" "webclient" - cd "$CWD" diff --git a/docs/implementation-notes/python_architecture.rst b/docs/ancient_architecture_notes.rst similarity index 93% rename from docs/implementation-notes/python_architecture.rst rename to docs/ancient_architecture_notes.rst index 8beaa615d..2a5a2613c 100644 --- a/docs/implementation-notes/python_architecture.rst +++ b/docs/ancient_architecture_notes.rst @@ -1,3 +1,9 @@ +.. WARNING:: + These architecture notes are spectacularly old, and date back to when Synapse + was just federation code in isolation. This should be merged into the main + spec. + + = Server to Server = == Server to Server Stack == diff --git a/docs/architecture.rst b/docs/architecture.rst new file mode 100644 index 000000000..98050428b --- /dev/null +++ b/docs/architecture.rst @@ -0,0 +1,68 @@ +Synapse Architecture +==================== + +As of the end of Oct 2014, Synapse's overall architecture looks like:: + + synapse + .-----------------------------------------------------. + | Notifier | + | ^ | | + | | | | + | .------------|------. | + | | handlers/ | | | + | | v | | + | | Event*Handler <--------> rest/* <=> Client + | | Rooms*Handler | | + HSes <=> federation/* <==> FederationHandler | | + | | | PresenceHandler | | + | | | TypingHandler | | + | | '-------------------' | + | | | | | + | | state/* | | + | | | | | + | | v v | + | `--------------> storage/* | + | | | + '--------------------------|--------------------------' + v + .----. + | DB | + '----' + +* Handlers: business logic of synapse itself. Follows a set contract of BaseHandler: + + - BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic): + + + handle_state(event) + + auth(event) + + persist_event(event) + + notify notifier or federation(event) + + - PresenceHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + - TypingHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + - EventsHandler: handles the events stream... + - FederationHandler: - gets PDU from Federation Layer; turns into an event; + follows basehandler functionality. + - RoomsHandler: does all the room logic, including members - lots of classes in + RoomsHandler. + - ProfileHandler: talks to the storage to store/retrieve profile info. + +* EventFactory: generates events of particular event types. +* Notifier: Backs the events handler +* REST: Interfaces handlers and events to the outside world via HTTP/JSON. + Converts events back and forth from JSON. +* Federation: holds the HTTP client & server to talk to other servers. Does + replication to make sure there's nothing missing in the graph. Handles + reliability. Handles txns. +* Distributor: generic event bus. used for presence & typing only currently. + Notifier could be implemented using Distributor - so far we are only using for + things which actually /require/ dynamic pluggability however as it can + obfuscate the actual flow of control. +* Auth: helper singleton to say whether a given event is allowed to do a given + thing (TODO: put this on the diagram) +* State: helper singleton: does state conflict resolution. You give it an event + and it tells you if it actually updates the state or not, and annotates the + event up properly and handles merge conflict resolution. +* Storage: abstracts the storage engine. diff --git a/docs/client-server/OLD_specification.rst b/docs/client-server/OLD_specification.rst deleted file mode 100644 index 47fba5eea..000000000 --- a/docs/client-server/OLD_specification.rst +++ /dev/null @@ -1,1283 +0,0 @@ -======================== -Matrix Client-Server API -======================== - - -.. WARNING:: - This specification is old. Please see /docs/specification.rst instead. - - - - - - - - - - - -The following specification outlines how a client can send and receive data from -a home server. - -[[TODO(kegan): 4/7/14 Grilling -- Mechanism for getting historical state changes (e.g. topic updates) - add - query param flag? -- Generic mechanism for linking first class events (e.g. feedback) with other s - first class events (e.g. messages)? -- Generic mechanism for updating 'stuff about the room' (e.g. favourite coffee) - AND specifying clobbering rules (clobber/add to list/etc)? -- How to ensure a consistent view for clients paginating through room lists? - They aren't really ordered in any way, and if you're paginating - through them, how can you show them a consistent result set? Temporary 'room - list versions' akin to event version? How does that work? -]] - -[[TODO(kegan): -Outstanding problems / missing spec: -- Push -- Typing notifications -]] - -Terminology ------------ -Stream Tokens: -An opaque token used to make further streaming requests. When using any -pagination streaming API, responses will contain a start and end stream token. -When reconnecting to the stream, these tokens can be used to tell the server -where the client got up to in the stream. - -Event ID: -Every event that comes down the event stream or that is returned from the REST -API has an associated event ID (event_id). This ID will be the same between the -REST API and the event stream, so any duplicate events can be clobbered -correctly without knowing anything else about the event. - -Message ID: -The ID of a message sent by a client in a room. Clients send IMs to each other -in rooms. Each IM sent by a client must have a unique message ID which is unique -for that particular client. - -User ID: -The @username:host style ID of the client. When registering for an account, the -client specifies their username. The user_id is this username along with the -home server's unique hostname. When federating between home servers, the user_id -is used to uniquely identify users across multiple home servers. - -Room ID: -The room_id@host style ID for the room. When rooms are created, the client either -specifies or is allocated a room ID. This room ID must be used to send messages -in that room. Like with clients, there may be multiple rooms with the same ID -across multiple home servers. The room_id is used to uniquely identify a room -when federating. - -Global message ID: -The globally unique ID for a message. This ID is formed from the msg_id, the -client's user_id and the room_id. This uniquely identifies any -message. It is represented with '-' as the delimeter between IDs. The -global_msg_id is of the form: room_id-user_id-msg_id - - -REST API and the Event Stream ------------------------------ -Clients send data to the server via a RESTful API. They can receive data via -this API or from an event stream. An event stream is a special path which -streams all events the client may be interested in. This makes it easy to -immediately receive updates from the REST API. All data is represented as JSON. - -Pagination streaming API -======================== -Clients are often interested in very large datasets. The data itself could -be 1000s of messages in a given room, 1000s of rooms in a public room list, or -1000s of events (presence, typing, messages, etc) in the system. It is not -practical to send vast quantities of data to the client every time they -request a list of public rooms for example. There needs to be a way to show a -subset of this data, and apply various filters to it. This is what the pagination -streaming API is. This API defines standard request/response parameters which -can be used when navigating this stream of data. - -Pagination Request Query Parameters ------------------------------------ -Clients may wish to paginate results from the event stream, or other sources of -information where the amount of information may be a problem, -e.g. in a room with 10,000s messages. The pagination query parameters provide a -way to navigate a 'window' around a large set of data. These -parameters are only valid for GET requests. - - S e r v e r - s i d e d a t a - |-------------------------------------------------| -START ^ ^ END - |_______________| - | - Client-extraction - -'START' and 'END' are magic token values which specify the start and end of the -dataset respectively. - -Query parameters: - from : $streamtoken - The opaque token to start streaming from. - to : $streamtoken - The opaque token to end streaming at. Typically, - clients will not know the item of data to end at, so this will usually be - START or END. - limit : integer - An integer representing the maximum number of items to - return. - -For example, the event stream has events E1 -> E15. The client wants the last 5 -events and doesn't know any previous events: - -S E -|-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| -| | | -| _____| | -|__________________ | ___________________| - | | | - GET /events?to=START&limit=5&from=END - Returns: - E15,E14,E13,E12,E11 - - -Another example: a public room list has rooms R1 -> R17. The client is showing 5 -rooms at a time on screen, and is on page 2. They want to -now show page 3 (rooms R11 -> 15): - -S E -| 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token -|-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room - |____________| |________________| - | | - Currently | - viewing | - | - GET /rooms/list?from=9&to=END&limit=5 - Returns: R11,R12,R13,R14,R15 - -Note that tokens are treated in an *exclusive*, not inclusive, manner. The end -token from the intial request was '9' which corresponded to R10. When the 2nd -request was made, R10 did not appear again, even though from=9 was specified. If -you know the token, you already have the data. - -Pagination Response -------------------- -Responses to pagination requests MUST follow the format: -{ - "chunk": [ ... , Responses , ... ], - "start" : $streamtoken, - "end" : $streamtoken -} -Where $streamtoken is an opaque token which can be used in another query to -get the next set of results. The "start" and "end" keys can only be omitted if -the complete dataset is provided in "chunk". - -If the client wants earlier results, they should use from=$start_streamtoken, -to=START. Likewise, if the client wants later results, they should use -from=$end_streamtoken, to=END. - -Unless specified, the default pagination parameters are from=START, to=END, -without a limit set. This allows you to hit an API like -/events without any query parameters to get everything. - -The Event Stream ----------------- -The event stream returns events using the pagination streaming API. When the -client disconnects for a while and wants to reconnect to the event stream, they -should specify from=$end_streamtoken. This lets the server know where in the -event stream the client is. These tokens are completely opaque, and the client -cannot infer anything from them. - - GET /events?from=$LAST_STREAM_TOKEN - REST Path: /events - Returns (success): A JSON array of Event Data. - Returns (failure): An Error Response - -LAST_STREAM_TOKEN is the last stream token obtained from the event stream. If the -client is connecting for the first time and does not know any stream tokens, -they can use "START" to request all events from the start. For more information -on this, see "Pagination Request Query Parameters". - -The event stream supports shortpoll and longpoll with the "timeout" query -parameter. This parameter specifies the number of milliseconds the server should -hold onto the connection waiting for incoming events. If no events occur in this -period, the connection will be closed and an empty chunk will be returned. To -use shortpoll, specify "timeout=0". - -Event Data ----------- -This is a JSON object which looks like: -{ - "event_id" : $EVENT_ID, - "type" : $EVENT_TYPE, - $URL_ARGS, - "content" : { - $EVENT_CONTENT - } -} - -EVENT_ID - An ID identifying this event. This is so duplicate events can be suppressed on - the client. - -EVENT_TYPE - The namespaced event type (m.*) - -URL_ARGS - Path specific data from the REST API. - -EVENT_CONTENT - The event content, matching the REST content PUT previously. - -Events are differentiated via the event type "type" key. This is the type of -event being received. This can be expanded upon by using different namespaces. -Every event MUST have a 'type' key. - -Most events will have a corresponding REST URL. This URL will generally have -data in it to represent the resource being modified, -e.g. /rooms/$room_id. The event data will contain extra top-level keys to expose -this information to clients listening on an event -stream. The event content maps directly to the contents submitted via the REST -API. - -For example: - Event Type: m.example.room.members - REST Path: /examples/room/$room_id/members/$user_id - REST Content: { "membership" : "invited" } - -is represented in the event stream as: - -{ - "event_id" : "e_some_event_id", - "type" : "m.example.room.members", - "room_id" : $room_id, - "user_id" : $user_id, - "content" : { - "membership" : "invited" - } -} - -As convention, the URL variable "$varname" will map directly onto the name -of the JSON key "varname". - -Error Responses ---------------- -If the client sends an invalid request, the server MAY respond with an error -response. This is of the form: -{ - "error" : "string", - "errcode" : "string" -} -The 'error' string will be a human-readable error message, usually a sentence -explaining what went wrong. - -The 'errcode' string will be a unique string which can be used to handle an -error message e.g. "M_FORBIDDEN". These error codes should have their namespace -first in ALL CAPS, followed by a single _. For example, if there was a custom -namespace com.mydomain.here, and a "FORBIDDEN" code, the error code should look -like "COM.MYDOMAIN.HERE_FORBIDDEN". There may be additional keys depending on -the error, but the keys 'error' and 'errcode' will always be present. - -Some standard error codes are below: - -M_FORBIDDEN: -Forbidden access, e.g. joining a room without permission, failed login. - -M_UNKNOWN_TOKEN: -The access token specified was not recognised. - -M_BAD_JSON: -Request contained valid JSON, but it was malformed in some way, e.g. missing -required keys, invalid values for keys. - -M_NOT_JSON: -Request did not contain valid JSON. - -M_NOT_FOUND: -No resource was found for this request. - -Some requests have unique error codes: - -M_USER_IN_USE: -Encountered when trying to register a user ID which has been taken. - -M_ROOM_IN_USE: -Encountered when trying to create a room which has been taken. - -M_BAD_PAGINATION: -Encountered when specifying bad pagination values to a Pagination Streaming API. - - -======== -REST API -======== - -All content must be application/json. Some keys are required, while others are -optional. Unless otherwise specified, -all HTTP PUT/POST/DELETEs will return a 200 OK with an empty response body on -success, and a 4xx/5xx with an optional Error Response on failure. When sending -data, if there are no keys to send, an empty JSON object should be sent. - -All POST/PUT/GET/DELETE requests MUST have an 'access_token' query parameter to -allow the server to authenticate the client. All -POST requests MUST be submitted as application/json. - -All paths MUST be namespaced by the version of the API being used. This should -be: - -/_matrix/client/api/v1 - -All REST paths in this section MUST be prefixed with this. E.g. - REST Path: /rooms/$room_id - Absolute Path: /_matrix/client/api/v1/rooms/$room_id - -Registration -============ -Clients must register with the server in order to use the service. After -registering, the client will be given an -access token which must be used in ALL requests as a query parameter -'access_token'. - -Registering for an account --------------------------- - POST /register - With: A JSON object containing the key "user_id" which contains the desired - user_id, or an empty JSON object to have the server allocate a user_id - automatically. - Returns (success): 200 OK with a JSON object: - { - "user_id" : "string [user_id]", - "access_token" : "string" - } - Returns (failure): An Error Response. M_USER_IN_USE if the user ID is taken. - - -Unregistering an account ------------------------- - POST /unregister - With query parameters: access_token=$ACCESS_TOKEN - Returns (success): 200 OK - Returns (failure): An Error Response. - - -Logging in to an existing account -================================= -If the client has already registered, they need to be able to login to their -account. The home server may provide many different ways of logging in, such -as user/password auth, login via a social network (OAuth), login by confirming -a token sent to their email address, etc. This section does NOT define how home -servers should authorise their users who want to login to their existing -accounts. This section defines the standard interface which implementations -should follow so that ANY client can login to ANY home server. - -The login process breaks down into the following: - 1: Get login process info. - 2: Submit the login stage credentials. - 3: Get access token or be told the next stage in the login process and repeat - step 2. - -Getting login process info: - GET /login - Returns (success): 200 OK with LoginInfo. - Returns (failure): An Error Response. - -Submitting the login stage credentials: - POST /login - With: LoginSubmission - Returns (success): 200 OK with LoginResult - Returns (failure): An Error Response - -Where LoginInfo is a JSON object which MUST have a "type" key which denotes the -login type. If there are multiple login stages, this object MUST also contain a -"stages" key, which has a JSON array of login types denoting all the steps in -order to login, including the first stage which is in "type". This allows the -client to make an informed decision as to whether or not they can natively -handle the entire login process, or whether they should fallback (see below). - -Where LoginSubmission is a JSON object which MUST have a "type" key. - -Where LoginResult is a JSON object which MUST have either a "next" key OR an -"access_token" key, depending if the login process is over or not. This object -MUST have a "session" key if multiple POSTs need to be sent to /login. - -Fallback --------- -If the client does NOT know how to handle the given type, they should: - GET /login/fallback -This MUST return an HTML page which can perform the entire login process. - -Password-based --------------- -Type: "m.login.password" -LoginSubmission: -{ - "type": "m.login.password", - "user": , - "password": -} - -Example: -Assume you are @bob:matrix.org and you wish to login on another mobile device. -First, you GET /login which returns: -{ - "type": "m.login.password" -} -Your client knows how to handle this, so your client prompts the user to enter -their username and password. This is then submitted: -{ - "type": "m.login.password", - "user": "@bob:matrix.org", - "password": "monkey" -} -The server checks this, finds it is valid, and returns: -{ - "access_token": "abcdef0123456789" -} -The server may optionally return "user_id" to confirm or change the user's ID. -This is particularly useful if the home server wishes to support localpart entry -of usernames (e.g. "bob" rather than "@bob:matrix.org"). - -OAuth2-based ------------- -Type: "m.login.oauth2" -This is a multi-stage login. - -LoginSubmission: -{ - "type": "m.login.oauth2", - "user": -} -Returns: -{ - "uri": -} - -The home server acts as a 'confidential' Client for the purposes of OAuth2. - -If the uri is a "sevice selection uri", it is a simple page which prompts the -user to choose which service to authorize with. On selection of a service, they -link through to Authorization Request URIs. If there is only 1 service which the -home server accepts when logging in, this indirection can be skipped and the -"uri" key can be the Authorization Request URI. - -The client visits the Authorization Request URI, which then shows the OAuth2 -Allow/Deny prompt. Hitting 'Allow' returns the redirect URI with the auth code. -Home servers can choose any path for the redirect URI. The client should visit -the redirect URI, which will then finish the OAuth2 login process, granting the -home server an access token for the chosen service. When the home server gets -this access token, it knows that the cilent has authed with the 3rd party, and -so can return a LoginResult. - -The OAuth redirect URI (with auth code) MUST return a LoginResult. - -Example: -Assume you are @bob:matrix.org and you wish to login on another mobile device. -First, you GET /login which returns: -{ - "type": "m.login.oauth2" -} -Your client knows how to handle this, so your client prompts the user to enter -their username. This is then submitted: -{ - "type": "m.login.oauth2", - "user": "@bob:matrix.org" -} -The server only accepts auth from Google, so returns the Authorization Request -URI for Google: -{ - "uri": "https://accounts.google.com/o/oauth2/auth?response_type=code& - client_id=CLIENT_ID&redirect_uri=REDIRECT_URI&scope=photos" -} -The client then visits this URI and authorizes the home server. The client then -visits the REDIRECT_URI with the auth code= query parameter which returns: -{ - "access_token": "0123456789abcdef" -} - -Email-based (code) ------------------- -Type: "m.login.email.code" -This is a multi-stage login. - -First LoginSubmission: -{ - "type": "m.login.email.code", - "user": - "email": -} -Returns: -{ - "session": -} - -The email contains a code which must be sent in the next LoginSubmission: -{ - "type": "m.login.email.code", - "session": , - "code": -} -Returns: -{ - "access_token": -} - -Example: -Assume you are @bob:matrix.org and you wish to login on another mobile device. -First, you GET /login which returns: -{ - "type": "m.login.email.code" -} -Your client knows how to handle this, so your client prompts the user to enter -their email address. This is then submitted: -{ - "type": "m.login.email.code", - "user": "@bob:matrix.org", - "email": "bob@mydomain.com" -} -The server confirms that bob@mydomain.com is linked to @bob:matrix.org, then -sends an email to this address and returns: -{ - "session": "ewuigf7462" -} -The client's screen changes to a code submission page. The email arrives and it -says something to the effect of "please enter 2348623 into the app". This is -the submitted along with the session: -{ - "type": "m.login.email.code", - "session": "ewuigf7462", - "code": "2348623" -} -The server accepts this and returns: -{ - "access_token": "abcdef0123456789" -} - -Email-based (url) ------------------ -Type: "m.login.email.url" -This is a multi-stage login. - -First LoginSubmission: -{ - "type": "m.login.email.url", - "user": - "email": -} -Returns: -{ - "session": -} - -The email contains a URL which must be clicked. After it has been clicked, the -client should perform a request: -{ - "type": "m.login.email.code", - "session": -} -Returns: -{ - "access_token": -} - -Example: -Assume you are @bob:matrix.org and you wish to login on another mobile device. -First, you GET /login which returns: -{ - "type": "m.login.email.url" -} -Your client knows how to handle this, so your client prompts the user to enter -their email address. This is then submitted: -{ - "type": "m.login.email.url", - "user": "@bob:matrix.org", - "email": "bob@mydomain.com" -} -The server confirms that bob@mydomain.com is linked to @bob:matrix.org, then -sends an email to this address and returns: -{ - "session": "ewuigf7462" -} -The client then starts polling the server with the following: -{ - "type": "m.login.email.url", - "session": "ewuigf7462" -} -(Alternatively, the server could send the device a push notification when the -email has been validated). The email arrives and it contains a URL to click on. -The user clicks on the which completes the login process with the server. The -next time the client polls, it returns: -{ - "access_token": "abcdef0123456789" -} - -N-Factor auth -------------- -Multiple login stages can be combined with the "next" key in the LoginResult. - -Example: -A server demands an email.code then password auth before logging in. First, the -client performs a GET /login which returns: -{ - "type": "m.login.email.code", - "stages": ["m.login.email.code", "m.login.password"] -} -The client performs the email login (See "Email-based (code)"), but instead of -returning an access_token, it returns: -{ - "next": "m.login.password" -} -The client then presents a user/password screen and the login continues until -this is complete (See "Password-based"), which then returns the "access_token". - -Rooms -===== -A room is a conceptual place where users can send and receive messages. Rooms -can be created, joined and left. Messages are sent -to a room, and all participants in that room will receive the message. Rooms are -uniquely identified via the room_id. - -Creating a room (with a room ID) --------------------------------- - Event Type: m.room.create [TODO(kegan): Do we generate events for this?] - REST Path: /rooms/$room_id - Valid methods: PUT - Required keys: None. - Optional keys: - visibility : [public|private] - Set whether this room shows up in the public - room list. - Returns: - On Failure: MAY return a suggested alternative room ID if this room ID is - taken. - { - suggested_room_id : $new_room_id - error : "Room already in use." - errcode : "M_ROOM_IN_USE" - } - - -Creating a room (without a room ID) ------------------------------------ - Event Type: m.room.create [TODO(kegan): Do we generate events for this?] - REST Path: /rooms - Valid methods: POST - Required keys: None. - Optional keys: - visibility : [public|private] - Set whether this room shows up in the public - room list. - Returns: - On Success: The allocated room ID. Additional information about the room - such as the visibility MAY be included as extra keys in this response. - { - room_id : $room_id - } - -Setting the topic for a room ----------------------------- - Event Type: m.room.topic - REST Path: /rooms/$room_id/topic - Valid methods: GET/PUT - Required keys: - topic : $topicname - Set the topic to $topicname in room $room_id. - - -See a list of public rooms --------------------------- - REST Path: /public/rooms?pagination_query_parameters - Valid methods: GET - This API can use pagination query parameters. - Returns: - { - "chunk" : JSON array of RoomInfo JSON objects - Required. - "start" : "string (start token)" - See Pagination Response. - "end" : "string (end token)" - See Pagination Response. - "total" : integer - Optional. The total number of rooms. - } - -RoomInfo: Information about a single room. - Servers MUST send the key: room_id - Servers MAY send the keys: topic, num_members - { - "room_id" : "string", - "topic" : "string", - "num_members" : integer - } - -Room Members -============ - -Invite/Joining/Leaving a room ------------------------------ - Event Type: m.room.member - REST Path: /rooms/$room_id/members/$user_id/state - Valid methods: PUT/GET/DELETE - Required keys: - membership : [join|invite] - The membership state of $user_id in room - $room_id. - Optional keys: - displayname, - avatar_url : String fields from the member user's profile - state, - status_msg, - mtime_age : Presence information - - These optional keys provide extra information that the client is likely to - be interested in so it doesn't have to perform an additional profile or - presence information fetch. - -Where: - join - Indicate you ($user_id) are joining the room $room_id. - invite - Indicate that $user_id has been invited to room $room_id. - -User $user_id can leave room $room_id by DELETEing this path. - -Checking the user list of a room --------------------------------- - REST Path: /rooms/$room_id/members/list - This API can use pagination query parameters. - Valid methods: GET - Returns: - A pagination response with chunk data as m.room.member events. - -Messages -======== -Users send messages to other users in rooms. These messages may be text, images, -video, etc. Clients may also want to acknowledge messages by sending feedback, -in the form of delivery/read receipts. - -Server-attached keys --------------------- -The server MAY attach additional keys to messages and feedback. If a client -submits keys with the same name, they will be clobbered by -the server. - -Required keys: -from : "string [user_id]" - The user_id of the user who sent the message/feedback. - -Optional keys: -hsob_ts : integer - A timestamp (ms resolution) representing when the message/feedback got to the - sender's home server ("home server outbound timestamp"). - -hsib_ts : integer - A timestamp (ms resolution) representing when the - message/feedback got to the receiver's home server ("home server inbound - timestamp"). This may be the same as hsob_ts if the sender/receiver are on the - same home server. - -Sending messages ----------------- - Event Type: m.room.message - REST Path: /rooms/$room_id/messages/$from/$msg_id - Valid methods: GET/PUT - URL parameters: - $from : user_id - The sender's user_id. This value will be clobbered by the - server before sending. - Required keys: - msgtype: [m.text|m.emote|m.image|m.audio|m.video|m.location|m.file] - - The type of message. Not to be confused with the Event 'type'. - Optional keys: - sender_ts : integer - A timestamp (ms resolution) representing the - wall-clock time when the message was sent from the client. - Reserved keys: - body : "string" - The human readable string for compatibility with clients - which cannot process a given msgtype. This key is optional, but - if it is included, it MUST be human readable text - describing the message. See individual msgtypes for more - info on what this means in practice. - -Each msgtype may have required fields of their own. - -msgtype: m.text ----------------- -Required keys: - body : "string" - The body of the message. -Optional keys: - None. - -msgtype: m.emote ------------------ -Required keys: - body : "string" - *tries to come up with a witty explanation*. -Optional keys: - None. - -msgtype: m.image ------------------ -Required keys: - url : "string" - The URL to the image. -Optional keys: - body : "string" - info : JSON object (ImageInfo) - The image info for image - referred to in 'url'. - thumbnail_url : "string" - The URL to the thumbnail. - thumbnail_info : JSON object (ImageInfo) - The image info for the image - referred to in 'thumbnail_url'. - -ImageInfo: Information about an image. -{ - "size" : integer (size of image in bytes), - "w" : integer (width of image in pixels), - "h" : integer (height of image in pixels), - "mimetype" : "string (e.g. image/jpeg)" -} - -Interpretation of 'body' key: The alt text of the image, or some kind of content -description for accessibility e.g. "image attachment". - -msgtype: m.audio ------------------ -Required keys: - url : "string" - The URL to the audio. -Optional keys: - info : JSON object (AudioInfo) - The audio info for the audio referred to in - 'url'. - -AudioInfo: Information about a piece of audio. -{ - "mimetype" : "string (e.g. audio/aac)", - "size" : integer (size of audio in bytes), - "duration" : integer (duration of audio in milliseconds) -} - -Interpretation of 'body' key: A description of the audio e.g. "Bee Gees - -Stayin' Alive", or some kind of content description for accessibility e.g. -"audio attachment". - -msgtype: m.video ------------------ -Required keys: - url : "string" - The URL to the video. -Optional keys: - info : JSON object (VideoInfo) - The video info for the video referred to in - 'url'. - -VideoInfo: Information about a video. -{ - "mimetype" : "string (e.g. video/mp4)", - "size" : integer (size of video in bytes), - "duration" : integer (duration of video in milliseconds), - "w" : integer (width of video in pixels), - "h" : integer (height of video in pixels), - "thumbnail_url" : "string (URL to image)", - "thumbanil_info" : JSON object (ImageInfo) -} - -Interpretation of 'body' key: A description of the video e.g. "Gangnam style", -or some kind of content description for accessibility e.g. "video attachment". - -msgtype: m.location --------------------- -Required keys: - geo_uri : "string" - The geo URI representing the location. -Optional keys: - thumbnail_url : "string" - The URL to a thumnail of the location being - represented. - thumbnail_info : JSON object (ImageInfo) - The image info for the image - referred to in 'thumbnail_url'. - -Interpretation of 'body' key: A description of the location e.g. "Big Ben, -London, UK", or some kind of content description for accessibility e.g. -"location attachment". - - -Sending feedback ----------------- -When you receive a message, you may want to send delivery receipt to let the -sender know that the message arrived. You may also want to send a read receipt -when the user has read the message. These receipts are collectively known as -'feedback'. - - Event Type: m.room.message.feedback - REST Path: /rooms/$room_id/messages/$msgfrom/$msg_id/feedback/$from/$feedback - Valid methods: GET/PUT - URL parameters: - $msgfrom - The sender of the message's user_id. - $from : user_id - The sender of the feedback's user_id. This value will be - clobbered by the server before sending. - $feedback : [d|r] - Specify if this is a [d]elivery or [r]ead receipt. - Required keys: - None. - Optional keys: - sender_ts : integer - A timestamp (ms resolution) representing the - wall-clock time when the receipt was sent from the client. - -Receiving messages (bulk/pagination) ------------------------------------- - Event Type: m.room.message - REST Path: /rooms/$room_id/messages/list - Valid methods: GET - Query Parameters: - feedback : [true|false] - Specify if feedback should be bundled with each - message. - This API can use pagination query parameters. - Returns: - A JSON array of Event Data in "chunk" (see Pagination Response). If the - "feedback" parameter was set, the Event Data will also contain a "feedback" - key which contains a JSON array of feedback, with each element as Event Data - with compressed feedback for this message. - -Event Data with compressed feedback is a special type of feedback with -contextual keys removed. It is designed to limit the amount of redundant data -being sent for feedback. This removes the type, event_id, room ID, -message sender ID and message ID keys. - - ORIGINAL (via event streaming) -{ - "event_id":"e1247632487", - "type":"m.room.message.feedback", - "from":"string [user_id]", - "feedback":"string [d|r]", - "room_id":"$room_id", - "msg_id":"$msg_id", - "msgfrom":"$msgfromid", - "content":{ - "sender_ts":139880943 - } -} - - COMPRESSED (via /messages/list) -{ - "from":"string [user_id]", - "feedback":"string [d|r]", - "content":{ - "sender_ts":139880943 - } -} - -When you join a room $room_id, you may want the last 10 messages with feedback. -This is represented as: - GET - /rooms/$room_id/messages/list?from=END&to=START&limit=10&feedback=true - -You may want to get 10 messages even earlier than that without feedback. If the -start stream token from the previous request was stok_019173, this request would -be: - GET - /rooms/$room_id/messages/list?from=stok_019173&to=START&limit=10& - feedback=false - -NOTE: Care must be taken when using this API in conjunction with event - streaming. It is possible that this will return a message which will - then come down the event stream, resulting in a duplicate message. Clients - should clobber based on the global message ID, or event ID. - - -Get current state for all rooms (aka IM Initial Sync API) -------------------------------- - REST Path: /im/sync - Valid methods: GET - This API can use pagination query parameters. Pagination is applied on a per - *room* basis. E.g. limit=1 means "get 1 message for each room" and not "get 1 - room's messages". If there is no limit, all messages for all rooms will be - returned. - If you want 1 room's messages, see "Receiving messages (bulk/pagination)". - Additional query parameters: - feedback: [true] - Bundles feedback with messages. - Returns: - An array of RoomStateInfo. - -RoomStateInfo: A snapshot of information about a single room. - { - "room_id" : "string", - "membership" : "string [join|invite]", - "messages" : { - "start": "string", - "end": "string", - "chunk": - m.room.message pagination stream events (with feedback if specified), - this is the same as "Receiving messages (bulk/pagination)". - } - } -The "membership" key is the calling user's membership state in the given -"room_id". The "messages" key may be omitted if the "membership" value is -"invite". Additional keys may be added to the top-level object, such as: - "topic" : "string" - The topic for the room in question. - "room_image_url" : "string" - The URL of the room image if specified. - "num_members" : integer - The number of members in the room. - - -Profiles -======== - -Getting/Setting your own displayname ------------------------------------- - REST Path: /profile/$user_id/displayname - Valid methods: GET/PUT - Required keys: - displayname : The displayname text - -Getting/Setting your own avatar image URL ------------------------------------------ -The homeserver does not currently store the avatar image itself, but offers -storage for the user to specify a web URL that points at the required image, -leaving it up to clients to fetch it themselves. - REST Path: /profile/$user_id/avatar_url - Valid methods: GET/PUT - Required keys: - avatar_url : The URL path to the required image - -Getting other user's profile information ----------------------------------------- -Either of the above REST methods may be used to fetch other user's profile -information by the client, either on other local users on the same homeserver or -for users from other servers entirely. - - -Presence -======== - -In the following messages, the presence state is a presence string as described in -the main specification document. - -Getting/Setting your own presence state ---------------------------------------- - REST Path: /presence/$user_id/status - Valid methods: GET/PUT - Required keys: - presence : - The user's new presence state - Optional keys: - status_msg : text string provided by the user to explain their status - -Fetching your presence list ---------------------------- - REST Path: /presence_list/$user_id - Valid methods: GET/(post) - Returns: - An array of presence list entries. Each entry is an object with the - following keys: - { - "user_id" : string giving the observed user's ID - "presence" : int giving their status - "status_msg" : optional text string - "displayname" : optional text string from the user's profile - "avatar_url" : optional text string from the user's profile - } - -Maintaining your presence list ------------------------------- - REST Path: /presence_list/$user_id - Valid methods: POST/(get) - With: A JSON object optionally containing either of the following keys: - "invite" : a list of strings giving user IDs to invite for presence - subscription - "drop" : a list of strings giving user IDs to remove from your presence - list - -Receiving presence update events --------------------------------- - Event Type: m.presence - Keys of the event's content are the same as those returned by the presence - list. - -Examples -======== - -The following example is the story of "bob", who signs up at "sy.org" and joins -the public room "room_beta@sy.org". They get the 2 most recent -messages (with feedback) in that room and then send a message in that room. - -For context, here is the complete chat log for room_beta@sy.org: - -Room: "Hello world" (room_beta@sy.org) -Members: (2) alice@randomhost.org, friend_of_alice@randomhost.org -Messages: - alice@randomhost.org : hi friend! - [friend_of_alice@randomhost.org DELIVERED] - alice@randomhost.org : you're my only friend - [friend_of_alice@randomhost.org DELIVERED] - alice@randomhost.org : afk - [friend_of_alice@randomhost.org DELIVERED] - [ bob@sy.org joins ] - bob@sy.org : Hi everyone - [ alice@randomhost.org changes the topic to "FRIENDS ONLY" ] - alice@randomhost.org : Hello!!!! - alice@randomhost.org : Let's go to another room - alice@randomhost.org : You're not my friend - [ alice@randomhost.org invites bob@sy.org to the room - commoners@randomhost.org] - - -REGISTER FOR AN ACCOUNT -POST: /register -Content: {} -Returns: { "user_id" : "bob@sy.org" , "access_token" : "abcdef0123456789" } - -GET PUBLIC ROOM LIST -GET: /rooms/list?access_token=abcdef0123456789 -Returns: -{ - "total":3, - "chunk": - [ - { "room_id":"room_alpha@sy.org", "topic":"I am a fish" }, - { "room_id":"room_beta@sy.org", "topic":"Hello world" }, - { "room_id":"room_xyz@sy.org", "topic":"Goodbye cruel world" } - ] -} - -JOIN ROOM room_beta@sy.org -PUT -/rooms/room_beta%40sy.org/members/bob%40sy.org/state? - access_token=abcdef0123456789 -Content: { "membership" : "join" } -Returns: 200 OK - -GET LATEST 2 MESSAGES WITH FEEDBACK -GET -/rooms/room_beta%40sy.org/messages/list?from=END&to=START&limit=2& - feedback=true&access_token=abcdef0123456789 -Returns: -{ - "chunk": - [ - { - "event_id":"01948374", - "type":"m.room.message", - "room_id":"room_beta@sy.org", - "msg_id":"avefifu", - "from":"alice@randomhost.org", - "hs_ts":139985736, - "content":{ - "msgtype":"m.text", - "body":"afk" - } - "feedback": [ - { - "from":"friend_of_alice@randomhost.org", - "feedback":"d", - "hs_ts":139985850, - "content":{ - "sender_ts":139985843 - } - } - ] - }, - { - "event_id":"028dfe8373", - "type":"m.room.message", - "room_id":"room_beta@sy.org", - "msg_id":"afhgfff", - "from":"alice@randomhost.org", - "hs_ts":139970006, - "content":{ - "msgtype":"m.text", - "body":"you're my only friend" - } - "feedback": [ - { - "from":"friend_of_alice@randomhost.org", - "feedback":"d", - "hs_ts":139970144, - "content":{ - "sender_ts":139970122 - } - } - ] - }, - ], - "start": "stok_04823947", - "end": "etok_1426425" -} - -SEND MESSAGE IN ROOM -PUT -/rooms/room_beta%40sy.org/messages/bob%40sy.org/m0001? - access_token=abcdef0123456789 -Content: { "msgtype" : "text" , "body" : "Hi everyone" } -Returns: 200 OK - - -Checking the event stream for this user: -GET: /events?from=START&access_token=abcdef0123456789 -Returns: -{ - "chunk": - [ - { - "event_id":"e10f3d2b", - "type":"m.room.member", - "room_id":"room_beta@sy.org", - "user_id":"bob@sy.org", - "content":{ - "membership":"join" - } - }, - { - "event_id":"1b352d32", - "type":"m.room.message", - "room_id":"room_beta@sy.org", - "msg_id":"m0001", - "from":"bob@sy.org", - "hs_ts":140193857, - "content":{ - "msgtype":"m.text", - "body":"Hi everyone" - } - } - ], - "start": "stok_9348635", - "end": "etok_1984723" -} - -Client disconnects for a while and the topic is updated in this room, 3 new -messages arrive whilst offline, and bob is invited to another room. - -GET /events?from=etok_1984723&access_token=abcdef0123456789 -Returns: -{ - "chunk": - [ - { - "event_id":"feee0294", - "type":"m.room.topic", - "room_id":"room_beta@sy.org", - "from":"alice@randomhost.org", - "content":{ - "topic":"FRIENDS ONLY", - } - }, - { - "event_id":"a028bd9e", - "type":"m.room.message", - "room_id":"room_beta@sy.org", - "msg_id":"z839409", - "from":"alice@randomhost.org", - "hs_ts":140195000, - "content":{ - "msgtype":"m.text", - "body":"Hello!!!" - } - }, - { - "event_id":"49372d9e", - "type":"m.room.message", - "room_id":"room_beta@sy.org", - "msg_id":"z839410", - "from":"alice@randomhost.org", - "hs_ts":140196000, - "content":{ - "msgtype":"m.text", - "body":"Let's go to another room" - } - }, - { - "event_id":"10abdd01", - "type":"m.room.message", - "room_id":"room_beta@sy.org", - "msg_id":"z839411", - "from":"alice@randomhost.org", - "hs_ts":140197000, - "content":{ - "msgtype":"m.text", - "body":"You're not my friend" - } - }, - { - "event_id":"0018453d", - "type":"m.room.member", - "room_id":"commoners@randomhost.org", - "from":"alice@randomhost.org", - "user_id":"bob@sy.org", - "content":{ - "membership":"invite" - } - }, - ], - "start": "stok_0184288", - "end": "etok_1348723" -} diff --git a/docs/client-server/model/presence.rst b/docs/client-server/model/presence.rst deleted file mode 100644 index 811bac3fa..000000000 --- a/docs/client-server/model/presence.rst +++ /dev/null @@ -1,149 +0,0 @@ -API Efficiency -============== - -A simple implementation of presence messaging has the ability to cause a large -amount of Internet traffic relating to presence updates. In order to minimise -the impact of such a feature, the following observations can be made: - - * There is no point in a Home Server polling status for peers in a user's - presence list if the user has no clients connected that care about it. - - * It is highly likely that most presence subscriptions will be symmetric - a - given user watching another is likely to in turn be watched by that user. - - * It is likely that most subscription pairings will be between users who share - at least one Room in common, and so their Home Servers are actively - exchanging message PDUs or transactions relating to that Room. - - * Presence update messages do not need realtime guarantees. It is acceptable to - delay delivery of updates for some small amount of time (10 seconds to a - minute). - -The general model of presence information is that of a HS registering its -interest in receiving presence status updates from other HSes, which then -promise to send them when required. Rather than actively polling for the -currentt state all the time, HSes can rely on their relative stability to only -push updates when required. - -A Home Server should not rely on the longterm validity of this presence -information, however, as this would not cover such cases as a user's server -crashing and thus failing to inform their peers that users it used to host are -no longer available online. Therefore, each promise of future updates should -carry with a timeout value (whether explicit in the message, or implicit as some -defined default in the protocol), after which the receiving HS should consider -the information potentially stale and request it again. - -However, because of the likelyhood that two home servers are exchanging messages -relating to chat traffic in a room common to both of them, the ongoing receipt -of these messages can be taken by each server as an implicit notification that -the sending server is still up and running, and therefore that no status changes -have happened; because if they had the server would have sent them. A second, -larger timeout should be applied to this implicit inference however, to protect -against implementation bugs or other reasons that the presence state cache may -become invalid; eventually the HS should re-enquire the current state of users -and update them with its own. - -The following workflows can therefore be used to handle presence updates: - - 1 When a user first appears online their HS sends a message to each other HS - containing at least one user to be watched; each message carrying both a - notification of the sender's new online status, and a request to obtain and - watch the target users' presence information. This message implicitly - promises the sending HS will now push updates to the target HSes. - - 2 The target HSes then respond a single message each, containing the current - status of the requested user(s). These messages too implicitly promise the - target HSes will themselves push updates to the sending HS. - - As these messages arrive at the sending user's HS they can be pushed to the - user's client(s), possibly batched again to ensure not too many small - messages which add extra protocol overheads. - -At this point, all the user's clients now have the current presence status -information for this moment in time, and have promised to send each other -updates in future. - - 3 The HS maintains two watchdog timers per peer HS it is exchanging presence - information with. The first timer should have a relatively small expiry - (perhaps 1 minute), and the second timer should have a much longer time - (perhaps 1 hour). - - 4 Any time any kind of message is received from a peer HS, the short-term - presence timer associated with it is reset. - - 5 Whenever either of these timers expires, an HS should push a status reminder - to the target HS whose timer has now expired, and request again from that - server the status of the subscribed users. - - 6 On receipt of one of these presence status reminders, an HS can reset both - of its presence watchdog timers. - -To avoid bursts of traffic, implementations should attempt to stagger the expiry -of the longer-term watchdog timers for different peer HSes. - -When individual users actively change their status (either by explicit requests -from clients, or inferred changes due to idle timers or client timeouts), the HS -should batch up any status changes for some reasonable amount of time (10 -seconds to a minute). This allows for reduced protocol overheads in the case of -multiple messages needing to be sent to the same peer HS; as is the likely -scenario in many cases, such as a given human user having multiple user -accounts. - - -API Requirements -================ - -The data model presented here puts the following requirements on the APIs: - -Client-Server -------------- - -Requests that a client can make to its Home Server - - * get/set current presence state - Basic enumeration + ability to set a custom piece of text - - * report per-device idle time - After some (configurable?) idle time the device should send a single message - to set the idle duration. The HS can then infer a "start of idle" instant and - use that to keep the device idleness up to date. At some later point the - device can cancel this idleness. - - * report per-device type - Inform the server that this device is a "mobile" device, or perhaps some - other to-be-defined category of reduced capability that could be presented to - other users. - - * start/stop presence polling for my presence list - It is likely that these messages could be implicitly inferred by other - messages, though having explicit control is always useful. - - * get my presence list - [implicit poll start?] - It is possible that the HS doesn't yet have current presence information when - the client requests this. There should be a "don't know" type too. - - * add/remove a user to my presence list - -Server-Server -------------- - -Requests that Home Servers make to others - - * request permission to add a user to presence list - - * allow/deny a request to add to a presence list - - * perform a combined presence state push and subscription request - For each sending user ID, the message contains their new status. - For each receiving user ID, the message should contain an indication on - whether the sending server is also interested in receiving status from that - user; either as an immediate update response now, or as a promise to send - future updates. - -Server to Client ----------------- - -[[TODO(paul): There also needs to be some way for a user's HS to push status -updates of the presence list to clients, but the general server-client event -model currently lacks a space to do that.]] diff --git a/docs/client-server/model/profiles.rst b/docs/client-server/model/profiles.rst deleted file mode 100644 index f7d6bd567..000000000 --- a/docs/client-server/model/profiles.rst +++ /dev/null @@ -1,232 +0,0 @@ -======== -Profiles -======== - -A description of Synapse user profile metadata support. - - -Overview -======== - -Internally within Synapse users are referred to by an opaque ID, which consists -of some opaque localpart combined with the domain name of their home server. -Obviously this does not yield a very nice user experience; users would like to -see readable names for other users that are in some way meaningful to them. -Additionally, users like to be able to publish "profile" details to inform other -users of other information about them. - -It is also conceivable that since we are attempting to provide a -worldwide-applicable messaging system, that users may wish to present different -subsets of information in their profile to different other people, from a -privacy and permissions perspective. - -A Profile consists of a display name, an (optional?) avatar picture, and a set -of other metadata fields that the user may wish to publish (email address, phone -numbers, website URLs, etc...). We put no requirements on the display name other -than it being a valid Unicode string. Since it is likely that users will end up -having multiple accounts (perhaps by necessity of being hosted in multiple -places, perhaps by choice of wanting multiple distinct identifies), it would be -useful that a metadata field type exists that can refer to another Synapse User -ID, so that clients and HSes can make use of this information. - -Metadata Fields ---------------- - -[[TODO(paul): Likely this list is incomplete; more fields can be defined as we -think of them. At the very least, any sort of supported ID for the 3rd Party ID -servers should be accounted for here.]] - - * Synapse Directory Server username(s) - - * Email address - - * Phone number - classify "home"/"work"/"mobile"/custom? - - * Twitter/Facebook/Google+/... social networks - - * Location - keep this deliberately vague to allow people to choose how - granular it is - - * "Bio" information - date of birth, etc... - - * Synapse User ID of another account - - * Web URL - - * Freeform description text - - -Visibility Permissions -====================== - -A home server implementation could offer the ability to set permissions on -limited visibility of those fields. When another user requests access to the -target user's profile, their own identity should form part of that request. The -HS implementation can then decide which fields to make available to the -requestor. - -A particular detail of implementation could allow the user to create one or more -ACLs; where each list is granted permission to see a given set of non-public -fields (compare to Google+ Circles) and contains a set of other people allowed -to use it. By giving these ACLs strong identities within the HS, they can be -referenced in communications with it, granting other users who encounter these -the "ACL Token" to use the details in that ACL. - -If we further allow an ACL Token to be present on Room join requests or stored -by 3PID servers, then users of these ACLs gain the extra convenience of not -having to manually curate people in the access list; anyone in the room or with -knowledge of the 3rd Party ID is automatically granted access. Every HS and -client implementation would have to be aware of the existence of these ACL -Token, and include them in requests if present, but not every HS implementation -needs to actually provide the full permissions model. This can be used as a -distinguishing feature among competing implementations. However, servers MUST -NOT serve profile information from a cache if there is a chance that its limited -understanding could lead to information leakage. - - -Client Concerns of Multiple Accounts -==================================== - -Because a given person may want to have multiple Synapse User accounts, client -implementations should allow the use of multiple accounts simultaneously -(especially in the field of mobile phone clients, which generally don't support -running distinct instances of the same application). Where features like address -books, presence lists or rooms are presented, the client UI should remember to -make distinct with user account is in use for each. - - -Directory Servers -================= - -Directory Servers can provide a forward mapping from human-readable names to -User IDs. These can provide a service similar to giving domain-namespaced names -for Rooms; in this case they can provide a way for a user to reference their -User ID in some external form (e.g. that can be printed on a business card). - -The format for Synapse user name will consist of a localpart specific to the -directory server, and the domain name of that directory server: - - @localname:some.domain.name - -The localname is separated from the domain name using a colon, so as to ensure -the localname can still contain periods, as users may want this for similarity -to email addresses or the like, which typically can contain them. The format is -also visually quite distinct from email addresses, phone numbers, etc... so -hopefully reasonably "self-describing" when written on e.g. a business card -without surrounding context. - -[[TODO(paul): we might have to think about this one - too close to email? - Twitter? Also it suggests a format scheme for room names of - #localname:domain.name, which I quite like]] - -Directory server administrators should be able to make some kind of policy -decision on how these are allocated. Servers within some "closed" domain (such -as company-specific ones) may wish to verify the validity of a mapping using -their own internal mechanisms; "public" naming servers can operate on a FCFS -basis. There are overlapping concerns here with the idea of the 3rd party -identity servers as well, though in this specific case we are creating a new -namespace to allocate names into. - -It would also be nice from a user experience perspective if the profile that a -given name links to can also declare that name as part of its metadata. -Furthermore as a security and consistency perspective it would be nice if each -end (the directory server and the user's home server) check the validity of the -mapping in some way. This needs investigation from a security perspective to -ensure against spoofing. - -One such model may be that the user starts by declaring their intent to use a -given user name link to their home server, which then contacts the directory -service. At some point later (maybe immediately for "public open FCFS servers", -maybe after some kind of human intervention for verification) the DS decides to -honour this link, and includes it in its served output. It should also tell the -HS of this fact, so that the HS can present this as fact when requested for the -profile information. For efficiency, it may further wish to provide the HS with -a cryptographically-signed certificate as proof, so the HS serving the profile -can provide that too when asked, avoiding requesting HSes from constantly having -to contact the DS to verify this mapping. (Note: This is similar to the security -model often applied in DNS to verify PTR <-> A bidirectional mappings). - - -Identity Servers -================ - -The identity servers should support the concept of pointing a 3PID being able to -store an ACL Token as well as the main User ID. It is however, beyond scope to -do any kind of verification that any third-party IDs that the profile is -claiming match up to the 3PID mappings. - - -User Interface and Expectations Concerns -======================================== - -Given the weak "security" of some parts of this model as compared to what users -might expect, some care should be taken on how it is presented to users, -specifically in the naming or other wording of user interface components. - -Most notably mere knowledge of an ACL Pointer is enough to read the information -stored in it. It is possible that Home or Identity Servers could leak this -information, allowing others to see it. This is a security-vs-convenience -balancing choice on behalf of the user who would choose, or not, to make use of -such a feature to publish their information. - -Additionally, unless some form of strong end-to-end user-based encryption is -used, a user of ACLs for information privacy has to trust other home servers not -to lie about the identify of the user requesting access to the Profile. - - -API Requirements -================ - -The data model presented here puts the following requirements on the APIs: - -Client-Server -------------- - -Requests that a client can make to its Home Server - - * get/set my Display Name - This should return/take a simple "text/plain" field - - * get/set my Avatar URL - The avatar image data itself is not stored by this API; we'll just store a - URL to let the clients fetch it. Optionally HSes could integrate this with - their generic content attacmhent storage service, allowing a user to set - upload their profile Avatar and update the URL to point to it. - - * get/add/remove my metadata fields - Also we need to actually define types of metadata - - * get another user's Display Name / Avatar / metadata fields - -[[TODO(paul): At some later stage we should consider the API for: - - * get/set ACL permissions on my metadata fields - - * manage my ACL tokens -]] - -Server-Server -------------- - -Requests that Home Servers make to others - - * get a user's Display Name / Avatar - - * get a user's full profile - name/avatar + MD fields - This request must allow for specifying the User ID of the requesting user, - for permissions purposes. It also needs to take into account any ACL Tokens - the requestor has. - - * push a change of Display Name to observers (overlaps with the presence API) - -Room Event PDU Types --------------------- - -Events that are pushed from Home Servers to other Home Servers or clients. - - * user Display Name change - - * user Avatar change - [[TODO(paul): should the avatar image itself be stored in all the room - histories? maybe this event should just be a hint to clients that they should - re-fetch the avatar image]] diff --git a/docs/client-server/model/protocol_examples.rst b/docs/client-server/model/protocol_examples.rst deleted file mode 100644 index 61a599b43..000000000 --- a/docs/client-server/model/protocol_examples.rst +++ /dev/null @@ -1,64 +0,0 @@ -PUT /send/abc/ HTTP/1.1 -Host: ... -Content-Length: ... -Content-Type: application/json - -{ - "origin": "localhost:5000", - "pdus": [ - { - "content": {}, - "context": "tng", - "depth": 12, - "is_state": false, - "origin": "localhost:5000", - "pdu_id": 1404381396854, - "pdu_type": "feedback", - "prev_pdus": [ - [ - "1404381395883", - "localhost:6000" - ] - ], - "ts": 1404381427581 - } - ], - "prev_ids": [ - "1404381396852" - ], - "ts": 1404381427823 -} - -HTTP/1.1 200 OK -... - -====================================== - -GET /pull/-1/ HTTP/1.1 -Host: ... -Content-Length: 0 - -HTTP/1.1 200 OK -Content-Length: ... -Content-Type: application/json - -{ - origin: ..., - prev_ids: ..., - data: [ - { - data_id: ..., - prev_pdus: [...], - depth: ..., - ts: ..., - context: ..., - origin: ..., - content: { - ... - } - }, - ..., - ] -} - - diff --git a/docs/client-server/model/room-join-workflow.rst b/docs/client-server/model/room-join-workflow.rst deleted file mode 100644 index c321a64fa..000000000 --- a/docs/client-server/model/room-join-workflow.rst +++ /dev/null @@ -1,113 +0,0 @@ -================== -Room Join Workflow -================== - -An outline of the workflows required when a user joins a room. - -Discovery -========= - -To join a room, a user has to discover the room by some mechanism in order to -obtain the (opaque) Room ID and a candidate list of likely home servers that -contain it. - -Sending an Invitation ---------------------- - -The most direct way a user discovers the existence of a room is from a -invitation from some other user who is a member of that room. - -The inviter's HS sets the membership status of the invitee to "invited" in the -"m.members" state key by sending a state update PDU. The HS then broadcasts this -PDU among the existing members in the usual way. An invitation message is also -sent to the invited user, containing the Room ID and the PDU ID of this -invitation state change and potentially a list of some other home servers to use -to accept the invite. The user's client can then choose to display it in some -way to alert the user. - -[[TODO(paul): At present, no API has been designed or described to actually send -that invite to the invited user. Likely it will be some facet of the larger -user-user API required for presence, profile management, etc...]] - -Directory Service ------------------ - -Alternatively, the user may discover the channel via a directory service; either -by performing a name lookup, or some kind of browse or search acitivty. However -this is performed, the end result is that the user's home server requests the -Room ID and candidate list from the directory service. - -[[TODO(paul): At present, no API has been designed or described for this -directory service]] - - -Joining -======= - -Once the ID and home servers are obtained, the user can then actually join the -room. - -Accepting an Invite -------------------- - -If a user has received and accepted an invitation to join a room, the invitee's -home server can now send an invite acceptance message to a chosen candidate -server from the list given in the invitation, citing also the PDU ID of the -invitation as "proof" of their invite. (This is required as due to late message -propagation it could be the case that the acceptance is received before the -invite by some servers). If this message is allowed by the candidate server, it -generates a new PDU that updates the invitee's membership status to "joined", -referring back to the acceptance PDU, and broadcasts that as a state change in -the usual way. The newly-invited user is now a full member of the room, and -state propagation proceeds as usual. - -Joining a Public Room ---------------------- - -If a user has discovered the existence of a room they wish to join but does not -have an active invitation, they can request to join it directly by sending a -join message to a candidate server on the list provided by the directory -service. As this list may be out of date, the HS should be prepared to retry -other candidates if the chosen one is no longer aware of the room, because it -has no users as members in it. - -Once a candidate server that is aware of the room has been found, it can -broadcast an update PDU to add the member into the "m.members" key setting their -state directly to "joined" (i.e. bypassing the two-phase invite semantics), -remembering to include the new user's HS in that list. - -Knocking on a Semi-Public Room ------------------------------- - -If a user requests to join a room but the join mode of the room is "knock", the -join is not immediately allowed. Instead, if the user wishes to proceed, they -can instead post a "knock" message, which informs other members of the room that -the would-be joiner wishes to become a member and sets their membership value to -"knocked". If any of them wish to accept this, they can then send an invitation -in the usual way described above. Knowing that the user has already knocked and -expressed an interest in joining, the invited user's home server should -immediately accept that invitation on the user's behalf, and go on to join the -room in the usual way. - -[[NOTE(Erik): Though this may confuse users who expect 'X has joined' to -actually be a user initiated action, i.e. they may expect that 'X' is actually -looking at synapse right now?]] - -[[NOTE(paul): Yes, a fair point maybe we should suggest HSes don't do that, and -just offer an invite to the user as normal]] - -Private and Non-Existent Rooms ------------------------------- - -If a user requests to join a room but the room is either unknown by the home -server receiving the request, or is known by the join mode is "invite" and the -user has not been invited, the server must respond that the room does not exist. -This is to prevent leaking information about the existence and identity of -private rooms. - - -Outstanding Questions -===================== - - * Do invitations or knocks time out and expire at some point? If so when? Time - is hard in distributed systems. diff --git a/docs/client-server/model/rooms.rst b/docs/client-server/model/rooms.rst deleted file mode 100644 index 0007e48e3..000000000 --- a/docs/client-server/model/rooms.rst +++ /dev/null @@ -1,274 +0,0 @@ -=========== -Rooms Model -=========== - -A description of the general data model used to implement Rooms, and the -user-level visible effects and implications. - - -Overview -======== - -"Rooms" in Synapse are shared messaging channels over which all the participant -users can exchange messages. Rooms have an opaque persistent identify, a -globally-replicated set of state (consisting principly of a membership set of -users, and other management and miscellaneous metadata), and a message history. - - -Room Identity and Naming -======================== - -Rooms can be arbitrarily created by any user on any home server; at which point -the home server will sign the message that creates the channel, and the -fingerprint of this signature becomes the strong persistent identify of the -room. This now identifies the room to any home server in the network regardless -of its original origin. This allows the identify of the room to outlive any -particular server. Subject to appropriate permissions [to be discussed later], -any current member of a room can invite others to join it, can post messages -that become part of its history, and can change the persistent state of the room -(including its current set of permissions). - -Home servers can provide a directory service, allowing a lookup from a -convenient human-readable form of room label to a room ID. This mapping is -scoped to the particular home server domain and so simply represents that server -administrator's opinion of what room should take that label; it does not have to -be globally replicated and does not form part of the stored state of that room. - -This room name takes the form - - #localname:some.domain.name - -for similarity and consistency with user names on directories. - -To join a room (and therefore to be allowed to inspect past history, post new -messages to it, and read its state), a user must become aware of the room's -fingerprint ID. There are two mechanisms to allow this: - - * An invite message from someone else in the room - - * A referral from a room directory service - -As room IDs are opaque and ephemeral, they can serve as a mechanism to create -"ad-hoc" rooms deliberately unnamed, for small group-chats or even private -one-to-one message exchange. - - -Stored State and Permissions -============================ - -Every room has a globally-replicated set of stored state. This state is a set of -key/value or key/subkey/value pairs. The value of every (sub)key is a -JSON-representable object. The main key of a piece of stored state establishes -its meaning; some keys store sub-keys to allow a sub-structure within them [more -detail below]. Some keys have special meaning to Synapse, as they relate to -management details of the room itself, storing such details as user membership, -and permissions of users to alter the state of the room itself. Other keys may -store information to present to users, which the system does not directly rely -on. The key space itself is namespaced, allowing 3rd party extensions, subject -to suitable permission. - -Permission management is based on the concept of "power-levels". Every user -within a room has an integer assigned, being their "power-level" within that -room. Along with its actual data value, each key (or subkey) also stores the -minimum power-level a user must have in order to write to that key, the -power-level of the last user who actually did write to it, and the PDU ID of -that state change. - -To be accepted as valid, a change must NOT: - - * Be made by a user having a power-level lower than required to write to the - state key - - * Alter the required power-level for that state key to a value higher than the - user has - - * Increase that user's own power-level - - * Grant any other user a power-level higher than the level of the user making - the change - -[[TODO(paul): consider if relaxations should be allowed; e.g. is the current -outright-winner allowed to raise their own level, to allow for "inflation"?]] - - -Room State Keys -=============== - -[[TODO(paul): if this list gets too big it might become necessary to move it -into its own doc]] - -The following keys have special semantics or meaning to Synapse itself: - -m.member (has subkeys) - Stores a sub-key for every Synapse User ID which is currently a member of - this room. Its value gives the membership type ("knocked", "invited", - "joined"). - -m.power_levels - Stores a mapping from Synapse User IDs to their power-level in the room. If - they are not present in this mapping, the default applies. - - The reason to store this as a single value rather than a value with subkeys - is that updates to it are atomic; allowing a number of colliding-edit - problems to be avoided. - -m.default_level - Gives the default power-level for members of the room that do not have one - specified in their membership key. - -m.invite_level - If set, gives the minimum power-level required for members to invite others - to join, or to accept knock requests from non-members requesting access. If - absent, then invites are not allowed. An invitation involves setting their - membership type to "invited", in addition to sending the invite message. - -m.join_rules - Encodes the rules on how non-members can join the room. Has the following - possibilities: - "public" - a non-member can join the room directly - "knock" - a non-member cannot join the room, but can post a single "knock" - message requesting access, which existing members may approve or deny - "invite" - non-members cannot join the room without an invite from an - existing member - "private" - nobody who is not in the 'may_join' list or already a member - may join by any mechanism - - In any of the first three modes, existing members with sufficient permission - can send invites to non-members if allowed by the "m.invite_level" key. A - "private" room is not allowed to have the "m.invite_level" set. - - A client may use the value of this key to hint at the user interface - expectations to provide; in particular, a private chat with one other use - might warrant specific handling in the client. - -m.may_join - A list of User IDs that are always allowed to join the room, regardless of any - of the prevailing join rules and invite levels. These apply even to private - rooms. These are stored in a single list with normal update-powerlevel - permissions applied; users cannot arbitrarily remove themselves from the list. - -m.add_state_level - The power-level required for a user to be able to add new state keys. - -m.public_history - If set and true, anyone can request the history of the room, without needing - to be a member of the room. - -m.archive_servers - For "public" rooms with public history, gives a list of home servers that - should be included in message distribution to the room, even if no users on - that server are present. These ensure that a public room can still persist - even if no users are currently members of it. This list should be consulted by - the dirctory servers as the candidate list they respond with. - -The following keys are provided by Synapse for user benefit, but their value is -not otherwise used by Synapse. - -m.name - Stores a short human-readable name for the room, such that clients can display - to a user to assist in identifying which room is which. - - This name specifically is not the strong ID used by the message transport - system to refer to the room, because it may be changed from time to time. - -m.topic - Stores the current human-readable topic - - -Room Creation Templates -======================= - -A client (or maybe home server?) could offer a few templates for the creation of -new rooms. For example, for a simple private one-to-one chat the channel could -assign the creator a power-level of 1, requiring a level of 1 to invite, and -needing an invite before members can join. An invite is then sent to the other -party, and if accepted and the other user joins, the creator's power-level can -now be reduced to 0. This now leaves a room with two participants in it being -unable to add more. - - -Rooms that Continue History -=========================== - -An option that could be considered for room creation, is that when a new room is -created the creator could specify a PDU ID into an existing room, as the history -continuation point. This would be stored as an extra piece of meta-data on the -initial PDU of the room's creation. (It does not appear in the normal previous -PDU linkage). - -This would allow users in rooms to "fork" a room, if it is considered that the -conversations in the room no longer fit its original purpose, and wish to -diverge. Existing permissions on the original room would continue to apply of -course, for viewing that history. If both rooms are considered "public" we might -also want to define a message to post into the original room to represent this -fork point, and give a reference to the new room. - - -User Direct Message Rooms -========================= - -There is no need to build a mechanism for directly sending messages between -users, because a room can handle this ability. To allow direct user-to-user chat -messaging we simply need to be able to create rooms with specific set of -permissions to allow this direct messaging. - -Between any given pair of user IDs that wish to exchange private messages, there -will exist a single shared Room, created lazily by either side. These rooms will -need a certain amount of special handling in both home servers and display on -clients, but as much as possible should be treated by the lower layers of code -the same as other rooms. - -Specially, a client would likely offer a special menu choice associated with -another user (in room member lists, presence list, etc..) as "direct chat". That -would perform all the necessary steps to create the private chat room. Receiving -clients should display these in a special way too as the room name is not -important; instead it should distinguish them on the Display Name of the other -party. - -Home Servers will need a client-API option to request setting up a new user-user -chat room, which will then need special handling within the server. It will -create a new room with the following - - m.member: the proposing user - m.join_rules: "private" - m.may_join: both users - m.power_levels: empty - m.default_level: 0 - m.add_state_level: 0 - m.public_history: False - -Having created the room, it can send an invite message to the other user in the -normal way - the room permissions state that no users can be set to the invited -state, but because they're in the may_join list then they'd be allowed to join -anyway. - -In this arrangement there is now a room with both users may join but neither has -the power to invite any others. Both users now have the confidence that (at -least within the messaging system itself) their messages remain private and -cannot later be provably leaked to a third party. They can freely set the topic -or name if they choose and add or edit any other state of the room. The update -powerlevel of each of these fixed properties should be 1, to lock out the users -from being able to alter them. - - -Anti-Glare -========== - -There exists the possibility of a race condition if two users who have no chat -history with each other simultaneously create a room and invite the other to it. -This is called a "glare" situation. There are two possible ideas for how to -resolve this: - - * Each Home Server should persist the mapping of (user ID pair) to room ID, so - that duplicate requests can be suppressed. On receipt of a room creation - request that the HS thinks there already exists a room for, the invitation to - join can be rejected if: - a) the HS believes the sending user is already a member of the room (and - maybe their HS has forgotten this fact), or - b) the proposed room has a lexicographically-higher ID than the existing - room (to resolve true race condition conflicts) - - * The room ID for a private 1:1 chat has a special form, determined by - concatenting the User IDs of both members in a deterministic order, such that - it doesn't matter which side creates it first; the HSes can just ignore - (or merge?) received PDUs that create the room twice. diff --git a/docs/client-server/model/terminology.rst b/docs/client-server/model/terminology.rst deleted file mode 100644 index cc6e6760a..000000000 --- a/docs/client-server/model/terminology.rst +++ /dev/null @@ -1,86 +0,0 @@ -=========== -Terminology -=========== - -A list of definitions of specific terminology used among these documents. -These terms were originally taken from the server-server documentation, and may -not currently match the exact meanings used in other places; though as a -medium-term goal we should encourage the unification of this terminology. - - -Terms -===== - -Backfilling: - The process of synchronising historic state from one home server to another, - to backfill the event storage so that scrollback can be presented to the - client(s). (Formerly, and confusingly, called 'pagination') - -Context: - A single human-level entity of interest (currently, a chat room) - -EDU (Ephemeral Data Unit): - A message that relates directly to a given pair of home servers that are - exchanging it. EDUs are short-lived messages that related only to one single - pair of servers; they are not persisted for a long time and are not forwarded - on to other servers. Because of this, they have no internal ID nor previous - EDUs reference chain. - -Event: - A record of activity that records a single thing that happened on to a context - (currently, a chat room). These are the "chat messages" that Synapse makes - available. - [[NOTE(paul): The current server-server implementation calls these simply - "messages" but the term is too ambiguous here; I've called them Events]] - -PDU (Persistent Data Unit): - A message that relates to a single context, irrespective of the server that - is communicating it. PDUs either encode a single Event, or a single State - change. A PDU is referred to by its PDU ID; the pair of its origin server - and local reference from that server. - -PDU ID: - The pair of PDU Origin and PDU Reference, that together globally uniquely - refers to a specific PDU. - -PDU Origin: - The name of the origin server that generated a given PDU. This may not be the - server from which it has been received, due to the way they are copied around - from server to server. The origin always records the original server that - created it. - -PDU Reference: - A local ID used to refer to a specific PDU from a given origin server. These - references are opaque at the protocol level, but may optionally have some - structured meaning within a given origin server or implementation. - -Presence: - The concept of whether a user is currently online, how available they declare - they are, and so on. See also: doc/model/presence - -Profile: - A set of metadata about a user, such as a display name, provided for the - benefit of other users. See also: doc/model/profiles - -Room ID: - An opaque string (of as-yet undecided format) that identifies a particular - room and used in PDUs referring to it. - -Room Alias: - A human-readable string of the form #name:some.domain that users can use as a - pointer to identify a room; a Directory Server will map this to its Room ID - -State: - A set of metadata maintained about a Context, which is replicated among the - servers in addition to the history of Events. - -User ID: - A string of the form @localpart:domain.name that identifies a user for - wire-protocol purposes. The localpart is meaningless outside of a particular - home server. This takes a human-readable form that end-users can use directly - if they so wish, avoiding the 3PIDs. - -Transaction: - A message which relates to the communication between a given pair of servers. - A transaction contains possibly-empty lists of PDUs and EDUs. - diff --git a/docs/client-server/model/third-party-id.rst b/docs/client-server/model/third-party-id.rst deleted file mode 100644 index 1f8138ddf..000000000 --- a/docs/client-server/model/third-party-id.rst +++ /dev/null @@ -1,108 +0,0 @@ -====================== -Third Party Identities -====================== - -A description of how email addresses, mobile phone numbers and other third -party identifiers can be used to authenticate and discover users in Matrix. - - -Overview -======== - -New users need to authenticate their account. An email or SMS text message can -be a convenient form of authentication. Users already have email addresses -and phone numbers for contacts in their address book. They want to communicate -with those contacts in Matrix without manually exchanging a Matrix User ID with -them. - -Third Party IDs ---------------- - -[[TODO(markjh): Describe the format of a 3PID]] - - -Third Party ID Associations ---------------------------- - -An Associaton is a binding between a Matrix User ID and a Third Party ID (3PID). -Each 3PID can be associated with one Matrix User ID at a time. - -[[TODO(markjh): JSON format of the association.]] - -Verification ------------- - -An Assocation must be verified by a trusted Verification Server. Email -addresses and phone numbers can be verified by sending a token to the address -which a client can supply to the verifier to confirm ownership. - -An email Verification Server may be capable of verifying all email 3PIDs or may -be restricted to verifying addresses for a particular domain. A phone number -Verification Server may be capable of verifying all phone numbers or may be -restricted to verifying numbers for a given country or phone prefix. - -Verification Servers fulfil a similar role to Certificate Authorities in PKI so -a similar level of vetting should be required before clients trust their -signatures. - -A Verification Server may wish to check for existing Associations for a 3PID -before creating a new Association. - -Discovery ---------- - -Users can discover Associations using a trusted Identity Server. Each -Association will be signed by the Identity Server. An Identity Server may store -the entire space of Associations or may delegate to other Identity Servers when -looking up Associations. - -Each Association returned from an Identity Server must be signed by a -Verification Server. Clients should check these signatures. - -Identity Servers fulfil a similar role to DNS servers. - -Privacy -------- - -A User may publish the association between their phone number and Matrix User ID -on the Identity Server without publishing the number in their Profile hosted on -their Home Server. - -Identity Servers should refrain from publishing reverse mappings and should -take steps, such as rate limiting, to prevent attackers enumerating the space of -mappings. - -Federation -========== - -Delegation ----------- - -Verification Servers could delegate signing to another server by issuing -certificate to that server allowing it to verify and sign a subset of 3PID on -its behalf. It would be necessary to provide a language for describing which -subset of 3PIDs that server had authority to validate. Alternatively it could -delegate the verification step to another server but sign the resulting -association itself. - -The 3PID space will have a heirachical structure like DNS so Identity Servers -can delegate lookups to other servers. An Identity Server should be prepared -to host or delegate any valid association within the subset of the 3PIDs it is -resonsible for. - -Multiple Root Verification Servers ----------------------------------- - -There can be multiple root Verification Servers and an Association could be -signed by multiple servers if different clients trust different subsets of -the verification servers. - -Multiple Root Identity Servers ------------------------------- - -There can be be multiple root Identity Servers. Clients will add each -Association to all root Identity Servers. - -[[TODO(markjh): Describe how clients find the list of root Identity Servers]] - - diff --git a/docs/client-server/web/README b/docs/client-server/web/README deleted file mode 100644 index 315d5794b..000000000 --- a/docs/client-server/web/README +++ /dev/null @@ -1,5 +0,0 @@ -To get this running: - ln -s ../swagger_matrix - python -m SimpleHTTPServer - -Go to http://localhost:8000/swagger.html diff --git a/docs/client-server/web/files/backbone-min.js b/docs/client-server/web/files/backbone-min.js deleted file mode 100644 index c1c0d4fff..000000000 --- a/docs/client-server/web/files/backbone-min.js +++ /dev/null @@ -1,38 +0,0 @@ -// Backbone.js 0.9.2 - -// (c) 2010-2012 Jeremy Ashkenas, DocumentCloud Inc. -// Backbone may be freely distributed under the MIT license. -// For all details and documentation: -// http://backbonejs.org -(function(){var l=this,y=l.Backbone,z=Array.prototype.slice,A=Array.prototype.splice,g;g="undefined"!==typeof exports?exports:l.Backbone={};g.VERSION="0.9.2";var f=l._;!f&&"undefined"!==typeof require&&(f=require("underscore"));var i=l.jQuery||l.Zepto||l.ender;g.setDomLibrary=function(a){i=a};g.noConflict=function(){l.Backbone=y;return this};g.emulateHTTP=!1;g.emulateJSON=!1;var p=/\s+/,k=g.Events={on:function(a,b,c){var d,e,f,g,j;if(!b)return this;a=a.split(p);for(d=this._callbacks||(this._callbacks= -{});e=a.shift();)f=(j=d[e])?j.tail:{},f.next=g={},f.context=c,f.callback=b,d[e]={tail:g,next:j?j.next:f};return this},off:function(a,b,c){var d,e,h,g,j,q;if(e=this._callbacks){if(!a&&!b&&!c)return delete this._callbacks,this;for(a=a?a.split(p):f.keys(e);d=a.shift();)if(h=e[d],delete e[d],h&&(b||c))for(g=h.tail;(h=h.next)!==g;)if(j=h.callback,q=h.context,b&&j!==b||c&&q!==c)this.on(d,j,q);return this}},trigger:function(a){var b,c,d,e,f,g;if(!(d=this._callbacks))return this;f=d.all;a=a.split(p);for(g= -z.call(arguments,1);b=a.shift();){if(c=d[b])for(e=c.tail;(c=c.next)!==e;)c.callback.apply(c.context||this,g);if(c=f){e=c.tail;for(b=[b].concat(g);(c=c.next)!==e;)c.callback.apply(c.context||this,b)}}return this}};k.bind=k.on;k.unbind=k.off;var o=g.Model=function(a,b){var c;a||(a={});b&&b.parse&&(a=this.parse(a));if(c=n(this,"defaults"))a=f.extend({},c,a);b&&b.collection&&(this.collection=b.collection);this.attributes={};this._escapedAttributes={};this.cid=f.uniqueId("c");this.changed={};this._silent= -{};this._pending={};this.set(a,{silent:!0});this.changed={};this._silent={};this._pending={};this._previousAttributes=f.clone(this.attributes);this.initialize.apply(this,arguments)};f.extend(o.prototype,k,{changed:null,_silent:null,_pending:null,idAttribute:"id",initialize:function(){},toJSON:function(){return f.clone(this.attributes)},get:function(a){return this.attributes[a]},escape:function(a){var b;if(b=this._escapedAttributes[a])return b;b=this.get(a);return this._escapedAttributes[a]=f.escape(null== -b?"":""+b)},has:function(a){return null!=this.get(a)},set:function(a,b,c){var d,e;f.isObject(a)||null==a?(d=a,c=b):(d={},d[a]=b);c||(c={});if(!d)return this;d instanceof o&&(d=d.attributes);if(c.unset)for(e in d)d[e]=void 0;if(!this._validate(d,c))return!1;this.idAttribute in d&&(this.id=d[this.idAttribute]);var b=c.changes={},h=this.attributes,g=this._escapedAttributes,j=this._previousAttributes||{};for(e in d){a=d[e];if(!f.isEqual(h[e],a)||c.unset&&f.has(h,e))delete g[e],(c.silent?this._silent: -b)[e]=!0;c.unset?delete h[e]:h[e]=a;!f.isEqual(j[e],a)||f.has(h,e)!=f.has(j,e)?(this.changed[e]=a,c.silent||(this._pending[e]=!0)):(delete this.changed[e],delete this._pending[e])}c.silent||this.change(c);return this},unset:function(a,b){(b||(b={})).unset=!0;return this.set(a,null,b)},clear:function(a){(a||(a={})).unset=!0;return this.set(f.clone(this.attributes),a)},fetch:function(a){var a=a?f.clone(a):{},b=this,c=a.success;a.success=function(d,e,f){if(!b.set(b.parse(d,f),a))return!1;c&&c(b,d)}; -a.error=g.wrapError(a.error,b,a);return(this.sync||g.sync).call(this,"read",this,a)},save:function(a,b,c){var d,e;f.isObject(a)||null==a?(d=a,c=b):(d={},d[a]=b);c=c?f.clone(c):{};if(c.wait){if(!this._validate(d,c))return!1;e=f.clone(this.attributes)}a=f.extend({},c,{silent:!0});if(d&&!this.set(d,c.wait?a:c))return!1;var h=this,i=c.success;c.success=function(a,b,e){b=h.parse(a,e);if(c.wait){delete c.wait;b=f.extend(d||{},b)}if(!h.set(b,c))return false;i?i(h,a):h.trigger("sync",h,a,c)};c.error=g.wrapError(c.error, -h,c);b=this.isNew()?"create":"update";b=(this.sync||g.sync).call(this,b,this,c);c.wait&&this.set(e,a);return b},destroy:function(a){var a=a?f.clone(a):{},b=this,c=a.success,d=function(){b.trigger("destroy",b,b.collection,a)};if(this.isNew())return d(),!1;a.success=function(e){a.wait&&d();c?c(b,e):b.trigger("sync",b,e,a)};a.error=g.wrapError(a.error,b,a);var e=(this.sync||g.sync).call(this,"delete",this,a);a.wait||d();return e},url:function(){var a=n(this,"urlRoot")||n(this.collection,"url")||t(); -return this.isNew()?a:a+("/"==a.charAt(a.length-1)?"":"/")+encodeURIComponent(this.id)},parse:function(a){return a},clone:function(){return new this.constructor(this.attributes)},isNew:function(){return null==this.id},change:function(a){a||(a={});var b=this._changing;this._changing=!0;for(var c in this._silent)this._pending[c]=!0;var d=f.extend({},a.changes,this._silent);this._silent={};for(c in d)this.trigger("change:"+c,this,this.get(c),a);if(b)return this;for(;!f.isEmpty(this._pending);){this._pending= -{};this.trigger("change",this,a);for(c in this.changed)!this._pending[c]&&!this._silent[c]&&delete this.changed[c];this._previousAttributes=f.clone(this.attributes)}this._changing=!1;return this},hasChanged:function(a){return!arguments.length?!f.isEmpty(this.changed):f.has(this.changed,a)},changedAttributes:function(a){if(!a)return this.hasChanged()?f.clone(this.changed):!1;var b,c=!1,d=this._previousAttributes,e;for(e in a)if(!f.isEqual(d[e],b=a[e]))(c||(c={}))[e]=b;return c},previous:function(a){return!arguments.length|| -!this._previousAttributes?null:this._previousAttributes[a]},previousAttributes:function(){return f.clone(this._previousAttributes)},isValid:function(){return!this.validate(this.attributes)},_validate:function(a,b){if(b.silent||!this.validate)return!0;var a=f.extend({},this.attributes,a),c=this.validate(a,b);if(!c)return!0;b&&b.error?b.error(this,c,b):this.trigger("error",this,c,b);return!1}});var r=g.Collection=function(a,b){b||(b={});b.model&&(this.model=b.model);b.comparator&&(this.comparator=b.comparator); -this._reset();this.initialize.apply(this,arguments);a&&this.reset(a,{silent:!0,parse:b.parse})};f.extend(r.prototype,k,{model:o,initialize:function(){},toJSON:function(a){return this.map(function(b){return b.toJSON(a)})},add:function(a,b){var c,d,e,g,i,j={},k={},l=[];b||(b={});a=f.isArray(a)?a.slice():[a];c=0;for(d=a.length;c=b))this.iframe=i('