Merge branch 'develop' of github.com:matrix-org/synapse into erikj/fix_get_missing_events_error

.buildkite/docker-compose.py27.pg94.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.4
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:2.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py27.pg95.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.5
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:2.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py35.pg94.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.4
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:3.5
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/format_tap.py

@@ -0,0 +1,33 @@
+import sys
+from tap.parser import Parser
+from tap.line import Result, Unknown, Diagnostic
+
+out = ["### TAP Output for " + sys.argv[2]]
+
+p = Parser()
+
+in_error = False
+
+for line in p.parse_file(sys.argv[1]):
+    if isinstance(line, Result):
+        if in_error:
+            out.append("")
+            out.append("</pre></code></details>")
+            out.append("")
+            out.append("----")
+            out.append("")
+        in_error = False
+
+        if not line.ok and not line.todo:
+            in_error = True
+
+            out.append("FAILURE Test #%d: ``%s``" % (line.number, line.description))
+            out.append("")
+            out.append("<details><summary>Show log</summary><code><pre>")
+
+    elif isinstance(line, Diagnostic) and in_error:
+        out.append(line.text)
+
+if out:
+    for line in out[:-3]:
+        print(line)

.buildkite/merge_base_branch.sh

@@ -1,22 +1,21 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
-# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
-# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
-echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
-source $BASH_ENV
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
+    echo "Not merging forward, as this is a release branch"
+    exit 0
+fi
 
-if [[ -z "${CIRCLE_PR_NUMBER}" ]]
-then
-    echo "Can't figure out what the PR number is! Assuming merge target is develop."
+if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
+    echo "Not a pull request, or hasn't had a PR opened yet..."
 
     # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
     # can probably assume it's based on it and will be merged into it.
     GITBASE="develop"
 else
     # Get the reference, using the GitHub API
-    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
 fi
 
 # Show what we are before

.buildkite/pipeline.yml

@@ -2,6 +2,7 @@ env:
   CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
 
 steps:
+
   - command:
       - "python -m pip install tox"
       - "tox -e pep8"
@@ -36,8 +37,6 @@ steps:
           image: "python:3.6"
           propagate-environment: true
 
-  - wait
-
   - command:
       - "python -m pip install tox"
       - "tox -e check-sampleconfig"
@@ -46,16 +45,25 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
 
+  - wait
+
+
   - command:
       - "python -m pip install tox"
-      - "tox -e py27,codecov"
-    label: ":python: 2.7 / SQLite"
+      - "tox -e py35-old,codecov"
+    label: ":python: 3.5 / SQLite / Old Deps"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "python:2.7"
+          image: "python:3.5"
           propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -67,6 +75,12 @@ steps:
       - docker#v3.0.1:
           image: "python:3.5"
           propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -78,6 +92,12 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
           propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -89,50 +109,12 @@ steps:
       - docker#v3.0.1:
           image: "python:3.7"
           propagate-environment: true
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e py27-old,codecov"
-    label: ":python: 2.7 / SQLite / Old Deps"
-    env:
-      TRIAL_FLAGS: "-j 2"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:2.7"
-          propagate-environment: true
-
-  - label: ":python: 2.7 / :postgres: 9.4"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py27.pg94.yaml
-
-  - label: ":python: 2.7 / :postgres: 9.5"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py27.pg95.yaml
-
-  - label: ":python: 3.5 / :postgres: 9.4"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py35.pg94.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - label: ":python: 3.5 / :postgres: 9.5"
     env:
@@ -144,6 +126,12 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py35.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
     env:
@@ -155,6 +143,12 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
     env:
@@ -166,3 +160,67 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg11.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+
+  - label: "SyTest - :python: 3.5 / SQLite / Monolith"
+    agents:
+      queue: "medium"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Monolith"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+      WORKERS: "1"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2

.buildkite/synapse_sytest.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+#
+# Fetch sytest, and then run the tests for synapse. The entrypoint for the
+# sytest-synapse docker images.
+
+set -ex
+
+if [ -n "$BUILDKITE" ]
+then
+    SYNAPSE_DIR=`pwd`
+else
+    SYNAPSE_DIR="/src"
+fi
+
+# Attempt to find a sytest to use.
+# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
+if [ -d "/sytest" ]; then
+    # If the user has mounted in a SyTest checkout, use that.
+    echo "Using local sytests..."
+
+    # create ourselves a working directory and dos2unix some scripts therein
+    mkdir -p /work/jenkins
+    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
+        dos2unix -n "/sytest/$i" "/work/$i"
+    done
+    ln -sf /sytest/tests /work
+    ln -sf /sytest/keys /work
+    SYTEST_LIB="/sytest/lib"
+else
+    if [ -n "BUILDKITE_BRANCH" ]
+    then
+        branch_name=$BUILDKITE_BRANCH
+    else
+        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
+        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
+    fi
+
+    # Try and fetch the branch
+    echo "Trying to get same-named sytest branch..."
+    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
+        # Probably a 404, fall back to develop
+        echo "Using develop instead..."
+        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
+    }
+
+    mkdir -p /work
+    tar -C /work --strip-components=1 -xf sytest.tar.gz
+    SYTEST_LIB="/work/lib"
+fi
+
+cd /work
+
+# PostgreSQL setup
+if [ -n "$POSTGRES" ]
+then
+    export PGUSER=postgres
+    export POSTGRES_DB_1=pg1
+    export POSTGRES_DB_2=pg2
+
+    # Start the database
+    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
+
+    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
+    jenkins/prep_sytest_for_postgres.sh
+
+    # Make the test databases for the two Synapse servers that will be spun up
+    su -c 'psql -c "CREATE DATABASE pg1;"' postgres
+    su -c 'psql -c "CREATE DATABASE pg2;"' postgres
+
+fi
+
+if [ -n "$OFFLINE" ]; then
+    # if we're in offline mode, just put synapse into the virtualenv, and
+    # hope that the deps are up-to-date.
+    #
+    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
+    # so we just run setup.py explicitly.)
+    #
+    (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
+else
+    # We've already created the virtualenv, but lets double check we have all
+    # deps.
+    /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
+    /venv/bin/pip install -q --upgrade --no-cache-dir \
+        lxml psycopg2 coverage codecov tap.py
+
+    # Make sure all Perl deps are installed -- this is done in the docker build
+    # so will only install packages added since the last Docker build
+    ./install-deps.pl
+fi
+
+
+# Run the tests
+>&2 echo "+++ Running tests"
+
+RUN_TESTS=(
+    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
+)
+
+TEST_STATUS=0
+
+if [ -n "$WORKERS" ]; then
+    RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
+else
+    RUN_TESTS+=(-I Synapse)
+fi
+
+"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
+
+if [ $TEST_STATUS -ne 0 ]; then
+    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
+else
+    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
+fi
+
+>&2 echo "--- Copying assets"
+
+# Copy out the logs
+mkdir -p /logs
+cp results.tap /logs/results.tap
+rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+
+# Upload coverage to codecov and upload files, if running on Buildkite
+if [ -n "$BUILDKITE" ]
+then
+    /venv/bin/coverage combine || true
+    /venv/bin/coverage xml || true
+    /venv/bin/codecov -X gcov -f coverage.xml
+
+    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
+    tar xvf buildkite.tar.gz
+    chmod +x ./buildkite-agent
+
+    # Upload the files
+    ./buildkite-agent artifact upload "/logs/**/*.log*"
+    ./buildkite-agent artifact upload "/logs/results.tap"
+
+    if [ $TEST_STATUS -ne 0 ]; then
+        # Annotate, if failure
+        /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
+    fi
+fi
+
+
+exit $TEST_STATUS

.circleci/config.yml

@@ -4,160 +4,23 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py2 .
       - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py2
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py2 .
       - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:latest-py2
       - run: docker push matrixdotorg/synapse:latest-py3
-  sytestpy2:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-
-  sytestpy3:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
 
 workflows:
   version: 2
   build:
     jobs:
-      - sytestpy2:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy2postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
       - dockerhubuploadrelease:
           filters:
             tags:

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# One username per supported platform and one custom link
+patreon: matrixdotorg
+liberapay: matrixdotorg
+custom: https://paypal.me/matrixdotorg

CHANGES.md

@@ -1,3 +1,290 @@
+Synapse 1.0.0 (2019-06-11)
+==========================
+
+Bugfixes
+--------
+
+- Fix bug where attempting to send transactions with large number of EDUs can fail. ([\#5418](https://github.com/matrix-org/synapse/issues/5418))
+
+
+Improved Documentation
+----------------------
+
+- Expand the federation guide to include relevant content from the MSC1711 FAQ ([\#5419](https://github.com/matrix-org/synapse/issues/5419))
+
+
+Internal Changes
+----------------
+
+- Move password reset links to /_matrix/client/unstable namespace. ([\#5424](https://github.com/matrix-org/synapse/issues/5424))
+
+
+Synapse 1.0.0rc3 (2019-06-10)
+=============================
+
+Security: Fix authentication bug introduced in 1.0.0rc1. Please upgrade to rc3 immediately
+
+
+Synapse 1.0.0rc2 (2019-06-10)
+=============================
+
+Bugfixes
+--------
+
+- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392))
+- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415))
+- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417))
+
+
+Synapse 1.0.0rc1 (2019-06-07)
+=============================
+
+Features
+--------
+
+- Synapse now more efficiently collates room statistics. ([\#4338](https://github.com/matrix-org/synapse/issues/4338), [\#5260](https://github.com/matrix-org/synapse/issues/5260), [\#5324](https://github.com/matrix-org/synapse/issues/5324))
+- Add experimental support for relations (aka reactions and edits). ([\#5220](https://github.com/matrix-org/synapse/issues/5220))
+- Ability to configure default room version. ([\#5223](https://github.com/matrix-org/synapse/issues/5223), [\#5249](https://github.com/matrix-org/synapse/issues/5249))
+- Allow configuring a range for the account validity startup job. ([\#5276](https://github.com/matrix-org/synapse/issues/5276))
+- CAS login will now hit the r0 API, not the deprecated v1 one. ([\#5286](https://github.com/matrix-org/synapse/issues/5286))
+- Validate federation server TLS certificates by default (implements [MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)). ([\#5359](https://github.com/matrix-org/synapse/issues/5359))
+- Update /_matrix/client/versions to reference support for r0.5.0. ([\#5360](https://github.com/matrix-org/synapse/issues/5360))
+- Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361))
+- Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371))
+- Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374))
+- Add ability to perform password reset via email without trusting the identity server. ([\#5377](https://github.com/matrix-org/synapse/issues/5377))
+- Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379))
+
+
+Bugfixes
+--------
+
+- Fixes client-server API not sending "m.heroes" to lazy-load /sync requests when a rooms name or its canonical alias are empty. Thanks to @dnaf for this work! ([\#5089](https://github.com/matrix-org/synapse/issues/5089))
+- Prevent federation device list updates breaking when processing multiple updates at once. ([\#5156](https://github.com/matrix-org/synapse/issues/5156))
+- Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. ([\#5200](https://github.com/matrix-org/synapse/issues/5200))
+- Fix race when backfilling in rooms with worker mode. ([\#5221](https://github.com/matrix-org/synapse/issues/5221))
+- Fix appservice timestamp massaging. ([\#5233](https://github.com/matrix-org/synapse/issues/5233))
+- Ensure that server_keys fetched via a notary server are correctly signed. ([\#5251](https://github.com/matrix-org/synapse/issues/5251))
+- Show the correct error when logging out and access token is missing. ([\#5256](https://github.com/matrix-org/synapse/issues/5256))
+- Fix error code when there is an invalid parameter on /_matrix/client/r0/publicRooms ([\#5257](https://github.com/matrix-org/synapse/issues/5257))
+- Fix error when downloading thumbnail with missing width/height parameter. ([\#5258](https://github.com/matrix-org/synapse/issues/5258))
+- Fix schema update for account validity. ([\#5268](https://github.com/matrix-org/synapse/issues/5268))
+- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
+- Fix "db txn 'update_presence' from sentinel context" log messages. ([\#5275](https://github.com/matrix-org/synapse/issues/5275))
+- Fix dropped logcontexts during high outbound traffic. ([\#5277](https://github.com/matrix-org/synapse/issues/5277))
+- Fix a bug where it is not possible to get events in the federation format with the request `GET /_matrix/client/r0/rooms/{roomId}/messages`. ([\#5293](https://github.com/matrix-org/synapse/issues/5293))
+- Fix performance problems with the rooms stats background update. ([\#5294](https://github.com/matrix-org/synapse/issues/5294))
+- Fix noisy 'no key for server' logs. ([\#5300](https://github.com/matrix-org/synapse/issues/5300))
+- Fix bug where a notary server would sometimes forget old keys. ([\#5307](https://github.com/matrix-org/synapse/issues/5307))
+- Prevent users from setting huge displaynames and avatar URLs. ([\#5309](https://github.com/matrix-org/synapse/issues/5309))
+- Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails. ([\#5317](https://github.com/matrix-org/synapse/issues/5317))
+- Ensure that we have an up-to-date copy of the signing key when validating incoming federation requests. ([\#5321](https://github.com/matrix-org/synapse/issues/5321))
+- Fix various problems which made the signing-key notary server time out for some requests. ([\#5333](https://github.com/matrix-org/synapse/issues/5333))
+- Fix bug which would make certain operations (such as room joins) block for 20 minutes while attemoting to fetch verification keys. ([\#5334](https://github.com/matrix-org/synapse/issues/5334))
+- Fix a bug where we could rapidly mark a server as unreachable even though it was only down for a few minutes. ([\#5335](https://github.com/matrix-org/synapse/issues/5335), [\#5340](https://github.com/matrix-org/synapse/issues/5340))
+- Fix a bug where account validity renewal emails could only be sent when email notifs were enabled. ([\#5341](https://github.com/matrix-org/synapse/issues/5341))
+- Fix failure when fetching batches of events during backfill, etc. ([\#5342](https://github.com/matrix-org/synapse/issues/5342))
+- Add a new room version where the timestamps on events are checked against the validity periods on signing keys. ([\#5348](https://github.com/matrix-org/synapse/issues/5348), [\#5354](https://github.com/matrix-org/synapse/issues/5354))
+- Fix room stats and presence background updates to correctly handle missing events. ([\#5352](https://github.com/matrix-org/synapse/issues/5352))
+- Include left members in room summaries' heroes. ([\#5355](https://github.com/matrix-org/synapse/issues/5355))
+- Fix `federation_custom_ca_list` configuration option. ([\#5362](https://github.com/matrix-org/synapse/issues/5362))
+- Fix missing logcontext warnings on shutdown. ([\#5369](https://github.com/matrix-org/synapse/issues/5369))
+
+
+Improved Documentation
+----------------------
+
+- Fix docs on resetting the user directory. ([\#5282](https://github.com/matrix-org/synapse/issues/5282))
+- Fix notes about ACME in the MSC1711 faq. ([\#5357](https://github.com/matrix-org/synapse/issues/5357))
+
+
+Internal Changes
+----------------
+
+- Synapse will now serve the experimental "room complexity" API endpoint. ([\#5216](https://github.com/matrix-org/synapse/issues/5216))
+- The base classes for the v1 and v2_alpha REST APIs have been unified. ([\#5226](https://github.com/matrix-org/synapse/issues/5226), [\#5328](https://github.com/matrix-org/synapse/issues/5328))
+- Simplifications and comments in do_auth. ([\#5227](https://github.com/matrix-org/synapse/issues/5227))
+- Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2. ([\#5230](https://github.com/matrix-org/synapse/issues/5230))
+- Preparatory work for key-validity features. ([\#5232](https://github.com/matrix-org/synapse/issues/5232), [\#5234](https://github.com/matrix-org/synapse/issues/5234), [\#5235](https://github.com/matrix-org/synapse/issues/5235), [\#5236](https://github.com/matrix-org/synapse/issues/5236), [\#5237](https://github.com/matrix-org/synapse/issues/5237), [\#5244](https://github.com/matrix-org/synapse/issues/5244), [\#5250](https://github.com/matrix-org/synapse/issues/5250), [\#5296](https://github.com/matrix-org/synapse/issues/5296), [\#5299](https://github.com/matrix-org/synapse/issues/5299), [\#5343](https://github.com/matrix-org/synapse/issues/5343), [\#5347](https://github.com/matrix-org/synapse/issues/5347), [\#5356](https://github.com/matrix-org/synapse/issues/5356))
+- Specify the type of reCAPTCHA key to use. ([\#5283](https://github.com/matrix-org/synapse/issues/5283))
+- Improve sample config for monthly active user blocking. ([\#5284](https://github.com/matrix-org/synapse/issues/5284))
+- Remove spurious debug from MatrixFederationHttpClient.get_json. ([\#5287](https://github.com/matrix-org/synapse/issues/5287))
+- Improve logging for logcontext leaks. ([\#5288](https://github.com/matrix-org/synapse/issues/5288))
+- Clarify that the admin change password API logs the user out. ([\#5303](https://github.com/matrix-org/synapse/issues/5303))
+- New installs will now use the v54 full schema, rather than the full schema v14 and applying incremental updates to v54. ([\#5320](https://github.com/matrix-org/synapse/issues/5320))
+- Improve docstrings on MatrixFederationClient. ([\#5332](https://github.com/matrix-org/synapse/issues/5332))
+- Clean up FederationClient.get_events for clarity. ([\#5344](https://github.com/matrix-org/synapse/issues/5344))
+- Various improvements to debug logging. ([\#5353](https://github.com/matrix-org/synapse/issues/5353))
+- Don't run CI build checks until sample config check has passed. ([\#5370](https://github.com/matrix-org/synapse/issues/5370))
+- Automatically retry buildkite builds (max twice) when an agent is lost. ([\#5380](https://github.com/matrix-org/synapse/issues/5380))
+
+
+Synapse 0.99.5.2 (2019-05-30)
+=============================
+
+Bugfixes
+--------
+
+- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
+
+
+Synapse 0.99.5.1 (2019-05-22)
+=============================
+
+0.99.5.1 supersedes 0.99.5 due to malformed debian changelog - no functional changes.
+
+Synapse 0.99.5 (2019-05-22)
+===========================
+
+No significant changes.
+
+
+Synapse 0.99.5rc1 (2019-05-21)
+==============================
+
+Features
+--------
+
+- Add ability to blacklist IP ranges for the federation client. ([\#5043](https://github.com/matrix-org/synapse/issues/5043))
+- Ratelimiting configuration for clients sending messages and the federation server has been altered to match login ratelimiting. The old configuration names will continue working. Check the sample config for details of the new names. ([\#5181](https://github.com/matrix-org/synapse/issues/5181))
+- Drop support for the undocumented /_matrix/client/v2_alpha API prefix. ([\#5190](https://github.com/matrix-org/synapse/issues/5190))
+- Add an option to disable per-room profiles. ([\#5196](https://github.com/matrix-org/synapse/issues/5196))
+- Stick an expiration date to any registered user missing one at startup if account validity is enabled. ([\#5204](https://github.com/matrix-org/synapse/issues/5204))
+- Add experimental support for relations (aka reactions and edits). ([\#5209](https://github.com/matrix-org/synapse/issues/5209), [\#5211](https://github.com/matrix-org/synapse/issues/5211), [\#5203](https://github.com/matrix-org/synapse/issues/5203), [\#5212](https://github.com/matrix-org/synapse/issues/5212))
+- Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). ([\#5210](https://github.com/matrix-org/synapse/issues/5210), [\#5217](https://github.com/matrix-org/synapse/issues/5217))
+
+
+Bugfixes
+--------
+
+- Fix image orientation when generating thumbnails (needs pillow>=4.3.0). Contributed by Pau Rodriguez-Estivill. ([\#5039](https://github.com/matrix-org/synapse/issues/5039))
+- Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error. ([\#5146](https://github.com/matrix-org/synapse/issues/5146))
+- Re-order stages in registration flows such that msisdn and email verification are done last. ([\#5174](https://github.com/matrix-org/synapse/issues/5174))
+- Fix 3pid guest invites. ([\#5177](https://github.com/matrix-org/synapse/issues/5177))
+- Fix a bug where the register endpoint would fail with M_THREEPID_IN_USE instead of returning an account previously registered in the same session. ([\#5187](https://github.com/matrix-org/synapse/issues/5187))
+- Prevent registration for user ids that are too long to fit into a state key. Contributed by Reid Anderson. ([\#5198](https://github.com/matrix-org/synapse/issues/5198))
+- Fix incompatibility between ACME support and Python 3.5.2. ([\#5218](https://github.com/matrix-org/synapse/issues/5218))
+- Fix error handling for rooms whose versions are unknown. ([\#5219](https://github.com/matrix-org/synapse/issues/5219))
+
+
+Internal Changes
+----------------
+
+- Make /sync attempt to return device updates for both joined and invited users. Note that this doesn't currently work correctly due to other bugs. ([\#3484](https://github.com/matrix-org/synapse/issues/3484))
+- Update tests to consistently be configured via the same code that is used when loading from configuration files. ([\#5171](https://github.com/matrix-org/synapse/issues/5171), [\#5185](https://github.com/matrix-org/synapse/issues/5185))
+- Allow client event serialization to be async. ([\#5183](https://github.com/matrix-org/synapse/issues/5183))
+- Expose DataStore._get_events as get_events_as_list. ([\#5184](https://github.com/matrix-org/synapse/issues/5184))
+- Make generating SQL bounds for pagination generic. ([\#5191](https://github.com/matrix-org/synapse/issues/5191))
+- Stop telling people to install the optional dependencies by default. ([\#5197](https://github.com/matrix-org/synapse/issues/5197))
+
+
+Synapse 0.99.4 (2019-05-15)
+===========================
+
+No significant changes.
+
+
+Synapse 0.99.4rc1 (2019-05-13)
+==============================
+
+Features
+--------
+
+- Add systemd-python to the optional dependencies to enable logging to the systemd journal. Install with `pip install matrix-synapse[systemd]`. ([\#4339](https://github.com/matrix-org/synapse/issues/4339))
+- Add a default .m.rule.tombstone push rule. ([\#4867](https://github.com/matrix-org/synapse/issues/4867))
+- Add ability for password provider modules to bind email addresses to users upon registration. ([\#4947](https://github.com/matrix-org/synapse/issues/4947))
+- Implementation of [MSC1711](https://github.com/matrix-org/matrix-doc/pull/1711) including config options for requiring valid TLS certificates for federation traffic, the ability to disable TLS validation for specific domains, and the ability to specify your own list of CA certificates. ([\#4967](https://github.com/matrix-org/synapse/issues/4967))
+- Remove presence list support as per MSC 1819. ([\#4989](https://github.com/matrix-org/synapse/issues/4989))
+- Reduce CPU usage starting pushers during start up. ([\#4991](https://github.com/matrix-org/synapse/issues/4991))
+- Add a delete group admin API. ([\#5002](https://github.com/matrix-org/synapse/issues/5002))
+- Add config option to block users from looking up 3PIDs. ([\#5010](https://github.com/matrix-org/synapse/issues/5010))
+- Add context to phonehome stats. ([\#5020](https://github.com/matrix-org/synapse/issues/5020))
+- Configure the example systemd units to have a log identifier of `matrix-synapse`
+  instead of the executable name, `python`.
+  Contributed by Christoph Müller. ([\#5023](https://github.com/matrix-org/synapse/issues/5023))
+- Add time-based account expiration. ([\#5027](https://github.com/matrix-org/synapse/issues/5027), [\#5047](https://github.com/matrix-org/synapse/issues/5047), [\#5073](https://github.com/matrix-org/synapse/issues/5073), [\#5116](https://github.com/matrix-org/synapse/issues/5116))
+- Add support for handling `/versions`, `/voip` and `/push_rules` client endpoints to client_reader worker. ([\#5063](https://github.com/matrix-org/synapse/issues/5063), [\#5065](https://github.com/matrix-org/synapse/issues/5065), [\#5070](https://github.com/matrix-org/synapse/issues/5070))
+- Add a configuration option to require authentication on /publicRooms and /profile endpoints. ([\#5083](https://github.com/matrix-org/synapse/issues/5083))
+- Move admin APIs to `/_synapse/admin/v1`. (The old paths are retained for backwards-compatibility, for now). ([\#5119](https://github.com/matrix-org/synapse/issues/5119))
+- Implement an admin API for sending server notices. Many thanks to @krombel who provided a foundation for this work. ([\#5121](https://github.com/matrix-org/synapse/issues/5121), [\#5142](https://github.com/matrix-org/synapse/issues/5142))
+
+
+Bugfixes
+--------
+
+- Avoid redundant URL encoding of redirect URL for SSO login in the fallback login page. Fixes a regression introduced in [#4220](https://github.com/matrix-org/synapse/pull/4220). Contributed by Marcel Fabian Krüger ("[zaugin](https://github.com/zauguin)"). ([\#4555](https://github.com/matrix-org/synapse/issues/4555))
+- Fix bug where presence updates were sent to all servers in a room when a new server joined, rather than to just the new server. ([\#4942](https://github.com/matrix-org/synapse/issues/4942), [\#5103](https://github.com/matrix-org/synapse/issues/5103))
+- Fix sync bug which made accepting invites unreliable in worker-mode synapses. ([\#4955](https://github.com/matrix-org/synapse/issues/4955), [\#4956](https://github.com/matrix-org/synapse/issues/4956))
+- start.sh: Fix the --no-rate-limit option for messages and make it bypass rate limit on registration and login too. ([\#4981](https://github.com/matrix-org/synapse/issues/4981))
+- Transfer related groups on room upgrade. ([\#4990](https://github.com/matrix-org/synapse/issues/4990))
+- Prevent the ability to kick users from a room they aren't in. ([\#4999](https://github.com/matrix-org/synapse/issues/4999))
+- Fix issue #4596 so synapse_port_db script works with --curses option on Python 3. Contributed by Anders Jensen-Waud <anders@jensenwaud.com>. ([\#5003](https://github.com/matrix-org/synapse/issues/5003))
+- Clients timing out/disappearing while downloading from the media repository will now no longer log a spurious "Producer was not unregistered" message. ([\#5009](https://github.com/matrix-org/synapse/issues/5009))
+- Fix "cannot import name execute_batch" error with postgres. ([\#5032](https://github.com/matrix-org/synapse/issues/5032))
+- Fix disappearing exceptions in manhole. ([\#5035](https://github.com/matrix-org/synapse/issues/5035))
+- Workaround bug in twisted where attempting too many concurrent DNS requests could cause it to hang due to running out of file descriptors. ([\#5037](https://github.com/matrix-org/synapse/issues/5037))
+- Make sure we're not registering the same 3pid twice on registration. ([\#5071](https://github.com/matrix-org/synapse/issues/5071))
+- Don't crash on lack of expiry templates. ([\#5077](https://github.com/matrix-org/synapse/issues/5077))
+- Fix the ratelimiting on third party invites. ([\#5104](https://github.com/matrix-org/synapse/issues/5104))
+- Add some missing limitations to room alias creation. ([\#5124](https://github.com/matrix-org/synapse/issues/5124), [\#5128](https://github.com/matrix-org/synapse/issues/5128))
+- Limit the number of EDUs in transactions to 100 as expected by synapse. Thanks to @superboum for this work! ([\#5138](https://github.com/matrix-org/synapse/issues/5138))
+
+Internal Changes
+----------------
+
+- Add test to verify threepid auth check added in #4435. ([\#4474](https://github.com/matrix-org/synapse/issues/4474))
+- Fix/improve some docstrings in the replication code. ([\#4949](https://github.com/matrix-org/synapse/issues/4949))
+- Split synapse.replication.tcp.streams into smaller files. ([\#4953](https://github.com/matrix-org/synapse/issues/4953))
+- Refactor replication row generation/parsing. ([\#4954](https://github.com/matrix-org/synapse/issues/4954))
+- Run `black` to clean up formatting on `synapse/storage/roommember.py` and `synapse/storage/events.py`. ([\#4959](https://github.com/matrix-org/synapse/issues/4959))
+- Remove log line for password via the admin API. ([\#4965](https://github.com/matrix-org/synapse/issues/4965))
+- Fix typo in TLS filenames in docker/README.md. Also add the '-p' commandline option to the 'docker run' example. Contributed by Jurrie Overgoor. ([\#4968](https://github.com/matrix-org/synapse/issues/4968))
+- Refactor room version definitions. ([\#4969](https://github.com/matrix-org/synapse/issues/4969))
+- Reduce log level of .well-known/matrix/client responses. ([\#4972](https://github.com/matrix-org/synapse/issues/4972))
+- Add `config.signing_key_path` that can be read by `synapse.config` utility. ([\#4974](https://github.com/matrix-org/synapse/issues/4974))
+- Track which identity server is used when binding a threepid and use that for unbinding, as per MSC1915. ([\#4982](https://github.com/matrix-org/synapse/issues/4982))
+- Rewrite KeyringTestCase as a HomeserverTestCase. ([\#4985](https://github.com/matrix-org/synapse/issues/4985))
+- README updates: Corrected the default POSTGRES_USER. Added port forwarding hint in TLS section. ([\#4987](https://github.com/matrix-org/synapse/issues/4987))
+- Remove a number of unused tables from the database schema. ([\#4992](https://github.com/matrix-org/synapse/issues/4992), [\#5028](https://github.com/matrix-org/synapse/issues/5028), [\#5033](https://github.com/matrix-org/synapse/issues/5033))
+- Run `black` on the remainder of `synapse/storage/`. ([\#4996](https://github.com/matrix-org/synapse/issues/4996))
+- Fix grammar in get_current_users_in_room and give it a docstring. ([\#4998](https://github.com/matrix-org/synapse/issues/4998))
+- Clean up some code in the server-key Keyring. ([\#5001](https://github.com/matrix-org/synapse/issues/5001))
+- Convert SYNAPSE_NO_TLS Docker variable to boolean for user friendliness. Contributed by Gabriel Eckerson. ([\#5005](https://github.com/matrix-org/synapse/issues/5005))
+- Refactor synapse.storage._base._simple_select_list_paginate. ([\#5007](https://github.com/matrix-org/synapse/issues/5007))
+- Store the notary server name correctly in server_keys_json. ([\#5024](https://github.com/matrix-org/synapse/issues/5024))
+- Rewrite Datastore.get_server_verify_keys to reduce the number of database transactions. ([\#5030](https://github.com/matrix-org/synapse/issues/5030))
+- Remove extraneous period from copyright headers. ([\#5046](https://github.com/matrix-org/synapse/issues/5046))
+- Update documentation for where to get Synapse packages. ([\#5067](https://github.com/matrix-org/synapse/issues/5067))
+- Add workarounds for pep-517 install errors. ([\#5098](https://github.com/matrix-org/synapse/issues/5098))
+- Improve logging when event-signature checks fail. ([\#5100](https://github.com/matrix-org/synapse/issues/5100))
+- Factor out an "assert_requester_is_admin" function. ([\#5120](https://github.com/matrix-org/synapse/issues/5120))
+- Remove the requirement to authenticate for /admin/server_version. ([\#5122](https://github.com/matrix-org/synapse/issues/5122))
+- Prevent an exception from being raised in a IResolutionReceiver and use a more generic error message for blacklisted URL previews. ([\#5155](https://github.com/matrix-org/synapse/issues/5155))
+- Run `black` on the tests directory. ([\#5170](https://github.com/matrix-org/synapse/issues/5170))
+- Fix CI after new release of isort. ([\#5179](https://github.com/matrix-org/synapse/issues/5179))
+- Fix bogus imports in unit tests. ([\#5154](https://github.com/matrix-org/synapse/issues/5154))
+
+
+Synapse 0.99.3.2 (2019-05-03)
+=============================
+
+Internal Changes
+----------------
+
+- Ensure that we have `urllib3` <1.25, to resolve incompatibility with `requests`. ([\#5135](https://github.com/matrix-org/synapse/issues/5135))
+
+
+Synapse 0.99.3.1 (2019-05-03)
+=============================
+
+Security update
+---------------
+
+This release includes two security fixes:
+
+- Switch to using a cryptographically-secure random number generator for token strings, ensuring they cannot be predicted by an attacker. Thanks to @opnsec for identifying and responsibly disclosing this issue! ([\#5133](https://github.com/matrix-org/synapse/issues/5133))
+- Blacklist 0.0.0.0 and :: by default for URL previews. Thanks to @opnsec for identifying and responsibly disclosing this issue too! ([\#5134](https://github.com/matrix-org/synapse/issues/5134))
+
 Synapse 0.99.3 (2019-04-01)
 ===========================
 

CONTRIBUTING.rst

@@ -30,21 +30,20 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI
-<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
-pull requests to synapse get automatically tested by Travis and CircleCI.
-If your change breaks the build, this will be shown in GitHub, so please
-keep an eye on the pull request for feedback.
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite
+<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration.
+Buildkite builds need to be authorised by a maintainer. If your change breaks
+the build, this will be shown in GitHub, so please keep an eye on the pull
+request for feedback.
 
 To run unit tests in a local development environment, you can use:
 
-- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
-  SQLite-backed Synapse on Python 2.7.
-- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
+  for SQLite-backed Synapse on Python 3.5.
 - ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
-- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
+- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
   (requires a running local PostgreSQL with access to create databases).
-- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
   (requires Docker). Entirely self-contained, recommended if you don't want to
   set up PostgreSQL yourself.
 

INSTALL.md

@@ -1,13 +1,14 @@
-* [Installing Synapse](#installing-synapse)
-  * [Installing from source](#installing-from-source)
-    * [Platform-Specific Instructions](#platform-specific-instructions)
-    * [Troubleshooting Installation](#troubleshooting-installation)
-  * [Prebuilt packages](#prebuilt-packages)
-* [Setting up Synapse](#setting-up-synapse)
-  * [TLS certificates](#tls-certificates)
-  * [Registering a user](#registering-a-user)
-  * [Setting up a TURN server](#setting-up-a-turn-server)
-  * [URL previews](#url-previews)
+- [Installing Synapse](#installing-synapse)
+  - [Installing from source](#installing-from-source)
+    - [Platform-Specific Instructions](#platform-specific-instructions)
+    - [Troubleshooting Installation](#troubleshooting-installation)
+  - [Prebuilt packages](#prebuilt-packages)
+- [Setting up Synapse](#setting-up-synapse)
+  - [TLS certificates](#tls-certificates)
+  - [Email](#email)
+  - [Registering a user](#registering-a-user)
+  - [Setting up a TURN server](#setting-up-a-turn-server)
+  - [URL previews](#url-previews)
 
 # Installing Synapse
 
@@ -35,7 +36,7 @@ virtualenv -p python3 ~/synapse/env
 source ~/synapse/env/bin/activate
 pip install --upgrade pip
 pip install --upgrade setuptools
-pip install matrix-synapse[all]
+pip install matrix-synapse
 ```
 
 This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
@@ -48,7 +49,7 @@ update flag:
 
 ```
 source ~/synapse/env/bin/activate
-pip install -U matrix-synapse[all]
+pip install -U matrix-synapse
 ```
 
 Before you can start Synapse, you will need to generate a configuration
@@ -257,18 +258,29 @@ https://github.com/spantaleev/matrix-docker-ansible-deploy
 #### Matrix.org packages
 
 Matrix.org provides Debian/Ubuntu packages of the latest stable version of
-Synapse via https://matrix.org/packages/debian/. To use them:
+Synapse via https://packages.matrix.org/debian/. They are available for Debian
+9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
 
 ```
-sudo apt install -y lsb-release curl apt-transport-https
-echo "deb https://matrix.org/packages/debian `lsb_release -cs` main" |
+sudo apt install -y lsb-release wget apt-transport-https
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
     sudo tee /etc/apt/sources.list.d/matrix-org.list
-curl "https://matrix.org/packages/debian/repo-key.asc" |
-    sudo apt-key add -
 sudo apt update
 sudo apt install matrix-synapse-py3
 ```
 
+**Note**: if you followed a previous version of these instructions which
+recommended using `apt-key add` to add an old key from
+`https://matrix.org/packages/debian/`, you should note that this key has been
+revoked. You should remove the old key with `sudo apt-key remove
+C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
+update your configuration.
+
+The fingerprint of the repository signing key (as shown by `gpg
+/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
+`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
+
 #### Downstream Debian/Ubuntu packages
 
 For `buster` and `sid`, Synapse is available in the Debian repositories and
@@ -383,8 +395,22 @@ To configure Synapse to expose an HTTPS port, you will need to edit
   instance, if using certbot, use `fullchain.pem` as your certificate, not
   `cert.pem`).
 
-For those of you upgrading your TLS certificate in readiness for Synapse 1.0,
-please take a look at [our guide](docs/MSC1711_certificates_FAQ.md#configuring-certificates-for-compatibility-with-synapse-100).
+For a more detailed guide to configuring your server for federation, see
+[federate.md](docs/federate.md)
+
+
+## Email
+
+It is desirable for Synapse to have the capability to send email. For example,
+this is required to support the 'password reset' feature.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
+and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
+``smtp_pass``, and ``require_transport_security``.
+
+If Synapse is not configured with an SMTP server, password reset via email will
+ be disabled by default.
 
 ## Registering a user
 

MANIFEST.in

@@ -9,14 +9,19 @@ include demo/*.py
 include demo/*.sh
 
 recursive-include synapse/storage/schema *.sql
+recursive-include synapse/storage/schema *.sql.postgres
+recursive-include synapse/storage/schema *.sql.sqlite
 recursive-include synapse/storage/schema *.py
+recursive-include synapse/storage/schema *.txt
 
 recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
-recursive-include tests *.pem
 recursive-include tests *.py
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css

README.rst

@@ -173,7 +173,7 @@ Synapse offers two database engines:
  * `PostgreSQL <https://www.postgresql.org>`_
 
 By default Synapse uses SQLite in and doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with 
+SQLite is only recommended in Synapse for testing purposes or for servers with
 light workloads.
 
 Almost all installations should opt to use PostreSQL. Advantages include:
@@ -272,7 +272,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python3 env
     source env/bin/activate
-    python -m pip install -e .[all]
+    python -m pip install --no-pep-517 -e .[all]
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.

UPGRADE.rst

@@ -49,6 +49,82 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
+Upgrading to v1.1
+=================
+
+Synapse 1.1 removes support for older Python and PostgreSQL versions, as
+outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_.
+
+Minimum Python Version
+----------------------
+
+Synapse v1.1 has a minimum Python requirement of Python 3.5. Python 3.6 or
+Python 3.7 are recommended as they have improved internal string handling,
+significantly reducing memory usage.
+
+If you use current versions of the Matrix.org-distributed Debian packages or
+Docker images, action is not required.
+
+If you install Synapse in a Python virtual environment, please see "Upgrading to
+v0.34.0" for notes on setting up a new virtualenv under Python 3.
+
+Minimum PostgreSQL Version
+--------------------------
+
+If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above.
+Please see the
+`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_
+for more details on upgrading your database.
+
+Upgrading to v1.0
+=================
+
+Validation of TLS certificates
+------------------------------
+
+Synapse v1.0 is the first release to enforce
+validation of TLS certificates for the federation API. It is therefore
+essential that your certificates are correctly configured. See the `FAQ
+<docs/MSC1711_certificates_FAQ.md>`_ for more information.
+
+Note, v1.0 installations will also no longer be able to federate with servers
+that have not correctly configured their certificates.
+
+In rare cases, it may be desirable to disable certificate checking: for
+example, it might be essential to be able to federate with a given legacy
+server in a closed federation. This can be done in one of two ways:-
+
+* Configure the global switch ``federation_verify_certificates`` to ``false``.
+* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``.
+
+See the `sample configuration file <docs/sample_config.yaml>`_
+for more details on these settings.
+
+Email
+-----
+When a user requests a password reset, Synapse will send an email to the
+user to confirm the request.
+
+Previous versions of Synapse delegated the job of sending this email to an
+identity server. If the identity server was somehow malicious or became
+compromised, it would be theoretically possible to hijack an account through
+this means.
+
+Therefore, by default, Synapse v1.0 will send the confirmation email itself. If
+Synapse is not configured with an SMTP server, password reset via email will be
+disabled.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
+and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
+``smtp_pass``, and ``require_transport_security``.
+
+If you are absolutely certain that you wish to continue using an identity
+server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``.
+
+See the `sample configuration file <docs/sample_config.yaml>`_
+for more details on these settings.
+
 Upgrading to v0.99.0
 ====================
 

changelog.d/4474.misc

@@ -1 +0,0 @@
-Add test to verify threepid auth check added in #4435.

changelog.d/4555.bugfix

@@ -1 +0,0 @@
-Avoid redundant URL encoding of redirect URL for SSO login in the fallback login page. Fixes a regression introduced in [#4220](https://github.com/matrix-org/synapse/pull/4220). Contributed by Marcel Fabian Krüger ("[zaugin](https://github.com/zauguin)").

changelog.d/4942.bugfix

@@ -1 +0,0 @@
-Fix bug where presence updates were sent to all servers in a room when a new server joined, rather than to just the new server.

changelog.d/4947.feature

@@ -1 +0,0 @@
-Add ability for password provider modules to bind email addresses to users upon registration.

changelog.d/4949.misc

@@ -1 +0,0 @@
-Fix/improve some docstrings in the replication code.

changelog.d/4953.misc

@@ -1,2 +0,0 @@
-Split synapse.replication.tcp.streams into smaller files.
-

changelog.d/4954.misc

@@ -1 +0,0 @@
-Refactor replication row generation/parsing.

changelog.d/4955.bugfix

@@ -1 +0,0 @@
-Fix sync bug which made accepting invites unreliable in worker-mode synapses.

changelog.d/4956.bugfix

@@ -1 +0,0 @@
-Fix sync bug which made accepting invites unreliable in worker-mode synapses.

changelog.d/4959.misc

@@ -1 +0,0 @@
-Run `black` to clean up formatting on `synapse/storage/roommember.py` and `synapse/storage/events.py`.

changelog.d/4965.misc

@@ -1 +0,0 @@
-Remove log line for password via the admin API.

changelog.d/4968.misc

@@ -1 +0,0 @@
-Fix typo in TLS filenames in docker/README.md. Also add the '-p' commandline option to the 'docker run' example. Contributed by Jurrie Overgoor.

changelog.d/4969.misc

@@ -1,2 +0,0 @@
-Refactor room version definitions.
-

changelog.d/4974.misc

@@ -1 +0,0 @@
-Add `config.signing_key_path` that can be read by `synapse.config` utility.

changelog.d/4981.bugfix

@@ -1 +0,0 @@
-start.sh: Fix the --no-rate-limit option for messages and make it bypass rate limit on registration and login too.

changelog.d/4982.misc

@@ -1 +0,0 @@
-Track which identity server is used when binding a threepid and use that for unbinding, as per MSC1915.

changelog.d/4985.misc

@@ -1 +0,0 @@
-Rewrite KeyringTestCase as a HomeserverTestCase.

changelog.d/4987.misc

@@ -1 +0,0 @@
-README updates: Corrected the default POSTGRES_USER. Added port forwarding hint in TLS section.

changelog.d/4989.feature

@@ -1 +0,0 @@
-Remove presence list support as per MSC 1819.

changelog.d/4990.bugfix

@@ -1 +0,0 @@
-Transfer related groups on room upgrade.

changelog.d/4991.feature

@@ -1 +0,0 @@
-Reduce CPU usage starting pushers during start up.

changelog.d/4992.misc

@@ -1 +0,0 @@
-Remove a number of unused tables from the database schema.

changelog.d/4996.misc

@@ -1 +0,0 @@
-Run `black` on the remainder of `synapse/storage/`.

changelog.d/4998.misc

@@ -1 +0,0 @@
-Fix grammar in get_current_users_in_room and give it a docstring.

changelog.d/4999.bugfix

@@ -1 +0,0 @@
-Prevent the ability to kick users from a room they aren't in.

changelog.d/5001.misc

@@ -1 +0,0 @@
-Clean up some code in the server-key Keyring.

changelog.d/5002.feature

@@ -1 +0,0 @@
-Add a delete group admin API.

changelog.d/5003.bugfix

@@ -1 +0,0 @@
-Fix issue #4596 so synapse_port_db script works with --curses option on Python 3. Contributed by Anders Jensen-Waud <anders@jensenwaud.com>.

changelog.d/5007.misc

@@ -1 +0,0 @@
-Refactor synapse.storage._base._simple_select_list_paginate.

changelog.d/5010.feature

@@ -1 +0,0 @@
-Add config option to block users from looking up 3PIDs.

changelog.d/5020.feature

@@ -1 +0,0 @@
-Add context to phonehome stats.

changelog.d/5024.misc

@@ -1 +0,0 @@
-Store the notary server name correctly in server_keys_json.

changelog.d/5027.feature

@@ -1 +0,0 @@
-Add time-based account expiration.

changelog.d/5028.misc

@@ -1 +0,0 @@
-Remove a number of unused tables from the database schema.

changelog.d/5030.misc

@@ -1 +0,0 @@
-Rewrite Datastore.get_server_verify_keys to reduce the number of database transactions.

changelog.d/5032.bugfix

@@ -1 +0,0 @@
-Fix "cannot import name execute_batch" error with postgres.

changelog.d/5035.bugfix

@@ -1 +0,0 @@
-Fix disappearing exceptions in manhole.

changelog.d/5252.feature

@@ -0,0 +1 @@
+Add monthly active users to phonehome stats.

changelog.d/5325.bugfix

@@ -0,0 +1 @@
+Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean.

changelog.d/5363.feature

@@ -0,0 +1 @@
+Allow expired user to trigger renewal email sending manually.

changelog.d/5378.misc

@@ -0,0 +1 @@
+Track deactivated accounts in the database.

changelog.d/5381.misc

@@ -0,0 +1 @@
+Clean up code for sending federation EDUs.

changelog.d/5382.misc

@@ -0,0 +1 @@
+Add a sponsor button to the repo.

changelog.d/5383.misc

@@ -0,0 +1 @@
+Don't log non-200 responses from federation queries as exceptions.

changelog.d/5384.feature

@@ -0,0 +1 @@
+Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5386.misc

@@ -0,0 +1 @@
+Add a sponsor button to the repo.

changelog.d/5387.bugfix

@@ -0,0 +1 @@
+Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset.

changelog.d/5388.bugfix

@@ -0,0 +1 @@
+Fix email notifications for unnamed rooms with multiple people.

changelog.d/5389.bugfix

@@ -0,0 +1 @@
+Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker.

changelog.d/5390.bugfix

@@ -0,0 +1 @@
+Fix handling of failures fetching remote content to not log failures as exceptions.

changelog.d/5394.bugfix

@@ -0,0 +1 @@
+Fix a bug where deactivated users could receive renewal emails if the account validity feature is on.

changelog.d/5412.feature

@@ -0,0 +1 @@
+Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste.

changelog.d/5425.removal

@@ -0,0 +1 @@
+Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run.

changelog.d/5440.feature

@@ -0,0 +1 @@
+Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5446.misc

@@ -0,0 +1 @@
+Update Python syntax in contrib/ to Python 3. 

changelog.d/5447.misc

@@ -0,0 +1 @@
+Update federation_client dev script to support `.well-known` and work with python3.

changelog.d/5448.removal

@@ -0,0 +1 @@
+PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support.

changelog.d/5458.feature

@@ -0,0 +1 @@
+Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5459.misc

@@ -0,0 +1 @@
+SyTest has been moved to Buildkite.

changelog.d/5460.misc

@@ -0,0 +1 @@
+Demo script now uses python3.

changelog.d/5461.feature

@@ -0,0 +1 @@
+Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5464.bugfix

@@ -0,0 +1 @@
+Fix missing invite state after exchanging 3PID invites over federaton.

changelog.d/5465.misc

@@ -0,0 +1,2 @@
+Track deactivated accounts in the database.
+

changelog.d/5474.feature

@@ -0,0 +1 @@
+Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5477.feature

@@ -0,0 +1 @@
+Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5478.misc

@@ -0,0 +1 @@
+The demo servers talk to each other again.

contrib/cmdclient/console.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
+from __future__ import print_function
 
 from twisted.internet import reactor, defer, threads
 from http import TwistedHttpClient
@@ -109,7 +110,7 @@ class SynapseCmd(cmd.Cmd):
         by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
         """
         if len(line) == 0:
-            print json.dumps(self.config, indent=4)
+            print(json.dumps(self.config, indent=4))
             return
 
         try:
@@ -123,8 +124,8 @@ class SynapseCmd(cmd.Cmd):
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print "%s value must be one of %s" % (args["key"],
-                                                          valid_vals)
+                    print("%s value must be one of %s" % (args["key"],
+                                                          valid_vals))
                     return
 
             # toggle the http client verbosity
@@ -133,11 +134,11 @@ class SynapseCmd(cmd.Cmd):
 
             # assign the new config
             self.config[args["key"]] = args["val"]
-            print json.dumps(self.config, indent=4)
+            print(json.dumps(self.config, indent=4))
 
             save_config(self.config)
         except Exception as e:
-            print e
+            print(e)
 
     def do_register(self, line):
         """Registers for a new account: "register <userid> <noupdate>"
@@ -153,7 +154,7 @@ class SynapseCmd(cmd.Cmd):
             pwd = getpass.getpass("Type a password for this user: ")
             pwd2 = getpass.getpass("Retype the password: ")
             if pwd != pwd2 or len(pwd) == 0:
-                print "Password mismatch."
+                print("Password mismatch.")
                 pwd = None
             else:
                 password = pwd
@@ -174,12 +175,12 @@ class SynapseCmd(cmd.Cmd):
         # check the registration flows
         url = self._url() + "/register"
         json_res = yield self.http_client.do_request("GET", url)
-        print json.dumps(json_res, indent=4)
+        print(json.dumps(json_res, indent=4))
 
         passwordFlow = None
         for flow in json_res["flows"]:
             if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
-                print "Unable to register: Home server requires captcha."
+                print("Unable to register: Home server requires captcha.")
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
                 passwordFlow = flow
@@ -189,7 +190,7 @@ class SynapseCmd(cmd.Cmd):
             return
 
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print json.dumps(json_res, indent=4)
+        print(json.dumps(json_res, indent=4))
         if update_config and "user_id" in json_res:
             self.config["user"] = json_res["user_id"]
             self.config["token"] = json_res["access_token"]
@@ -215,7 +216,7 @@ class SynapseCmd(cmd.Cmd):
                 reactor.callFromThread(self._do_login, user, p)
                 #print " got %s " % p
         except Exception as e:
-            print e
+            print(e)
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
@@ -227,13 +228,13 @@ class SynapseCmd(cmd.Cmd):
         }
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print json_res
+        print(json_res)
 
         if "access_token" in json_res:
             self.config["user"] = user
             self.config["token"] = json_res["access_token"]
             save_config(self.config)
-            print "Login successful."
+            print("Login successful.")
 
     @defer.inlineCallbacks
     def _check_can_login(self):
@@ -242,10 +243,10 @@ class SynapseCmd(cmd.Cmd):
         # submitting!
         url = self._url() + path
         json_res = yield self.http_client.do_request("GET", url)
-        print json_res
+        print(json_res)
 
         if "flows" not in json_res:
-            print "Failed to find any login flows."
+            print("Failed to find any login flows.")
             defer.returnValue(False)
 
         flow = json_res["flows"][0] # assume first is the one we want.
@@ -275,9 +276,9 @@ class SynapseCmd(cmd.Cmd):
 
         json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
                                                      headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
+        print(json_res)
         if 'sid' in json_res:
-            print "Token sent. Your session ID is %s" % (json_res['sid'])
+            print("Token sent. Your session ID is %s" % (json_res['sid']))
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -297,7 +298,7 @@ class SynapseCmd(cmd.Cmd):
 
         json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
                                                      headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
+        print(json_res)
 
     def do_3pidbind(self, line):
         """Validate and associate a third party ID
@@ -317,7 +318,7 @@ class SynapseCmd(cmd.Cmd):
 
         json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
                                                      headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
+        print(json_res)
 
     def do_join(self, line):
         """Joins a room: "join <roomid>" """
@@ -325,7 +326,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "join", self._usr())
         except Exception as e:
-            print e
+            print(e)
 
     def do_joinalias(self, line):
         try:
@@ -333,7 +334,7 @@ class SynapseCmd(cmd.Cmd):
             path = "/join/%s" % urllib.quote(args["roomname"])
             reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
-            print e
+            print(e)
 
     def do_topic(self, line):
         """"topic [set|get] <roomid> [<newtopic>]"
@@ -343,17 +344,17 @@ class SynapseCmd(cmd.Cmd):
         try:
             args = self._parse(line, ["action", "roomid", "topic"])
             if "action" not in args or "roomid" not in args:
-                print "Must specify set|get and a room ID."
+                print("Must specify set|get and a room ID.")
                 return
             if args["action"].lower() not in ["set", "get"]:
-                print "Must specify set|get, not %s" % args["action"]
+                print("Must specify set|get, not %s" % args["action"])
                 return
 
             path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
 
             if args["action"].lower() == "set":
                 if "topic" not in args:
-                    print "Must specify a new topic."
+                    print("Must specify a new topic.")
                     return
                 body = {
                     "topic": args["topic"]
@@ -362,7 +363,7 @@ class SynapseCmd(cmd.Cmd):
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
         except Exception as e:
-            print e
+            print(e)
 
     def do_invite(self, line):
         """Invite a user to a room: "invite <userid> <roomid>" """
@@ -373,7 +374,7 @@ class SynapseCmd(cmd.Cmd):
 
             reactor.callFromThread(self._do_invite, args["roomid"], user_id)
         except Exception as e:
-            print e
+            print(e)
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
@@ -393,29 +394,29 @@ class SynapseCmd(cmd.Cmd):
                 if 'public_key' in pubKeyObj:
                     pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
                 else:
-                    print "No public key found in pubkey response!"
+                    print("No public key found in pubkey response!")
 
                 sigValid = False
 
                 if pubKey:
                     for signame in json_res['signatures']:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print "Ignoring signature from untrusted server %s" % (signame)
+                            print("Ignoring signature from untrusted server %s" % (signame))
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
+                                print("Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame))
                                 break
                             except SignatureVerifyException as e:
-                                print "Invalid signature from %s" % (signame)
-                                print e
+                                print("Invalid signature from %s" % (signame))
+                                print(e)
 
                 if sigValid:
-                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
+                    print("Resolved 3pid %s to %s" % (userstring, json_res['mxid']))
                     mxid = json_res['mxid']
                 else:
-                    print "Got association for %s but couldn't verify signature" % (userstring)
+                    print("Got association for %s but couldn't verify signature" % (userstring))
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -428,7 +429,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "leave", self._usr())
         except Exception as e:
-            print e
+            print(e)
 
     def do_send(self, line):
         """Sends a message. "send <roomid> <body>" """
@@ -453,10 +454,10 @@ class SynapseCmd(cmd.Cmd):
         """
         args = self._parse(line, ["type", "roomid", "qp"])
         if not "type" in args or not "roomid" in args:
-            print "Must specify type and room ID."
+            print("Must specify type and room ID.")
             return
         if args["type"] not in ["members", "messages"]:
-            print "Unrecognised type: %s" % args["type"]
+            print("Unrecognised type: %s" % args["type"])
             return
         room_id = args["roomid"]
         path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
@@ -468,7 +469,7 @@ class SynapseCmd(cmd.Cmd):
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
                 except:
-                    print "Bad query param: %s" % key_value
+                    print("Bad query param: %s" % key_value)
                     return
 
         reactor.callFromThread(self._run_and_pprint, "GET", path,
@@ -508,14 +509,14 @@ class SynapseCmd(cmd.Cmd):
         args = self._parse(line, ["method", "path", "data"])
         # sanity check
         if "method" not in args or "path" not in args:
-            print "Must specify path and method."
+            print("Must specify path and method.")
             return
 
         args["method"] = args["method"].upper()
         valid_methods = ["PUT", "GET", "POST", "DELETE",
                          "XPUT", "XGET", "XPOST", "XDELETE"]
         if args["method"] not in valid_methods:
-            print "Unsupported method: %s" % args["method"]
+            print("Unsupported method: %s" % args["method"])
             return
 
         if "data" not in args:
@@ -524,7 +525,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 args["data"] = json.loads(args["data"])
             except Exception as e:
-                print "Data is not valid JSON. %s" % e
+                print("Data is not valid JSON. %s" % e)
                 return
 
         qp = {"access_token": self._tok()}
@@ -553,7 +554,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 timeout = int(args["timeout"])
             except ValueError:
-                print "Timeout must be in milliseconds."
+                print("Timeout must be in milliseconds.")
                 return
         reactor.callFromThread(self._do_event_stream, timeout)
 
@@ -566,7 +567,7 @@ class SynapseCmd(cmd.Cmd):
                     "timeout": str(timeout),
                     "from": self.event_stream_token
                 })
-        print json.dumps(res, indent=4)
+        print(json.dumps(res, indent=4))
 
         if "chunk" in res:
             for event in res["chunk"]:
@@ -669,9 +670,9 @@ class SynapseCmd(cmd.Cmd):
                                                     data=data,
                                                     qparams=query_params)
         if alt_text:
-            print alt_text
+            print(alt_text)
         else:
-            print json.dumps(json_res, indent=4)
+            print(json.dumps(json_res, indent=4))
 
 
 def save_config(config):
@@ -680,16 +681,16 @@ def save_config(config):
 
 
 def main(server_url, identity_server_url, username, token, config_path):
-    print "Synapse command line client"
-    print "==========================="
-    print "Server: %s" % server_url
-    print "Type 'help' to get started."
-    print "Close this console with CTRL+C then CTRL+D."
+    print("Synapse command line client")
+    print("===========================")
+    print("Server: %s" % server_url)
+    print("Type 'help' to get started.")
+    print("Close this console with CTRL+C then CTRL+D.")
     if not username or not token:
-        print "-  'register <username>' - Register an account"
-        print "-  'stream' - Connect to the event stream"
-        print "-  'create <roomid>' - Create a room"
-        print "-  'send <roomid> <message>' - Send a message"
+        print("-  'register <username>' - Register an account")
+        print("-  'stream' - Connect to the event stream")
+        print("-  'create <roomid>' - Create a room")
+        print("-  'send <roomid> <message>' - Send a message")
     http_client = TwistedHttpClient()
 
     # the command line client
@@ -705,7 +706,7 @@ def main(server_url, identity_server_url, username, token, config_path):
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
             except:
                 pass
-            print "Loaded config from %s" % config_path
+            print("Loaded config from %s" % config_path)
     except:
         pass
 
@@ -736,7 +737,7 @@ if __name__ == '__main__':
     args = parser.parse_args()
 
     if not args.server:
-        print "You must supply a server URL to communicate with."
+        print("You must supply a server URL to communicate with.")
         parser.print_help()
         sys.exit(1)
 

contrib/cmdclient/http.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
 from twisted.internet import defer, reactor
@@ -141,15 +142,15 @@ class TwistedHttpClient(HttpClient):
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
-        print "%s to %s with headers %s" % (method, url, headers_dict)
+        print("%s to %s with headers %s" % (method, url, headers_dict))
         if self.verbose and producer:
             if "password" in producer.data:
                 temp = producer.data["password"]
                 producer.data["password"] = "[REDACTED]"
-                print json.dumps(producer.data, indent=4)
+                print(json.dumps(producer.data, indent=4))
                 producer.data["password"] = temp
             else:
-                print json.dumps(producer.data, indent=4)
+                print(json.dumps(producer.data, indent=4))
 
         while True:
             try:
@@ -161,7 +162,7 @@ class TwistedHttpClient(HttpClient):
                 )
                 break
             except Exception as e:
-                print "uh oh: %s" % e
+                print("uh oh: %s" % e)
                 if retries_left:
                     yield self.sleep(2 ** (5 - retries_left))
                     retries_left -= 1
@@ -169,8 +170,8 @@ class TwistedHttpClient(HttpClient):
                     raise e
 
         if self.verbose:
-            print "Status %s %s" % (response.code, response.phrase)
-            print pformat(list(response.headers.getAllRawHeaders()))
+            print("Status %s %s" % (response.code, response.phrase))
+            print(pformat(list(response.headers.getAllRawHeaders())))
         defer.returnValue(response)
 
     def sleep(self, seconds):

contrib/graph/graph.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -48,7 +49,7 @@ def make_graph(pdus, room, filename_prefix):
             c = colors.pop()
             color_map[o] = c
         except:
-            print "Run out of colours!"
+            print("Run out of colours!")
             color_map[o] = "black"
 
     graph = pydot.Dot(graph_name="Test")
@@ -93,7 +94,7 @@ def make_graph(pdus, room, filename_prefix):
             end_name = make_name(i, o)
 
             if end_name not in node_map:
-                print "%s not in nodes" % end_name
+                print("%s not in nodes" % end_name)
                 continue
 
             edge = pydot.Edge(node_map[start_name], node_map[end_name])

contrib/graph/graph3.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,19 +27,19 @@ from six import string_types
 
 
 def make_graph(file_name, room_id, file_prefix, limit):
-    print "Reading lines"
+    print("Reading lines")
     with open(file_name) as f:
         lines = f.readlines()
 
-    print "Read lines"
+    print("Read lines")
 
     events = [FrozenEvent(json.loads(line)) for line in lines]
 
-    print "Loaded events."
+    print("Loaded events.")
 
     events.sort(key=lambda e: e.depth)
 
-    print "Sorted events"
+    print("Sorted events")
 
     if limit:
         events = events[-int(limit):]
@@ -55,7 +56,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4)
         content = content.replace("\n", "<br/>\n")
 
-        print content
+        print(content)
         content = []
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
@@ -74,7 +75,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
 
         content = "<br/>\n".join(content)
 
-        print content
+        print(content)
 
         label = (
             "<"
@@ -102,7 +103,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         node_map[event.event_id] = node
         graph.add_node(node)
 
-    print "Created Nodes"
+    print("Created Nodes")
 
     for event in events:
         for prev_id, _ in event.prev_events:
@@ -120,15 +121,15 @@ def make_graph(file_name, room_id, file_prefix, limit):
             edge = pydot.Edge(node_map[event.event_id], end_node)
             graph.add_edge(edge)
 
-    print "Created edges"
+    print("Created edges")
 
     graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
 
-    print "Created Dot"
+    print("Created Dot")
 
     graph.write_svg("%s.svg" % file_prefix, prog='dot')
 
-    print "Created svg"
+    print("Created svg")
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -8,8 +8,9 @@ we set the remote SDP at which point the stream ends. Our video never gets to
 the bridge.
 
 Requires:
-npm install jquery jsdom 
+npm install jquery jsdom
 """
+from __future__ import print_function
 
 import gevent
 import grequests
@@ -51,7 +52,7 @@ class TrivialMatrixClient:
             req = grequests.get(url)
             resps = grequests.map([req])
             obj = json.loads(resps[0].content)
-            print "incoming from matrix",obj
+            print("incoming from matrix",obj)
             if 'end' not in obj:
                 continue
             self.token = obj['end']
@@ -60,22 +61,22 @@ class TrivialMatrixClient:
 
     def joinRoom(self, roomId):
         url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
-        print url
+        print(url)
         headers={ 'Content-Type': 'application/json' }
         req = grequests.post(url, headers=headers, data='{}')
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print "response: ",obj
+        print("response: ",obj)
 
     def sendEvent(self, roomId, evType, event):
         url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
-        print url
-        print json.dumps(event)
+        print(url)
+        print(json.dumps(event))
         headers={ 'Content-Type': 'application/json' }
         req = grequests.post(url, headers=headers, data=json.dumps(event))
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print "response: ",obj
+        print("response: ",obj)
 
 
 
@@ -85,31 +86,31 @@ xmppClients = {}
 def matrixLoop():
     while True:
         ev = matrixCli.getEvent()
-        print ev
+        print(ev)
         if ev['type'] == 'm.room.member':
-            print 'membership event'
+            print('membership event')
             if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME:
                 roomId = ev['room_id']
-                print "joining room %s" % (roomId)
+                print("joining room %s" % (roomId))
                 matrixCli.joinRoom(roomId)
         elif ev['type'] == 'm.room.message':
             if ev['room_id'] in xmppClients:
-                print "already have a bridge for that user, ignoring"
+                print("already have a bridge for that user, ignoring")
                 continue
-            print "got message, connecting"
+            print("got message, connecting")
             xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
             gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
         elif ev['type'] == 'm.call.invite':
-            print "Incoming call"
+            print("Incoming call")
             #sdp = ev['content']['offer']['sdp']
             #print "sdp: %s" % (sdp)
             #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
             #gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
         elif ev['type'] == 'm.call.answer':
-            print "Call answered"
+            print("Call answered")
             sdp = ev['content']['answer']['sdp']
             if ev['room_id'] not in xmppClients:
-                print "We didn't have a call for that room"
+                print("We didn't have a call for that room")
                 continue
             # should probably check call ID too
             xmppCli = xmppClients[ev['room_id']]
@@ -146,7 +147,7 @@ class TrivialXmppClient:
         return obj
 
     def sendAnswer(self, answer):
-        print "sdp from matrix client",answer
+        print("sdp from matrix client",answer)
         p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
         jingle, out_err = p.communicate(answer)
         jingle = jingle % {
@@ -156,28 +157,28 @@ class TrivialXmppClient:
             'responder': self.jid,
             'sid': self.callsid
         }
-        print "answer jingle from sdp",jingle
+        print("answer jingle from sdp",jingle)
         res = self.sendIq(jingle)
-        print "reply from answer: ",res
+        print("reply from answer: ",res)
 
         self.ssrcs = {}
         jingleSoup = BeautifulSoup(jingle)
         for cont in jingleSoup.iq.jingle.findAll('content'):
             if cont.description:
                 self.ssrcs[cont['name']] = cont.description['ssrc']
-        print "my ssrcs:",self.ssrcs
+        print("my ssrcs:",self.ssrcs)
 
         gevent.joinall([
                 gevent.spawn(self.advertiseSsrcs)
         ])
 
     def advertiseSsrcs(self):
-                time.sleep(7)
-        print "SSRC spammer started"
+        time.sleep(7)
+        print("SSRC spammer started")
         while self.running:
             ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] }
             res = self.sendIq(ssrcMsg)
-            print "reply from ssrc announce: ",res
+            print("reply from ssrc announce: ",res)
             time.sleep(10)
 
 
@@ -186,19 +187,19 @@ class TrivialXmppClient:
         self.matrixCallId = time.time()
         res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST))
 
-        print res
+        print(res)
         self.sid = res.body['sid']
-        print "sid %s" % (self.sid)
+        print("sid %s" % (self.sid))
 
         res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>")
 
         res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST))
 
         res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>")
-        print res
+        print(res)
 
         self.jid = res.body.iq.bind.jid.string
-        print "jid: %s" % (self.jid)
+        print("jid: %s" % (self.jid))
         self.shortJid = self.jid.split('-')[0]
 
         res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>")
@@ -217,13 +218,13 @@ class TrivialXmppClient:
             if p.c and p.c.nick:
                 u['nick'] = p.c.nick.string
             self.muc['users'].append(u)
-        print "muc: ",self.muc
+        print("muc: ",self.muc)
 
         # wait for stuff
         while True:
-            print "waiting..."
+            print("waiting...")
             res = self.sendIq("")
-            print "got from stream: ",res
+            print("got from stream: ",res)
             if res.body.iq:
                 jingles = res.body.iq.findAll('jingle')
                 if len(jingles):
@@ -232,15 +233,15 @@ class TrivialXmppClient:
             elif 'type' in res.body and res.body['type'] == 'terminate':
                 self.running = False
                 del xmppClients[self.matrixRoom]
-                        return
+                return
 
     def handleInvite(self, jingle):
         self.initiator = jingle['initiator']
         self.callsid = jingle['sid']
         p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        print "raw jingle invite",str(jingle)
+        print("raw jingle invite",str(jingle))
         sdp, out_err = p.communicate(str(jingle))
-        print "transformed remote offer sdp",sdp
+        print("transformed remote offer sdp",sdp)
         inviteEvent = {
             'offer': {
                 'type': 'offer',
@@ -252,7 +253,7 @@ class TrivialXmppClient:
         }
         matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent)
 
-matrixCli = TrivialMatrixClient(ACCESS_TOKEN)
+matrixCli = TrivialMatrixClient(ACCESS_TOKEN)  # Undefined name
 
 gevent.joinall([
     gevent.spawn(matrixLoop)

contrib/scripts/kick_users.py

@@ -1,10 +1,16 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from argparse import ArgumentParser
 import json
 import requests
 import sys
 import urllib
 
+try:
+    raw_input
+except NameError:  # Python 3
+    raw_input = input
+
 def _mkurl(template, kws):
     for key in kws:
         template = template.replace(key, kws[key])
@@ -13,7 +19,7 @@ def _mkurl(template, kws):
 def main(hs, room_id, access_token, user_id_prefix, why):
     if not why:
         why = "Automated kick."
-    print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
+    print("Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix))
     room_state_url = _mkurl(
         "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN",
         {
@@ -22,13 +28,13 @@ def main(hs, room_id, access_token, user_id_prefix, why):
             "$TOKEN": access_token
         }
     )
-    print "Getting room state => %s" % room_state_url
+    print("Getting room state => %s" % room_state_url)
     res = requests.get(room_state_url)
-    print "HTTP %s" % res.status_code
+    print("HTTP %s" % res.status_code)
     state_events = res.json()
     if "error" in state_events:
-        print "FATAL"
-        print state_events
+        print("FATAL")
+        print(state_events)
         return
 
     kick_list = []
@@ -44,15 +50,15 @@ def main(hs, room_id, access_token, user_id_prefix, why):
             kick_list.append(event["state_key"])
 
     if len(kick_list) == 0:
-        print "No user IDs match the prefix '%s'" % user_id_prefix
+        print("No user IDs match the prefix '%s'" % user_id_prefix)
         return
 
-    print "The following user IDs will be kicked from %s" % room_name
+    print("The following user IDs will be kicked from %s" % room_name)
     for uid in kick_list:
-        print uid
+        print(uid)
     doit = raw_input("Continue? [Y]es\n")
     if len(doit) > 0 and doit.lower() == 'y':
-        print "Kicking members..."
+        print("Kicking members...")
         # encode them all
         kick_list = [urllib.quote(uid) for uid in kick_list]
         for uid in kick_list:
@@ -69,14 +75,14 @@ def main(hs, room_id, access_token, user_id_prefix, why):
                 "membership": "leave",
                 "reason": why
             }
-            print "Kicking %s" % uid
+            print("Kicking %s" % uid)
             res = requests.put(kick_url, data=json.dumps(kick_body))
             if res.status_code != 200:
-                print "ERROR: HTTP %s" % res.status_code
+                print("ERROR: HTTP %s" % res.status_code)
             if res.json().get("error"):
-                print "ERROR: JSON %s" % res.json()
-            
-    
+                print("ERROR: JSON %s" % res.json())
+
+
 
 if __name__ == "__main__":
     parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.")

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -12,6 +12,7 @@ ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.%i --config-path=/
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
+SyslogIdentifier=matrix-synapse-%i
 
 [Install]
 WantedBy=matrix-synapse.service

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -11,6 +11,7 @@ ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --confi
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
+SyslogIdentifier=matrix-synapse
 
 [Install]
 WantedBy=matrix.target

contrib/systemd/matrix-synapse.service

@@ -22,10 +22,10 @@ Group=nogroup
 
 WorkingDirectory=/opt/synapse
 ExecStart=/opt/synapse/env/bin/python -m synapse.app.homeserver --config-path=/opt/synapse/homeserver.yaml
+SyslogIdentifier=matrix-synapse
 
 # adjust the cache factor if necessary
 # Environment=SYNAPSE_CACHE_FACTOR=2.0
 
 [Install]
 WantedBy=multi-user.target
-

debian/changelog

@@ -1,3 +1,43 @@
+matrix-synapse-py3 (1.0.0) stable; urgency=medium
+
+  * New synapse release 1.0.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jun 2019 17:09:53 +0100
+
+matrix-synapse-py3 (0.99.5.2) stable; urgency=medium
+
+  * New synapse release 0.99.5.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 30 May 2019 16:28:07 +0100
+
+matrix-synapse-py3 (0.99.5.1) stable; urgency=medium
+
+  * New synapse release 0.99.5.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 22 May 2019 16:22:24 +0000
+
+matrix-synapse-py3 (0.99.4) stable; urgency=medium
+
+  [ Christoph Müller ]
+  * Configure the systemd units to have a log identifier of `matrix-synapse`
+
+  [ Synapse Packaging team ]
+  * New synapse release 0.99.4.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 15 May 2019 13:58:08 +0100
+
+matrix-synapse-py3 (0.99.3.2) stable; urgency=medium
+
+  * New synapse release 0.99.3.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 03 May 2019 18:56:20 +0100
+
+matrix-synapse-py3 (0.99.3.1) stable; urgency=medium
+
+  * New synapse release 0.99.3.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 03 May 2019 16:02:43 +0100
+
 matrix-synapse-py3 (0.99.3) stable; urgency=medium
 
   [ Richard van der Hoff ]

debian/matrix-synapse.service

@@ -11,6 +11,7 @@ ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --confi
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
+SyslogIdentifier=matrix-synapse
 
 [Install]
 WantedBy=multi-user.target

debian/test/.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+*.log

debian/test/provision.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# provisioning script for vagrant boxes for testing the matrix-synapse debs.
+#
+# Will install the most recent matrix-synapse-py3 deb for this platform from
+# the /debs directory.
+
+set -e
+
+apt-get update
+apt-get install -y lsb-release
+
+deb=`ls /debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb | sort | tail -n1`
+
+debconf-set-selections <<EOF
+matrix-synapse matrix-synapse/report-stats boolean false
+matrix-synapse matrix-synapse/server-name string localhost:18448
+EOF
+
+dpkg -i "$deb"
+
+sed -i -e '/port: 8...$/{s/8448/18448/; s/8008/18008/}' -e '$aregistration_shared_secret: secret' /etc/matrix-synapse/homeserver.yaml
+systemctl restart matrix-synapse

debian/test/stretch/Vagrantfile

@@ -0,0 +1,13 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+ver = `cd ../../..; dpkg-parsechangelog -S Version`.strip()
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "debian/stretch64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs", type: "nfs"
+
+  config.vm.provision "shell", path: "../provision.sh"
+end

debian/test/xenial/Vagrantfile

@@ -0,0 +1,10 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs"
+  config.vm.provision "shell", path: "../provision.sh"
+end

demo/README

@@ -1,9 +1,13 @@
+DO NOT USE THESE DEMO SERVERS IN PRODUCTION
+
 Requires you to have done:
     python setup.py develop
 
 
-The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required. 
-It will also start a web server on port 8000 pointed at the webclient.
+The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
+
+To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
+and are configured in a highly insecure way. Do not use these configuration files in production.
 
 stop.sh will stop the synapse servers and the webclient.
 

demo/start.sh

@@ -21,14 +21,76 @@ for port in 8080 8081 8082; do
     pushd demo/$port
 
     #rm $DIR/etc/$port.config
-    python -m synapse.app.homeserver \
+    python3 -m synapse.app.homeserver \
         --generate-config \
         -H "localhost:$https_port" \
         --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
-    printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-    echo 'enable_registration: true' >> $DIR/etc/$port.config
+    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
+        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
+        
+        echo 'enable_registration: true' >> $DIR/etc/$port.config
+
+        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
+        # accidentaly bork me with your fancy settings.
+		listeners=$(cat <<-PORTLISTENERS
+		# Configure server to listen on both $https_port and $port
+		# This overides some of the default settings above
+		listeners:
+		  - port: $https_port
+		    type: http
+		    tls: true
+		    resources:
+		      - names: [client, federation]
+		
+		  - port: $port
+		    tls: false
+		    bind_addresses: ['::1', '127.0.0.1']
+		    type: http
+		    x_forwarded: true
+		    resources:
+		      - names: [client, federation]
+		        compress: false
+		PORTLISTENERS
+		)
+        echo "${listeners}" >> $DIR/etc/$port.config
+
+        # Disable tls for the servers
+        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
+        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
+        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
+        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
+
+        # Set tls paths
+        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
+        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
+
+        # Generate tls keys
+        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
+        
+        # Ignore keys from the trusted keys server
+        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
+        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
+        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
+        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
+
+        # Reduce the blacklist
+        blacklist=$(cat <<-BLACK
+		# Set the blacklist so that it doesn't include 127.0.0.1
+		federation_ip_range_blacklist:
+		  - '10.0.0.0/8'
+		  - '172.16.0.0/12'
+		  - '192.168.0.0/16'
+		  - '100.64.0.0/10'
+		  - '169.254.0.0/16'
+		  - '::1/128'
+		  - 'fe80::/64'
+		  - 'fc00::/7'
+		BLACK
+		)
+        echo "${blacklist}" >> $DIR/etc/$port.config
+    fi
 
     # Check script parameters
     if [ $# -eq 1 ]; then
@@ -55,7 +117,7 @@ for port in 8080 8081 8082; do
         echo "report_stats: false" >> $DIR/etc/$port.config
     fi
 
-    python -m synapse.app.homeserver \
+    python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
         -vv \

docker/Dockerfile-dhvirtualenv

@@ -50,12 +50,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         debhelper \
         devscripts \
         dh-systemd \
+        libsystemd-dev \
         lsb-release \
+        pkg-config \
         python3-dev \
         python3-pip \
         python3-setuptools \
         python3-venv \
-        sqlite3
+        sqlite3 \
+        libpq-dev
 
 COPY --from=builder /dh-virtualenv_1.1-1_all.deb /