0
0
Fork 1
mirror of https://mau.dev/maunium/synapse.git synced 2024-12-24 11:34:16 +01:00

Merge branch 'develop' into travis/login-terms

This commit is contained in:
Travis Ralston 2018-10-18 09:57:58 -06:00
commit 49a044aa5f
43 changed files with 308 additions and 163 deletions

View file

@ -23,99 +23,106 @@ jobs:
- run: docker push matrixdotorg/synapse:latest - run: docker push matrixdotorg/synapse:latest
- run: docker push matrixdotorg/synapse:latest-py3 - run: docker push matrixdotorg/synapse:latest-py3
sytestpy2: sytestpy2:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy2
working_directory: /src
steps: steps:
- checkout - checkout
- run: docker pull matrixdotorg/sytest-synapsepy2 - run: /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
sytestpy2postgres: sytestpy2postgres:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy2
working_directory: /src
steps: steps:
- checkout - checkout
- run: docker pull matrixdotorg/sytest-synapsepy2 - run: POSTGRES=1 /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
sytestpy2merged: sytestpy2merged:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy2
working_directory: /src
steps: steps:
- checkout - checkout
- run: bash .circleci/merge_base_branch.sh - run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy2 - run: /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
sytestpy2postgresmerged: sytestpy2postgresmerged:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy2
working_directory: /src
steps: steps:
- checkout - checkout
- run: bash .circleci/merge_base_branch.sh - run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy2 - run: POSTGRES=1 /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
sytestpy3: sytestpy3:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy3
working_directory: /src
steps: steps:
- checkout - checkout
- run: docker pull matrixdotorg/sytest-synapsepy3 - run: /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
sytestpy3postgres: sytestpy3postgres:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy3
working_directory: /src
steps: steps:
- checkout - checkout
- run: docker pull matrixdotorg/sytest-synapsepy3 - run: POSTGRES=1 /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
sytestpy3merged: sytestpy3merged:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy3
working_directory: /src
steps: steps:
- checkout - checkout
- run: bash .circleci/merge_base_branch.sh - run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy3 - run: /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
sytestpy3postgresmerged: sytestpy3postgresmerged:
machine: true docker:
- image: matrixdotorg/sytest-synapsepy3
working_directory: /src
steps: steps:
- checkout - checkout
- run: bash .circleci/merge_base_branch.sh - run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy3 - run: POSTGRES=1 /synapse_sytest.sh
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
- store_artifacts: - store_artifacts:
path: ~/project/logs path: /logs
destination: logs destination: logs
- store_test_results: - store_test_results:
path: logs path: /logs
workflows: workflows:
version: 2 version: 2

View file

@ -16,7 +16,7 @@ then
GITBASE="develop" GITBASE="develop"
else else
# Get the reference, using the GitHub API # Get the reference, using the GitHub API
GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'` GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
fi fi
# Show what we are before # Show what we are before

View file

@ -1,3 +1,62 @@
Synapse 0.33.7 (2018-10-18)
===========================
**Warning**: This release removes the example email notification templates from
`res/templates` (they are now internal to the python package). This should only
affect you if you (a) deploy your Synapse instance from a git checkout or a
github snapshot URL, and (b) have email notifications enabled.
If you have email notifications enabled, you should ensure that
`email.template_dir` is either configured to point at a directory where you
have installed customised templates, or leave it unset to use the default
templates.
The configuration parser will try to detect the situation where
`email.template_dir` is incorrectly set to `res/templates` and do the right
thing, but will warn about this.
Synapse 0.33.7rc2 (2018-10-17)
==============================
Features
--------
- Ship the example email templates as part of the package ([\#4052](https://github.com/matrix-org/synapse/issues/4052))
Bugfixes
--------
- Fix bug which made get_missing_events return too few events ([\#4045](https://github.com/matrix-org/synapse/issues/4045))
Synapse 0.33.7rc1 (2018-10-15)
==============================
Features
--------
- Add support for end-to-end key backup (MSC1687) ([\#4019](https://github.com/matrix-org/synapse/issues/4019))
Bugfixes
--------
- Fix bug in event persistence logic which caused 'NoneType is not iterable' ([\#3995](https://github.com/matrix-org/synapse/issues/3995))
- Fix exception in background metrics collection ([\#3996](https://github.com/matrix-org/synapse/issues/3996))
- Fix exception handling in fetching remote profiles ([\#3997](https://github.com/matrix-org/synapse/issues/3997))
- Fix handling of rejected threepid invites ([\#3999](https://github.com/matrix-org/synapse/issues/3999))
- Workers now start on Python 3. ([\#4027](https://github.com/matrix-org/synapse/issues/4027))
- Synapse now starts on Python 3.7. ([\#4033](https://github.com/matrix-org/synapse/issues/4033))
Internal Changes
----------------
- Log exceptions in looping calls ([\#4008](https://github.com/matrix-org/synapse/issues/4008))
- Optimisation for serving federation requests ([\#4017](https://github.com/matrix-org/synapse/issues/4017))
- Add metric to count number of non-empty sync responses ([\#4022](https://github.com/matrix-org/synapse/issues/4022))
Synapse 0.33.6 (2018-10-04) Synapse 0.33.6 (2018-10-04)
=========================== ===========================

View file

@ -12,12 +12,12 @@ recursive-include synapse/storage/schema *.sql
recursive-include synapse/storage/schema *.py recursive-include synapse/storage/schema *.py
recursive-include docs * recursive-include docs *
recursive-include res *
recursive-include scripts * recursive-include scripts *
recursive-include scripts-dev * recursive-include scripts-dev *
recursive-include synapse *.pyi recursive-include synapse *.pyi
recursive-include tests *.py recursive-include tests *.py
recursive-include synapse/res *
recursive-include synapse/static *.css recursive-include synapse/static *.css
recursive-include synapse/static *.gif recursive-include synapse/static *.gif
recursive-include synapse/static *.html recursive-include synapse/static *.html

View file

@ -1 +0,0 @@
Fix bug in event persistence logic which caused 'NoneType is not iterable'

View file

@ -1 +0,0 @@
Fix exception in background metrics collection

View file

@ -1 +0,0 @@
Fix exception handling in fetching remote profiles

View file

@ -1 +0,0 @@
Fix handling of rejected threepid invites

View file

@ -1 +0,0 @@
Log exceptions in looping calls

View file

@ -1 +0,0 @@
Optimisation for serving federation requests

View file

@ -1 +0,0 @@
Add support for end-to-end key backup (MSC1687)

View file

@ -1 +0,0 @@
Add metric to count number of non-empty sync responses

View file

@ -1 +0,0 @@
Workers now start on Python 3.

1
changelog.d/4031.misc Normal file
View file

@ -0,0 +1 @@
Various cleanups in the federation client code

View file

@ -1,2 +0,0 @@
Synapse now starts on Python 3.7.
_All_ workers now start on Python 3.

1
changelog.d/4041.misc Normal file
View file

@ -0,0 +1 @@
Run the CircleCI builds in docker containers

1
changelog.d/4046.bugfix Normal file
View file

@ -0,0 +1 @@
Fix issue where Python 3 users couldn't paginate /publicRooms

1
changelog.d/4049.misc Normal file
View file

@ -0,0 +1 @@
Only colourise synctl output when attached to tty

1
changelog.d/4050.bugfix Normal file
View file

@ -0,0 +1 @@
Fix URL priewing to work in Python 3.7

View file

@ -211,7 +211,9 @@ email:
require_transport_security: False require_transport_security: False
notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}" notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
app_name: Matrix app_name: Matrix
template_dir: res/templates # if template_dir is unset, uses the example templates that are part of
# the Synapse distribution.
#template_dir: res/templates
notif_template_html: notif_mail.html notif_template_html: notif_mail.html
notif_template_text: notif_mail.txt notif_template_text: notif_mail.txt
notif_for_new_users: True notif_for_new_users: True

View file

@ -27,4 +27,4 @@ try:
except ImportError: except ImportError:
pass pass
__version__ = "0.33.6" __version__ = "0.33.7"

View file

@ -13,11 +13,25 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from __future__ import print_function
# This file can't be called email.py because if it is, we cannot: # This file can't be called email.py because if it is, we cannot:
import email.utils import email.utils
import logging
import os
import sys
import textwrap
from ._base import Config from ._base import Config
logger = logging.getLogger(__name__)
TEMPLATE_DIR_WARNING = """\
WARNING: The email notifier is configured to look for templates in '%(template_dir)s',
but no templates could be found there. We will fall back to using the example templates;
to get rid of this warning, leave 'email.template_dir' unset.
"""
class EmailConfig(Config): class EmailConfig(Config):
def read_config(self, config): def read_config(self, config):
@ -38,7 +52,6 @@ class EmailConfig(Config):
"smtp_host", "smtp_host",
"smtp_port", "smtp_port",
"notif_from", "notif_from",
"template_dir",
"notif_template_html", "notif_template_html",
"notif_template_text", "notif_template_text",
] ]
@ -62,9 +75,24 @@ class EmailConfig(Config):
self.email_smtp_host = email_config["smtp_host"] self.email_smtp_host = email_config["smtp_host"]
self.email_smtp_port = email_config["smtp_port"] self.email_smtp_port = email_config["smtp_port"]
self.email_notif_from = email_config["notif_from"] self.email_notif_from = email_config["notif_from"]
self.email_template_dir = email_config["template_dir"]
self.email_notif_template_html = email_config["notif_template_html"] self.email_notif_template_html = email_config["notif_template_html"]
self.email_notif_template_text = email_config["notif_template_text"] self.email_notif_template_text = email_config["notif_template_text"]
self.email_template_dir = email_config.get("template_dir")
# backwards-compatibility hack
if (
self.email_template_dir == "res/templates"
and not os.path.isfile(
os.path.join(self.email_template_dir, self.email_notif_template_text)
)
):
t = TEMPLATE_DIR_WARNING % {
"template_dir": self.email_template_dir,
}
print(textwrap.fill(t, width=80) + "\n", file=sys.stderr)
self.email_template_dir = None
self.email_notif_for_new_users = email_config.get( self.email_notif_for_new_users = email_config.get(
"notif_for_new_users", True "notif_for_new_users", True
) )
@ -113,7 +141,9 @@ class EmailConfig(Config):
# require_transport_security: False # require_transport_security: False
# notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>" # notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
# app_name: Matrix # app_name: Matrix
# template_dir: res/templates # # if template_dir is unset, uses the example templates that are part of
# # the Synapse distribution.
# #template_dir: res/templates
# notif_template_html: notif_mail.html # notif_template_html: notif_mail.html
# notif_template_text: notif_mail.txt # notif_template_text: notif_mail.txt
# notif_for_new_users: True # notif_for_new_users: True

View file

@ -507,19 +507,19 @@ class FederationServer(FederationBase):
@defer.inlineCallbacks @defer.inlineCallbacks
@log_function @log_function
def on_get_missing_events(self, origin, room_id, earliest_events, def on_get_missing_events(self, origin, room_id, earliest_events,
latest_events, limit, min_depth): latest_events, limit):
with (yield self._server_linearizer.queue((origin, room_id))): with (yield self._server_linearizer.queue((origin, room_id))):
origin_host, _ = parse_server_name(origin) origin_host, _ = parse_server_name(origin)
yield self.check_server_matches_acl(origin_host, room_id) yield self.check_server_matches_acl(origin_host, room_id)
logger.info( logger.info(
"on_get_missing_events: earliest_events: %r, latest_events: %r," "on_get_missing_events: earliest_events: %r, latest_events: %r,"
" limit: %d, min_depth: %d", " limit: %d",
earliest_events, latest_events, limit, min_depth earliest_events, latest_events, limit,
) )
missing_events = yield self.handler.on_get_missing_events( missing_events = yield self.handler.on_get_missing_events(
origin, room_id, earliest_events, latest_events, limit, min_depth origin, room_id, earliest_events, latest_events, limit,
) )
if len(missing_events) < 5: if len(missing_events) < 5:

View file

@ -633,14 +633,6 @@ class TransactionQueue(object):
transaction, json_data_cb transaction, json_data_cb
) )
code = 200 code = 200
if response:
for e_id, r in response.get("pdus", {}).items():
if "error" in r:
logger.warn(
"Transaction returned error for %s: %s",
e_id, r,
)
except HttpResponseException as e: except HttpResponseException as e:
code = e.code code = e.code
response = e.response response = e.response
@ -657,19 +649,24 @@ class TransactionQueue(object):
destination, txn_id, code destination, txn_id, code
) )
logger.debug("TX [%s] Sent transaction", destination)
logger.debug("TX [%s] Marking as delivered...", destination)
yield self.transaction_actions.delivered( yield self.transaction_actions.delivered(
transaction, code, response transaction, code, response
) )
logger.debug("TX [%s] Marked as delivered", destination) logger.debug("TX [%s] {%s} Marked as delivered", destination, txn_id)
if code != 200: if code == 200:
for e_id, r in response.get("pdus", {}).items():
if "error" in r:
logger.warn(
"TX [%s] {%s} Remote returned error for %s: %s",
destination, txn_id, e_id, r,
)
else:
for p in pdus: for p in pdus:
logger.info( logger.warn(
"Failed to send event %s to %s", p.event_id, destination "TX [%s] {%s} Failed to send event %s",
destination, txn_id, p.event_id,
) )
success = False success = False

View file

@ -143,9 +143,17 @@ class TransportLayerClient(object):
transaction (Transaction) transaction (Transaction)
Returns: Returns:
Deferred: Results of the deferred is a tuple in the form of Deferred: Succeeds when we get a 2xx HTTP response. The result
(response_code, response_body) where the response_body is a will be the decoded JSON body.
python dict decoded from json
Fails with ``HTTPRequestException`` if we get an HTTP response
code >= 300.
Fails with ``NotRetryingDestination`` if we are not yet ready
to retry this server.
Fails with ``FederationDeniedError`` if this destination
is not on our federation whitelist
""" """
logger.debug( logger.debug(
"send_data dest=%s, txid=%s", "send_data dest=%s, txid=%s",
@ -170,11 +178,6 @@ class TransportLayerClient(object):
backoff_on_404=True, # If we get a 404 the other side has gone backoff_on_404=True, # If we get a 404 the other side has gone
) )
logger.debug(
"send_data dest=%s, txid=%s, got response: 200",
transaction.destination, transaction.transaction_id,
)
defer.returnValue(response) defer.returnValue(response)
@defer.inlineCallbacks @defer.inlineCallbacks

View file

@ -560,7 +560,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet):
@defer.inlineCallbacks @defer.inlineCallbacks
def on_POST(self, origin, content, query, room_id): def on_POST(self, origin, content, query, room_id):
limit = int(content.get("limit", 10)) limit = int(content.get("limit", 10))
min_depth = int(content.get("min_depth", 0))
earliest_events = content.get("earliest_events", []) earliest_events = content.get("earliest_events", [])
latest_events = content.get("latest_events", []) latest_events = content.get("latest_events", [])
@ -569,7 +568,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet):
room_id=room_id, room_id=room_id,
earliest_events=earliest_events, earliest_events=earliest_events,
latest_events=latest_events, latest_events=latest_events,
min_depth=min_depth,
limit=limit, limit=limit,
) )

View file

@ -309,8 +309,8 @@ class FederationHandler(BaseHandler):
if sent_to_us_directly: if sent_to_us_directly:
logger.warn( logger.warn(
"[%s %s] Failed to fetch %d prev events: rejecting", "[%s %s] Rejecting: failed to fetch %d prev events: %s",
room_id, event_id, len(prevs - seen), room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
) )
raise FederationError( raise FederationError(
"ERROR", "ERROR",
@ -452,8 +452,8 @@ class FederationHandler(BaseHandler):
latest |= seen latest |= seen
logger.info( logger.info(
"[%s %s]: Requesting %d prev_events: %s", "[%s %s]: Requesting missing events between %s and %s",
room_id, event_id, len(prevs - seen), shortstr(prevs - seen) room_id, event_id, shortstr(latest), event_id,
) )
# XXX: we set timeout to 10s to help workaround # XXX: we set timeout to 10s to help workaround
@ -1852,7 +1852,7 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks @defer.inlineCallbacks
def on_get_missing_events(self, origin, room_id, earliest_events, def on_get_missing_events(self, origin, room_id, earliest_events,
latest_events, limit, min_depth): latest_events, limit):
in_room = yield self.auth.check_host_in_room( in_room = yield self.auth.check_host_in_room(
room_id, room_id,
origin origin
@ -1861,14 +1861,12 @@ class FederationHandler(BaseHandler):
raise AuthError(403, "Host not in room.") raise AuthError(403, "Host not in room.")
limit = min(limit, 20) limit = min(limit, 20)
min_depth = max(min_depth, 0)
missing_events = yield self.store.get_missing_events( missing_events = yield self.store.get_missing_events(
room_id=room_id, room_id=room_id,
earliest_events=earliest_events, earliest_events=earliest_events,
latest_events=latest_events, latest_events=latest_events,
limit=limit, limit=limit,
min_depth=min_depth,
) )
missing_events = yield filter_events_for_server( missing_events = yield filter_events_for_server(

View file

@ -16,7 +16,7 @@
import logging import logging
from collections import namedtuple from collections import namedtuple
from six import iteritems from six import PY3, iteritems
from six.moves import range from six.moves import range
import msgpack import msgpack
@ -444,9 +444,16 @@ class RoomListNextBatch(namedtuple("RoomListNextBatch", (
@classmethod @classmethod
def from_token(cls, token): def from_token(cls, token):
if PY3:
# The argument raw=False is only available on new versions of
# msgpack, and only really needed on Python 3. Gate it behind
# a PY3 check to avoid causing issues on Debian-packaged versions.
decoded = msgpack.loads(decode_base64(token), raw=False)
else:
decoded = msgpack.loads(decode_base64(token))
return RoomListNextBatch(**{ return RoomListNextBatch(**{
cls.REVERSE_KEY_DICT[key]: val cls.REVERSE_KEY_DICT[key]: val
for key, val in msgpack.loads(decode_base64(token)).items() for key, val in decoded.items()
}) })
def to_token(self): def to_token(self):

View file

@ -195,7 +195,7 @@ class MatrixFederationHttpClient(object):
) )
self.clock = hs.get_clock() self.clock = hs.get_clock()
self._store = hs.get_datastore() self._store = hs.get_datastore()
self.version_string = hs.version_string.encode('ascii') self.version_string_bytes = hs.version_string.encode('ascii')
self.default_timeout = 60 self.default_timeout = 60
def schedule(x): def schedule(x):
@ -261,8 +261,8 @@ class MatrixFederationHttpClient(object):
ignore_backoff=ignore_backoff, ignore_backoff=ignore_backoff,
) )
method = request.method method_bytes = request.method.encode("ascii")
destination = request.destination destination_bytes = request.destination.encode("ascii")
path_bytes = request.path.encode("ascii") path_bytes = request.path.encode("ascii")
if request.query: if request.query:
query_bytes = encode_query_args(request.query) query_bytes = encode_query_args(request.query)
@ -270,8 +270,8 @@ class MatrixFederationHttpClient(object):
query_bytes = b"" query_bytes = b""
headers_dict = { headers_dict = {
"User-Agent": [self.version_string], b"User-Agent": [self.version_string_bytes],
"Host": [request.destination], b"Host": [destination_bytes],
} }
with limiter: with limiter:
@ -282,50 +282,51 @@ class MatrixFederationHttpClient(object):
else: else:
retries_left = MAX_SHORT_RETRIES retries_left = MAX_SHORT_RETRIES
url = urllib.parse.urlunparse(( url_bytes = urllib.parse.urlunparse((
b"matrix", destination.encode("ascii"), b"matrix", destination_bytes,
path_bytes, None, query_bytes, b"", path_bytes, None, query_bytes, b"",
)).decode('ascii') ))
url_str = url_bytes.decode('ascii')
http_url = urllib.parse.urlunparse(( url_to_sign_bytes = urllib.parse.urlunparse((
b"", b"", b"", b"",
path_bytes, None, query_bytes, b"", path_bytes, None, query_bytes, b"",
)).decode('ascii') ))
while True: while True:
try: try:
json = request.get_json() json = request.get_json()
if json: if json:
data = encode_canonical_json(json) headers_dict[b"Content-Type"] = [b"application/json"]
headers_dict["Content-Type"] = ["application/json"]
self.sign_request( self.sign_request(
destination, method, http_url, headers_dict, json destination_bytes, method_bytes, url_to_sign_bytes,
headers_dict, json,
) )
else: data = encode_canonical_json(json)
data = None
self.sign_request(destination, method, http_url, headers_dict)
logger.info(
"{%s} [%s] Sending request: %s %s",
request.txn_id, destination, method, url
)
if data:
producer = FileBodyProducer( producer = FileBodyProducer(
BytesIO(data), BytesIO(data),
cooperator=self._cooperator cooperator=self._cooperator,
) )
else: else:
producer = None producer = None
self.sign_request(
destination_bytes, method_bytes, url_to_sign_bytes,
headers_dict,
)
request_deferred = treq.request( logger.info(
method, "{%s} [%s] Sending request: %s %s",
url, request.txn_id, request.destination, request.method,
url_str,
)
# we don't want all the fancy cookie and redirect handling that
# treq.request gives: just use the raw Agent.
request_deferred = self.agent.request(
method_bytes,
url_bytes,
headers=Headers(headers_dict), headers=Headers(headers_dict),
data=producer, bodyProducer=producer,
agent=self.agent,
reactor=self.hs.get_reactor(),
unbuffered=True
) )
request_deferred = timeout_deferred( request_deferred = timeout_deferred(
@ -344,9 +345,9 @@ class MatrixFederationHttpClient(object):
logger.warn( logger.warn(
"{%s} [%s] Request failed: %s %s: %s", "{%s} [%s] Request failed: %s %s: %s",
request.txn_id, request.txn_id,
destination, request.destination,
method, request.method,
url, url_str,
_flatten_response_never_received(e), _flatten_response_never_received(e),
) )
@ -366,7 +367,7 @@ class MatrixFederationHttpClient(object):
logger.debug( logger.debug(
"{%s} [%s] Waiting %ss before re-sending...", "{%s} [%s] Waiting %ss before re-sending...",
request.txn_id, request.txn_id,
destination, request.destination,
delay, delay,
) )
@ -378,7 +379,7 @@ class MatrixFederationHttpClient(object):
logger.info( logger.info(
"{%s} [%s] Got response headers: %d %s", "{%s} [%s] Got response headers: %d %s",
request.txn_id, request.txn_id,
destination, request.destination,
response.code, response.code,
response.phrase.decode('ascii', errors='replace'), response.phrase.decode('ascii', errors='replace'),
) )
@ -411,8 +412,9 @@ class MatrixFederationHttpClient(object):
destination_is must be non-None. destination_is must be non-None.
method (bytes): The HTTP method of the request method (bytes): The HTTP method of the request
url_bytes (bytes): The URI path of the request url_bytes (bytes): The URI path of the request
headers_dict (dict): Dictionary of request headers to append to headers_dict (dict[bytes, list[bytes]]): Dictionary of request headers to
content (bytes): The body of the request append to
content (object): The body of the request
destination_is (bytes): As 'destination', but if the destination is an destination_is (bytes): As 'destination', but if the destination is an
identity server identity server

View file

@ -528,7 +528,10 @@ def load_jinja2_templates(config):
""" """
logger.info("loading jinja2") logger.info("loading jinja2")
if config.email_template_dir:
loader = jinja2.FileSystemLoader(config.email_template_dir) loader = jinja2.FileSystemLoader(config.email_template_dir)
else:
loader = jinja2.PackageLoader('synapse', 'res/templates')
env = jinja2.Environment(loader=loader) env = jinja2.Environment(loader=loader)
env.filters["format_ts"] = format_ts_filter env.filters["format_ts"] = format_ts_filter
env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config) env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)

View file

@ -55,7 +55,7 @@ REQUIREMENTS = {
"sortedcontainers>=1.4.4": ["sortedcontainers"], "sortedcontainers>=1.4.4": ["sortedcontainers"],
"pysaml2>=3.0.0": ["saml2"], "pysaml2>=3.0.0": ["saml2"],
"pymacaroons-pynacl>=0.9.3": ["pymacaroons"], "pymacaroons-pynacl>=0.9.3": ["pymacaroons"],
"msgpack-python>=0.3.0": ["msgpack"], "msgpack-python>=0.4.2": ["msgpack"],
"phonenumbers>=8.2.0": ["phonenumbers"], "phonenumbers>=8.2.0": ["phonenumbers"],
"six>=1.10": ["six"], "six>=1.10": ["six"],

View file

@ -596,10 +596,13 @@ def _iterate_over_text(tree, *tags_to_ignore):
# to be returned. # to be returned.
elements = iter([tree]) elements = iter([tree])
while True: while True:
el = next(elements) el = next(elements, None)
if el is None:
return
if isinstance(el, string_types): if isinstance(el, string_types):
yield el yield el
elif el is not None and el.tag not in tags_to_ignore: elif el.tag not in tags_to_ignore:
# el.text is the text before the first child, so we can immediately # el.text is the text before the first child, so we can immediately
# return it if the text exists. # return it if the text exists.
if el.text: if el.text:

View file

@ -376,33 +376,25 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
@defer.inlineCallbacks @defer.inlineCallbacks
def get_missing_events(self, room_id, earliest_events, latest_events, def get_missing_events(self, room_id, earliest_events, latest_events,
limit, min_depth): limit):
ids = yield self.runInteraction( ids = yield self.runInteraction(
"get_missing_events", "get_missing_events",
self._get_missing_events, self._get_missing_events,
room_id, earliest_events, latest_events, limit, min_depth room_id, earliest_events, latest_events, limit,
) )
events = yield self._get_events(ids) events = yield self._get_events(ids)
defer.returnValue(events)
events = sorted(
[ev for ev in events if ev.depth >= min_depth],
key=lambda e: e.depth,
)
defer.returnValue(events[:limit])
def _get_missing_events(self, txn, room_id, earliest_events, latest_events, def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
limit, min_depth): limit):
earliest_events = set(earliest_events) seen_events = set(earliest_events)
front = set(latest_events) - earliest_events front = set(latest_events) - seen_events
event_results = []
event_results = set()
query = ( query = (
"SELECT prev_event_id FROM event_edges " "SELECT prev_event_id FROM event_edges "
"WHERE event_id = ? AND is_state = ? " "WHERE room_id = ? AND event_id = ? AND is_state = ? "
"LIMIT ?" "LIMIT ?"
) )
@ -411,18 +403,20 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
for event_id in front: for event_id in front:
txn.execute( txn.execute(
query, query,
(event_id, False, limit - len(event_results)) (room_id, event_id, False, limit - len(event_results))
) )
for e_id, in txn: new_results = set(t[0] for t in txn) - seen_events
new_front.add(e_id)
new_front -= earliest_events new_front |= new_results
new_front -= event_results seen_events |= new_results
event_results.extend(new_results)
front = new_front front = new_front
event_results |= new_front
# we built the list working backwards from latest_events; we now need to
# reverse it so that the events are approximately chronological.
event_results.reverse()
return event_results return event_results

11
synctl
View file

@ -48,7 +48,16 @@ def pid_running(pid):
def write(message, colour=NORMAL, stream=sys.stdout): def write(message, colour=NORMAL, stream=sys.stdout):
if colour == NORMAL: # Lets check if we're writing to a TTY before colouring
should_colour = False
try:
should_colour = stream.isatty()
except AttributeError:
# Just in case `isatty` isn't defined on everything. The python
# docs are incredibly vague.
pass
if not should_colour:
stream.write(message + "\n") stream.write(message + "\n")
else: else:
stream.write(colour + message + NORMAL + "\n") stream.write(colour + message + NORMAL + "\n")

View file

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse.handlers.room_list import RoomListNextBatch
import tests.unittest
import tests.utils
class RoomListTestCase(tests.unittest.TestCase):
""" Tests RoomList's RoomListNextBatch. """
def setUp(self):
pass
def test_check_read_batch_tokens(self):
batch_token = RoomListNextBatch(
stream_ordering="abcdef",
public_room_stream_id="123",
current_limit=20,
direction_is_forward=True,
).to_token()
next_batch = RoomListNextBatch.from_token(batch_token)
self.assertEquals(next_batch.stream_ordering, "abcdef")
self.assertEquals(next_batch.public_room_stream_id, "123")
self.assertEquals(next_batch.current_limit, 20)
self.assertEquals(next_batch.direction_is_forward, True)