MirrorHub/synapse
0
0
Fork 1
mirror of https://mau.dev/maunium/synapse.git synced 2024-12-15 18:04:07 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'develop' into pushers

Browse source
This commit is contained in:
David Baker 2015-01-13 13:15:51 +00:00
commit c06a9063e1
150 changed files with 1098 additions and 439 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
CHANGES.rst
View file

@ -1,7 +1,21 @@
Changes in synapse 0.6.1 (2015-01-07)
=====================================
* Major optimizations to improve performance of initial sync and event sending
in large rooms (by up to 10x)
* Media repository now includes a Content-Length header on media downloads.
* Improve quality of thumbnails by changing resizing algorithm.
Changes in synapse 0.6.0 (2014-12-16)
=====================================
* Add new API for media upload and download that supports thumbnailing.
* Replicate media uploads over multiple homeservers so media is always served
to clients from their local homeserver. This obsoletes the
--content-addr parameter and confusion over accessing content directly
from remote homeservers.
* Implement exponential backoff when retrying federation requests when
sending to remote homeservers which are offline.
* Implement typing notifications.
* Fix bugs where we sent events with invalid signatures due to bugs where
we incorrectly persisted events.

17
README.rst
View file

@ -108,6 +108,18 @@ To install the synapse homeserver run::
This installs synapse, along with the libraries it uses, into
``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX.
Your python may not give priority to locally installed libraries over system
libraries, in which case you must add your local packages to your python path::
$ # on Linux:
$ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages:$PYTHONPATH
$ # on OSX:
$ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages:$PYTHONPATH
For reliable VoIP calls to be routed via this homeserver, you MUST configure
a TURN server. See docs/turn-howto.rst for details.
Troubleshooting Installation
----------------------------
@ -239,6 +251,11 @@ Upgrading an existing homeserver
IMPORTANT: Before upgrading an existing homeserver to a new version, please
refer to UPGRADE.rst for any additional instructions.
Otherwise, simply re-install the new codebase over the current one - e.g.
by ``pip install --user --process-dependency-links
https://github.com/matrix-org/synapse/tarball/master``
if using pip, or by ``git pull`` if running off a git working copy.
Setting up Federation
=====================

4
UPGRADE.rst
View file

@ -1,6 +1,10 @@
Upgrading to v0.6.0
===================
To pull in new dependencies, run::
python setup.py develop --user
This update includes a change to the database schema. To upgrade you first need
to upgrade the database by running::

2
VERSION
View file

@ -1 +1 @@
0.6.0
0.6.1b

4
docs/media_repository.rst
View file

@ -1,6 +1,8 @@
Media Repository
Media Repository
================
*Synapse implementation-specific details for the media repository*
The media repository is where attachments and avatar photos are stored.
It stores attachment content and thumbnails for media uploaded by local users.
It caches attachment content and thumbnails for media uploaded by remote users.

30
graph/graph2.py
View file

@ -23,14 +23,27 @@ import argparse
from synapse.events import FrozenEvent
def make_graph(db_name, room_id, file_prefix):
def make_graph(db_name, room_id, file_prefix, limit):
conn = sqlite3.connect(db_name)
c = conn.execute(
"SELECT json FROM event_json where room_id = ?",
(room_id,)
sql = (
"SELECT json FROM event_json as j "
"INNER JOIN events as e ON e.event_id = j.event_id "
"WHERE j.room_id = ?"
)
args = [room_id]
if limit:
sql += (
" ORDER BY topological_ordering DESC, stream_ordering DESC "
"LIMIT ?"
)
args.append(limit)
c = conn.execute(sql, args)
events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()]
events.sort(key=lambda e: e.depth)
@ -128,11 +141,16 @@ if __name__ == "__main__":
)
parser.add_argument(
"-p", "--prefix", dest="prefix",
help="String to prefix output files with"
help="String to prefix output files with",
default="graph_output"
)
parser.add_argument(
"-l", "--limit",
help="Only retrieve the last N events.",
)
parser.add_argument('db')
parser.add_argument('room')
args = parser.parse_args()
make_graph(args.db, args.room, args.prefix)
make_graph(args.db, args.room, args.prefix, args.limit)

33
scripts/copyrighter-sql.pl Executable file
View file

@ -0,0 +1,33 @@
#!/usr/bin/perl -pi
# Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
$copyright = <<EOT;
/* Copyright 2015 OpenMarket Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
EOT
s/^(# -\*- coding: utf-8 -\*-\n)?/$1$copyright/ if ($. == 1);

2
scripts/copyrighter.pl
View file

@ -14,7 +14,7 @@
# limitations under the License.
$copyright = <<EOT;
# Copyright 2014 OpenMarket Ltd
# Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

4
setup.py
View file

@ -32,7 +32,7 @@ setup(
description="Reference Synapse Home Server",
install_requires=[
"syutil==0.0.2",
"matrix_angular_sdk==0.5.3b",
"matrix_angular_sdk==0.6.0",
"Twisted>=14.0.0",
"service_identity>=1.0.0",
"pyopenssl>=0.14",
@ -47,7 +47,7 @@ setup(
dependency_links=[
"https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2",
"https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0",
"https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3b/#egg=matrix_angular_sdk-0.5.3b",
"https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0",
],
setup_requires=[
"setuptools_trial",

6
synapse/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
""" This is a reference implementation of a synapse home server.
""" This is a reference implementation of a Matrix home server.
"""
__version__ = "0.6.0"
__version__ = "0.6.1b"

2
synapse/api/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/api/auth.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/api/constants.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/api/errors.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/api/ratelimiting.py
View file

@ -1,4 +1,4 @@
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/api/urls.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/app/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

12
synapse/app/homeserver.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,6 +18,8 @@ from synapse.storage import prepare_database, UpgradeDatabaseException
from synapse.server import HomeServer
from synapse.python_dependencies import check_requirements
from twisted.internet import reactor
from twisted.enterprise import adbapi
from twisted.web.resource import Resource
@ -39,6 +41,8 @@ from synapse.util.logcontext import LoggingContext
from daemonize import Daemonize
import twisted.manhole.telnet
import synapse
import logging
import os
import re
@ -198,7 +202,10 @@ def setup():
config.setup_logging()
check_requirements()
logger.info("Server hostname: %s", config.server_name)
logger.info("Server version: %s", synapse.__version__)
if re.search(":[0-9]+$", config.server_name):
domain_with_port = config.server_name
@ -234,7 +241,7 @@ def setup():
except UpgradeDatabaseException:
sys.stderr.write(
"\nFailed to upgrade database.\n"
"Have you followed any instructions in UPGRADES.rst?\n"
"Have you checked for version specific instructions in UPGRADES.rst?\n"
)
sys.exit(1)
@ -279,6 +286,7 @@ def run():
def main():
with LoggingContext("main"):
check_requirements()
setup()

2
synapse/app/synctl.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/config/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/config/_base.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/config/captcha.py
View file

@ -1,4 +1,4 @@
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/config/database.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/config/email.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/config/homeserver.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

7
synapse/config/logger.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -66,7 +66,10 @@ class LoggingConfig(Config):
formatter = logging.Formatter(log_format)
if self.log_file:
handler = logging.FileHandler(self.log_file)
# TODO: Customisable file size / backup count
handler = logging.handlers.RotatingFileHandler(
self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3
)
else:
handler = logging.StreamHandler()
handler.setFormatter(formatter)

2
synapse/config/ratelimiting.py
View file

@ -1,4 +1,4 @@
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

4
synapse/config/repository.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 matrix.org
# Copyright 2014, 2015 matrix.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -37,7 +37,7 @@ class ContentRepositoryConfig(Config):
super(ContentRepositoryConfig, cls).add_arguments(parser)
db_group = parser.add_argument_group("content_repository")
db_group.add_argument(
"--max-upload-size", default="1M"
"--max-upload-size", default="10M"
)
db_group.add_argument(
"--media-store-path", default=cls.default_path("media_store")

10
synapse/config/server.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -47,8 +47,12 @@ class ServerConfig(Config):
def add_arguments(cls, parser):
super(ServerConfig, cls).add_arguments(parser)
server_group = parser.add_argument_group("server")
server_group.add_argument("-H", "--server-name", default="localhost",
help="The name of the server")
server_group.add_argument(
"-H", "--server-name", default="localhost",
help="The domain name of the server, with optional explicit port. "
"This is used by remote servers to connect to this server, "
"e.g. matrix.org, localhost:8080, etc."
)
server_group.add_argument("--signing-key-path",
help="The signing key to sign messages with")
server_group.add_argument("-p", "--bind-port", metavar="PORT",

2
synapse/config/tls.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/config/voip.py
View file

@ -1,4 +1,4 @@
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/crypto/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/crypto/context_factory.py
View file

@ -1,4 +1,4 @@
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/crypto/event_signing.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/crypto/keyclient.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/crypto/keyring.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

24
synapse/events/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,12 +15,10 @@
from synapse.util.frozenutils import freeze, unfreeze
import copy
class _EventInternalMetadata(object):
def __init__(self, internal_metadata_dict):
self.__dict__ = copy.deepcopy(internal_metadata_dict)
self.__dict__ = internal_metadata_dict
def get_dict(self):
return dict(self.__dict__)
@ -49,10 +47,10 @@ def _event_dict_property(key):
class EventBase(object):
def __init__(self, event_dict, signatures={}, unsigned={},
internal_metadata_dict={}):
self.signatures = copy.deepcopy(signatures)
self.unsigned = copy.deepcopy(unsigned)
self.signatures = signatures
self.unsigned = unsigned
self._event_dict = copy.deepcopy(event_dict)
self._event_dict = event_dict
self.internal_metadata = _EventInternalMetadata(
internal_metadata_dict
@ -112,10 +110,16 @@ class EventBase(object):
class FrozenEvent(EventBase):
def __init__(self, event_dict, internal_metadata_dict={}):
event_dict = copy.deepcopy(event_dict)
event_dict = dict(event_dict)
signatures = copy.deepcopy(event_dict.pop("signatures", {}))
unsigned = copy.deepcopy(event_dict.pop("unsigned", {}))
# Signatures is a dict of dicts, and this is faster than doing a
# copy.deepcopy
signatures = {
name: {sig_id: sig for sig_id, sig in sigs.items()}
for name, sigs in event_dict.pop("signatures", {}).items()
}
unsigned = dict(event_dict.pop("unsigned", {}))
frozen_dict = freeze(event_dict)

2
synapse/events/builder.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/events/snapshot.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

17
synapse/events/utils.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -89,16 +89,24 @@ def prune_event(event):
return type(event)(allowed_fields)
def serialize_event(hs, e):
def serialize_event(hs, e, client_event=True):
# FIXME(erikj): To handle the case of presence events and the like
if not isinstance(e, EventBase):
return e
# Should this strip out None's?
d = {k: v for k, v in e.get_dict().items()}
if not client_event:
# set the age and keep all other keys
if "age_ts" in d["unsigned"]:
now = int(hs.get_clock().time_msec())
d["unsigned"]["age"] = now - d["unsigned"]["age_ts"]
return d
if "age_ts" in d["unsigned"]:
now = int(hs.get_clock().time_msec())
d["unsigned"]["age"] = now - d["unsigned"]["age_ts"]
d["age"] = now - d["unsigned"]["age_ts"]
del d["unsigned"]["age_ts"]
d["user_id"] = d.pop("sender", None)
@ -126,5 +134,8 @@ def serialize_event(hs, e):
del d["prev_events"]
del d["hashes"]
del d["signatures"]
d.pop("depth", None)
d.pop("unsigned", None)
d.pop("origin", None)
return d

2
synapse/events/validator.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/federation/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/federation/persistence.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

43
synapse/federation/replication.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -256,31 +256,35 @@ class ReplicationLayer(object):
@defer.inlineCallbacks
@log_function
def get_state_for_context(self, destination, context, event_id=None):
def get_state_for_context(self, destination, context, event_id):
"""Requests all of the `current` state PDUs for a given context from
a remote home server.
Args:
destination (str): The remote homeserver to query for the state.
context (str): The context we're interested in.
event_id (str): The id of the event we want the state at.
Returns:
Deferred: Results in a list of PDUs.
"""
transaction_data = yield self.transport_layer.get_context_state(
result = yield self.transport_layer.get_context_state(
destination,
context,
event_id=event_id,
)
transaction = Transaction(**transaction_data)
pdus = [
self.event_from_pdu_json(p, outlier=True)
for p in transaction.pdus
self.event_from_pdu_json(p, outlier=True) for p in result["pdus"]
]
defer.returnValue(pdus)
auth_chain = [
self.event_from_pdu_json(p, outlier=True)
for p in result.get("auth_chain", [])
]
defer.returnValue((pdus, auth_chain))
@defer.inlineCallbacks
@log_function
@ -383,10 +387,16 @@ class ReplicationLayer(object):
context,
event_id,
)
auth_chain = yield self.store.get_auth_chain(
[pdu.event_id for pdu in pdus]
)
else:
raise NotImplementedError("Specify an event")
defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict()))
defer.returnValue((200, {
"pdus": [pdu.get_pdu_json() for pdu in pdus],
"auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
}))
@defer.inlineCallbacks
@log_function
@ -562,8 +572,8 @@ class ReplicationLayer(object):
already_seen = (
existing and (
not existing.internal_metadata.outlier
or pdu.internal_metadata.outlier
not existing.internal_metadata.is_outlier()
or pdu.internal_metadata.is_outlier()
)
)
if already_seen:
@ -573,6 +583,8 @@ class ReplicationLayer(object):
state = None
auth_chain = []
# We need to make sure we have all the auth events.
# for e_id, _ in pdu.auth_events:
# exists = yield self._get_persisted_pdu(
@ -604,7 +616,7 @@ class ReplicationLayer(object):
# )
# Get missing pdus if necessary.
if not pdu.internal_metadata.outlier:
if not pdu.internal_metadata.is_outlier():
# We only backfill backwards to the min depth.
min_depth = yield self.handler.get_min_depth_for_context(
pdu.room_id
@ -645,7 +657,7 @@ class ReplicationLayer(object):
"_handle_new_pdu getting state for %s",
pdu.room_id
)
state = yield self.get_state_for_context(
state, auth_chain = yield self.get_state_for_context(
origin, pdu.room_id, pdu.event_id,
)
@ -655,6 +667,7 @@ class ReplicationLayer(object):
pdu,
backfilled=backfilled,
state=state,
auth_chain=auth_chain,
)
else:
ret = None
@ -717,6 +730,7 @@ class _TransactionQueue(object):
destinations = set(destinations)
destinations.discard(self.server_name)
destinations.discard("localhost")
logger.debug("Sending to: %s", str(destinations))
@ -801,6 +815,8 @@ class _TransactionQueue(object):
else:
logger.info("TX [%s] is ready for retry", destination)
logger.info("TX [%s] _attempt_new_transaction", destination)
if destination in self.pending_transactions:
# XXX: pending_transactions can get stuck on by a never-ending
# request at which point pending_pdus_by_dest just keeps growing.
@ -813,6 +829,9 @@ class _TransactionQueue(object):
pending_edus = self.pending_edus_by_dest.pop(destination, [])
pending_failures = self.pending_failures_by_dest.pop(destination, [])
if pending_pdus:
logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", destination, len(pending_pdus))
if not pending_pdus and not pending_edus and not pending_failures:
return

2
synapse/federation/transport.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/federation/units.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/handlers/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/handlers/_base.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/handlers/admin.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

4
synapse/handlers/directory.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -155,4 +155,4 @@ class DirectoryHandler(BaseHandler):
"room_id": room_id,
"sender": user_id,
"content": {"aliases": aliases},
})
}, ratelimit=False)

9
synapse/handlers/events.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -46,7 +46,8 @@ class EventStreamHandler(BaseHandler):
@defer.inlineCallbacks
@log_function
def get_stream(self, auth_user_id, pagin_config, timeout=0):
def get_stream(self, auth_user_id, pagin_config, timeout=0,
as_client_event=True):
auth_user = self.hs.parse_userid(auth_user_id)
try:
@ -78,7 +79,9 @@ class EventStreamHandler(BaseHandler):
auth_user, room_ids, pagin_config, timeout
)
chunks = [self.hs.serialize_event(e) for e in events]
chunks = [
self.hs.serialize_event(e, as_client_event) for e in events
]
chunk = {
"chunk": chunks,

87
synapse/handlers/federation.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -91,11 +91,12 @@ class FederationHandler(BaseHandler):
yield run_on_reactor()
yield self.replication_layer.send_pdu(event, destinations)
self.replication_layer.send_pdu(event, destinations)
@log_function
@defer.inlineCallbacks
def on_receive_pdu(self, origin, pdu, backfilled, state=None):
def on_receive_pdu(self, origin, pdu, backfilled, state=None,
auth_chain=None):
""" Called by the ReplicationLayer when we have a new pdu. We need to
do auth checks and put it through the StateHandler.
"""
@ -150,40 +151,41 @@ class FederationHandler(BaseHandler):
if not is_in_room and not event.internal_metadata.outlier:
logger.debug("Got event for room we're not in.")
replication_layer = self.replication_layer
auth_chain = yield replication_layer.get_event_auth(
origin,
context=event.room_id,
event_id=event.event_id,
)
for e in auth_chain:
e.internal_metadata.outlier = True
try:
yield self._handle_new_event(e, fetch_missing=False)
except:
logger.exception(
"Failed to parse auth event %s",
e.event_id,
)
replication = self.replication_layer
if not state:
state = yield replication_layer.get_state_for_context(
state, auth_chain = yield replication.get_state_for_context(
origin, context=event.room_id, event_id=event.event_id,
)
if not auth_chain:
auth_chain = yield replication.get_event_auth(
origin,
context=event.room_id,
event_id=event.event_id,
)
for e in auth_chain:
e.internal_metadata.outlier = True
try:
yield self._handle_new_event(e, fetch_auth_from=origin)
except:
logger.exception(
"Failed to handle auth event %s",
e.event_id,
)
current_state = state
if state:
for e in state:
logging.info("A :) %r", e)
e.internal_metadata.outlier = True
try:
yield self._handle_new_event(e)
except:
logger.exception(
"Failed to parse state event %s",
"Failed to handle state event %s",
e.event_id,
)
@ -288,7 +290,7 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks
def on_event_auth(self, event_id):
auth = yield self.store.get_auth_chain(event_id)
auth = yield self.store.get_auth_chain([event_id])
for event in auth:
event.signatures.update(
@ -391,10 +393,10 @@ class FederationHandler(BaseHandler):
for e in auth_chain:
e.internal_metadata.outlier = True
try:
yield self._handle_new_event(e, fetch_missing=False)
yield self._handle_new_event(e)
except:
logger.exception(
"Failed to parse auth event %s",
"Failed to handle auth event %s",
e.event_id,
)
@ -403,12 +405,11 @@ class FederationHandler(BaseHandler):
e.internal_metadata.outlier = True
try:
yield self._handle_new_event(
e,
fetch_missing=True
e, fetch_auth_from=target_host
)
except:
logger.exception(
"Failed to parse state event %s",
"Failed to handle state event %s",
e.event_id,
)
@ -526,9 +527,12 @@ class FederationHandler(BaseHandler):
event.signatures,
)
yield self.replication_layer.send_pdu(new_pdu, destinations)
self.replication_layer.send_pdu(new_pdu, destinations)
auth_chain = yield self.store.get_auth_chain(event.event_id)
state_ids = [e.event_id for e in context.current_state.values()]
auth_chain = yield self.store.get_auth_chain(set(
[event.event_id] + state_ids
))
defer.returnValue({
"state": context.current_state.values(),
@ -613,13 +617,13 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks
@log_function
def on_backfill_request(self, origin, context, pdu_list, limit):
in_room = yield self.auth.check_host_in_room(context, origin)
def on_backfill_request(self, origin, room_id, pdu_list, limit):
in_room = yield self.auth.check_host_in_room(room_id, origin)
if not in_room:
raise AuthError(403, "Host not in room.")
events = yield self.store.get_backfill_events(
context,
room_id,
pdu_list,
limit
)
@ -678,7 +682,7 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks
def _handle_new_event(self, event, state=None, backfilled=False,
current_state=None, fetch_missing=True):
current_state=None, fetch_auth_from=None):
logger.debug(
"_handle_new_event: Before annotate: %s, sigs: %s",
@ -699,11 +703,20 @@ class FederationHandler(BaseHandler):
known_ids = set(
[s.event_id for s in context.auth_events.values()]
)
for e_id, _ in event.auth_events:
if e_id not in known_ids:
e = yield self.store.get_event(
e_id, allow_none=True,
)
e = yield self.store.get_event(e_id, allow_none=True)
if not e and fetch_auth_from is not None:
# Grab the auth_chain over federation if we are missing
# auth events.
auth_chain = yield self.replication_layer.get_event_auth(
fetch_auth_from, event.event_id, event.room_id
)
for auth_event in auth_chain:
yield self._handle_new_event(auth_event)
e = yield self.store.get_event(e_id, allow_none=True)
if not e:
# TODO: Do some conflict res to make sure that we're
@ -713,7 +726,7 @@ class FederationHandler(BaseHandler):
event.event_id, e_id, known_ids,
)
# FIXME: How does raising AuthError work with federation?
raise AuthError(403, "Auth events are stale")
raise AuthError(403, "Cannot find auth event")
context.auth_events[(e.type, e.state_key)] = e

2
synapse/handlers/login.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

30
synapse/handlers/message.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -19,6 +19,7 @@ from synapse.api.constants import EventTypes, Membership
from synapse.api.errors import RoomError
from synapse.streams.config import PaginationConfig
from synapse.events.validator import EventValidator
from synapse.util.logcontext import PreserveLoggingContext
from ._base import BaseHandler
@ -66,7 +67,7 @@ class MessageHandler(BaseHandler):
@defer.inlineCallbacks
def get_messages(self, user_id=None, room_id=None, pagin_config=None,
feedback=False):
feedback=False, as_client_event=True):
"""Get messages in a room.
Args:
@ -75,6 +76,7 @@ class MessageHandler(BaseHandler):
pagin_config (synapse.api.streams.PaginationConfig): The pagination
config rules to apply, if any.
feedback (bool): True to get compressed feedback with the messages
as_client_event (bool): True to get events in client-server format.
Returns:
dict: Pagination API results
"""
@ -98,7 +100,9 @@ class MessageHandler(BaseHandler):
)
chunk = {
"chunk": [self.hs.serialize_event(e) for e in events],
"chunk": [
self.hs.serialize_event(e, as_client_event) for e in events
],
"start": pagin_config.from_token.to_string(),
"end": next_token.to_string(),
}
@ -106,7 +110,7 @@ class MessageHandler(BaseHandler):
defer.returnValue(chunk)
@defer.inlineCallbacks
def create_and_send_event(self, event_dict):
def create_and_send_event(self, event_dict, ratelimit=True):
""" Given a dict from a client, create and handle a new event.
Creates an FrozenEvent object, filling out auth_events, prev_events,
@ -123,7 +127,8 @@ class MessageHandler(BaseHandler):
self.validator.validate_new(builder)
self.ratelimit(builder.user_id)
if ratelimit:
self.ratelimit(builder.user_id)
# TODO(paul): Why does 'event' not have a 'user' object?
user = self.hs.parse_userid(builder.user_id)
assert self.hs.is_mine(user), "User must be our own: %s" % (user,)
@ -152,6 +157,11 @@ class MessageHandler(BaseHandler):
context=context,
)
if event.type == EventTypes.Message:
presence = self.hs.get_handlers().presence_handler
with PreserveLoggingContext():
presence.bump_presence_active_time(user)
defer.returnValue(event)
@defer.inlineCallbacks
@ -204,7 +214,7 @@ class MessageHandler(BaseHandler):
@defer.inlineCallbacks
def snapshot_all_rooms(self, user_id=None, pagin_config=None,
feedback=False):
feedback=False, as_client_event=True):
"""Retrieve a snapshot of all rooms the user is invited or has joined.
This snapshot may include messages for all rooms where the user is
@ -215,6 +225,7 @@ class MessageHandler(BaseHandler):
pagin_config (synapse.api.streams.PaginationConfig): The pagination
config used to determine how many messages *PER ROOM* to return.
feedback (bool): True to get feedback along with these messages.
as_client_event (bool): True to get events in client-server format.
Returns:
A list of dicts with "room_id" and "membership" keys for all rooms
the user is currently invited or joined in on. Rooms where the user
@ -256,7 +267,7 @@ class MessageHandler(BaseHandler):
}
if event.membership == Membership.INVITE:
d["inviter"] = event.user_id
d["inviter"] = event.sender
rooms_ret.append(d)
@ -273,7 +284,10 @@ class MessageHandler(BaseHandler):
end_token = now_token.copy_and_replace("room_key", token[1])
d["messages"] = {
"chunk": [self.hs.serialize_event(m) for m in messages],
"chunk": [
self.hs.serialize_event(m, as_client_event)
for m in messages
],
"start": start_token.to_string(),
"end": end_token.to_string(),
}

2
synapse/handlers/presence.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

16
synapse/handlers/profile.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
from twisted.internet import defer
from synapse.api.errors import SynapseError, AuthError, CodeMessageException
from synapse.api.constants import Membership
from synapse.api.constants import EventTypes, Membership
from synapse.util.logcontext import PreserveLoggingContext
from ._base import BaseHandler
@ -194,6 +194,8 @@ class ProfileHandler(BaseHandler):
if not self.hs.is_mine(user):
return
self.ratelimit(user.to_string())
joins = yield self.store.get_rooms_for_user_where_membership_is(
user.to_string(),
[Membership.JOIN],
@ -201,7 +203,7 @@ class ProfileHandler(BaseHandler):
for j in joins:
content = {
"membership": j.content["membership"],
"membership": Membership.JOIN,
}
yield self.distributor.fire(
@ -210,9 +212,9 @@ class ProfileHandler(BaseHandler):
msg_handler = self.hs.get_handlers().message_handler
yield msg_handler.create_and_send_event({
"type": j.type,
"type": EventTypes.Member,
"room_id": j.room_id,
"state_key": j.state_key,
"state_key": user.to_string(),
"content": content,
"sender": j.state_key,
})
"sender": user.to_string()
}, ratelimit=False)

2
synapse/handlers/register.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

41
synapse/handlers/room.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -130,6 +130,7 @@ class RoomCreationHandler(BaseHandler):
"type": EventTypes.Name,
"room_id": room_id,
"sender": user_id,
"state_key": "",
"content": {"name": name},
})
@ -139,6 +140,7 @@ class RoomCreationHandler(BaseHandler):
"type": EventTypes.Topic,
"room_id": room_id,
"sender": user_id,
"state_key": "",
"content": {"topic": topic},
})
@ -147,7 +149,7 @@ class RoomCreationHandler(BaseHandler):
"type": EventTypes.Member,
"state_key": invitee,
"room_id": room_id,
"user_id": user_id,
"sender": user_id,
"content": {"membership": Membership.INVITE},
})
@ -243,14 +245,12 @@ class RoomMemberHandler(BaseHandler):
self.distributor.declare("user_left_room")
@defer.inlineCallbacks
def get_room_members(self, room_id, membership=Membership.JOIN):
def get_room_members(self, room_id):
hs = self.hs
memberships = yield self.store.get_room_members(
room_id=room_id, membership=membership
)
users = yield self.store.get_users_in_room(room_id)
defer.returnValue([hs.parse_userid(m.user_id) for m in memberships])
defer.returnValue([hs.parse_userid(u) for u in users])
@defer.inlineCallbacks
def fetch_room_distributions_into(self, room_id, localusers=None,
@ -390,6 +390,11 @@ class RoomMemberHandler(BaseHandler):
host = hosts[0]
# If event doesn't include a display name, add one.
yield self.distributor.fire(
"collect_presencelike_data", joinee, content
)
content.update({"membership": Membership.JOIN})
builder = self.event_builder_factory.new({
"type": EventTypes.Member,
@ -420,10 +425,22 @@ class RoomMemberHandler(BaseHandler):
event.room_id,
self.hs.hostname
)
if not is_host_in_room:
# is *anyone* in the room?
room_member_keys = [
v for (k, v) in context.current_state.keys() if (
k == "m.room.member"
)
]
if len(room_member_keys) == 0:
# has the room been created so we can join it?
create_event = context.current_state.get(("m.room.create", ""))
if create_event:
is_host_in_room = True
if is_host_in_room:
should_do_dance = False
elif room_host:
elif room_host: # TODO: Shouldn't this be remote_room_host?
should_do_dance = True
else:
# TODO(markjh): get prev_state from snapshot
@ -437,7 +454,8 @@ class RoomMemberHandler(BaseHandler):
should_do_dance = not self.hs.is_mine(inviter)
room_host = inviter.domain
else:
should_do_dance = False
# return the same error as join_room_alias does
raise SynapseError(404, "No known servers")
if should_do_dance:
handler = self.hs.get_handlers().federation_handler
@ -524,11 +542,10 @@ class RoomListHandler(BaseHandler):
def get_public_room_list(self):
chunk = yield self.store.get_rooms(is_public=True)
for room in chunk:
joined_members = yield self.store.get_room_members(
joined_users = yield self.store.get_users_in_room(
room_id=room["room_id"],
membership=Membership.JOIN
)
room["num_joined_members"] = len(joined_members)
room["num_joined_members"] = len(joined_users)
# FIXME (erikj): START is no longer a valid value
defer.returnValue({"start": "START", "end": "END", "chunk": chunk})

10
synapse/handlers/typing.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -83,9 +83,15 @@ class TypingNotificationHandler(BaseHandler):
if member in self._member_typing_timer:
self.clock.cancel_call_later(self._member_typing_timer[member])
def _cb():
logger.debug(
"%s has timed out in %s", target_user.to_string(), room_id
)
self._stopped_typing(member)
self._member_typing_until[member] = until
self._member_typing_timer[member] = self.clock.call_later(
timeout / 1000, lambda: self._stopped_typing(member)
timeout / 1000.0, _cb
)
if was_present:

2
synapse/http/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

18
synapse/http/agent_name.py Normal file
View file

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse import __version__
AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii")

12
synapse/http/client.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,6 +14,7 @@
# limitations under the License.
from synapse.http.agent_name import AGENT_NAME
from twisted.internet import defer, reactor
from twisted.web.client import (
Agent, readBody, FileBodyProducer, PartialDownloadError
@ -51,7 +52,8 @@ class SimpleHttpClient(object):
"POST",
uri.encode("ascii"),
headers=Headers({
"Content-Type": ["application/x-www-form-urlencoded"]
b"Content-Type": [b"application/x-www-form-urlencoded"],
b"User-Agent": [AGENT_NAME],
}),
bodyProducer=FileBodyProducer(StringIO(query_bytes))
)
@ -105,6 +107,9 @@ class SimpleHttpClient(object):
response = yield self.agent.request(
"GET",
uri.encode("ascii"),
headers=Headers({
b"User-Agent": [AGENT_NAME],
})
)
body = yield readBody(response)
@ -127,7 +132,8 @@ class CaptchaServerHttpClient(SimpleHttpClient):
url.encode("ascii"),
bodyProducer=FileBodyProducer(StringIO(query_bytes)),
headers=Headers({
"Content-Type": ["application/x-www-form-urlencoded"]
b"Content-Type": [b"application/x-www-form-urlencoded"],
b"User-Agent": [AGENT_NAME],
})
)

2
synapse/http/endpoint.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

6
synapse/http/matrixfederationclient.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -20,6 +20,7 @@ from twisted.web.client import readBody, _AgentBase, _URI
from twisted.web.http_headers import Headers
from twisted.web._newclient import ResponseDone
from synapse.http.agent_name import AGENT_NAME
from synapse.http.endpoint import matrix_federation_endpoint
from synapse.util.async import sleep
from synapse.util.logcontext import PreserveLoggingContext
@ -71,6 +72,7 @@ class MatrixFederationHttpClient(object):
requests.
"""
def __init__(self, hs):
self.hs = hs
self.signing_key = hs.config.signing_key[0]
@ -83,7 +85,7 @@ class MatrixFederationHttpClient(object):
query_bytes=b"", retry_on_dns_fail=True):
""" Creates and sends a request to the given url
"""
headers_dict[b"User-Agent"] = [b"Synapse"]
headers_dict[b"User-Agent"] = [AGENT_NAME]
headers_dict[b"Host"] = [destination]
url_bytes = urlparse.urlunparse(

12
synapse/http/server.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,14 +14,16 @@
# limitations under the License.
from syutil.jsonutil import (
encode_canonical_json, encode_pretty_printed_json
)
from synapse.http.agent_name import AGENT_NAME
from synapse.api.errors import (
cs_exception, SynapseError, CodeMessageException
)
from synapse.util.logcontext import LoggingContext
from syutil.jsonutil import (
encode_canonical_json, encode_pretty_printed_json
)
from twisted.internet import defer, reactor
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
@ -230,6 +232,8 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
request.setResponseCode(code, message=response_code_message)
request.setHeader(b"Content-Type", b"application/json")
request.setHeader(b"Server", AGENT_NAME)
request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
if send_cors:
request.setHeader("Access-Control-Allow-Origin", "*")

2
synapse/http/server_key_resource.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/media/v0/content_repository.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

45
synapse/media/v1/__init__.py
View file

@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import PIL.Image
# check for JPEG support.
try:
PIL.Image._getdecoder("rgb", "jpeg", None)
except IOError as e:
if str(e).startswith("decoder jpeg not available"):
raise Exception(
"FATAL: jpeg codec not supported. Install pillow correctly! "
" 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
" pip install pillow --user'"
)
except Exception:
# any other exception is fine
pass
# check for PNG support.
try:
PIL.Image._getdecoder("rgb", "zip", None)
except IOError as e:
if str(e).startswith("decoder zip not available"):
raise Exception(
"FATAL: zip codec not supported. Install pillow correctly! "
" 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
" pip install pillow --user'"
)
except Exception:
# any other exception is fine
pass

15
synapse/media/v1/base_resource.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -139,6 +139,7 @@ class BaseMediaResource(Resource):
@download.addBoth
def callback(media_info):
del self.downloads[key]
return media_info
return download
@defer.inlineCallbacks
@ -201,7 +202,8 @@ class BaseMediaResource(Resource):
defer.returnValue(media_info)
@defer.inlineCallbacks
def _respond_with_file(self, request, media_type, file_path):
def _respond_with_file(self, request, media_type, file_path,
file_size=None):
logger.debug("Responding with %r", file_path)
if os.path.isfile(file_path):
@ -215,13 +217,20 @@ class BaseMediaResource(Resource):
request.setHeader(
b"Cache-Control", b"public,max-age=86400,s-maxage=86400"
)
if file_size is None:
stat = os.stat(file_path)
file_size = stat.st_size
request.setHeader(
b"Content-Length", b"%d" % (file_size,)
)
with open(file_path, "rb") as f:
yield FileSender().beginFileTransfer(f, request)
request.finish()
else:
self._respond_404()
self._respond_404(request)
def _get_thumbnail_requirements(self, media_type):
if media_type == "image/jpeg":

14
synapse/media/v1/download_resource.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -46,23 +46,29 @@ class DownloadResource(BaseMediaResource):
def _respond_local_file(self, request, media_id):
media_info = yield self.store.get_local_media(media_id)
if not media_info:
self._respond_404()
self._respond_404(request)
return
media_type = media_info["media_type"]
media_length = media_info["media_length"]
file_path = self.filepaths.local_media_filepath(media_id)
yield self._respond_with_file(request, media_type, file_path)
yield self._respond_with_file(
request, media_type, file_path, media_length
)
@defer.inlineCallbacks
def _respond_remote_file(self, request, server_name, media_id):
media_info = yield self._get_remote_media(server_name, media_id)
media_type = media_info["media_type"]
media_length = media_info["media_length"]
filesystem_id = media_info["filesystem_id"]
file_path = self.filepaths.remote_media_filepath(
server_name, filesystem_id
)
yield self._respond_with_file(request, media_type, file_path)
yield self._respond_with_file(
request, media_type, file_path, media_length
)

2
synapse/media/v1/filepath.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/media/v1/media_repository.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

27
synapse/media/v1/thumbnail_resource.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -100,11 +100,12 @@ class ThumbnailResource(BaseMediaResource):
t_type = thumbnail_info["thumbnail_type"]
t_method = thumbnail_info["thumbnail_method"]
file_id = thumbnail_info["filesystem_id"]
t_length = thumbnail_info["thumbnail_length"]
file_path = self.filepaths.remote_media_thumbnail(
server_name, file_id, t_width, t_height, t_type, t_method,
)
yield self._respond_with_file(request, t_type, file_path)
yield self._respond_with_file(request, t_type, file_path, t_length)
else:
yield self._respond_default_thumbnail(
request, media_info, width, height, method, m_type,
@ -139,11 +140,12 @@ class ThumbnailResource(BaseMediaResource):
t_height = thumbnail_info["thumbnail_height"]
t_type = thumbnail_info["thumbnail_type"]
t_method = thumbnail_info["thumbnail_method"]
t_length = thumbnail_info["thumbnail_length"]
file_path = self.filepaths.default_thumbnail(
top_level_type, sub_type, t_width, t_height, t_type, t_method,
)
yield self.respond_with_file(request, t_type, file_path)
yield self.respond_with_file(request, t_type, file_path, t_length)
def _select_thumbnail(self, desired_width, desired_height, desired_method,
desired_type, thumbnail_infos):
@ -165,18 +167,27 @@ class ThumbnailResource(BaseMediaResource):
aspect_quality, size_quality, type_quality,
length_quality, info
))
return min(info_list)[-1]
if info_list:
return min(info_list)[-1]
else:
info_list = []
info_list2 = []
for info in thumbnail_infos:
t_w = info["thumbnail_width"]
t_h = info["thumbnail_height"]
t_method = info["thumbnail_method"]
size_quality = abs((d_w - t_w) * (d_h - t_h))
type_quality = desired_type != info["thumbnail_type"]
length_quality = info["thumbnail_length"]
if t_method == "scale" and (t_w >= d_w or t_h >= d_h):
size_quality = abs((d_w - t_w) * (d_h - t_h))
type_quality = desired_type != info["thumbnail_type"]
length_quality = info["thumbnail_length"]
info_list.append((
size_quality, type_quality, length_quality, info
))
return min(info_list)[-1]
elif t_method == "scale":
info_list2.append((
size_quality, type_quality, length_quality, info
))
if info_list:
return min(info_list)[-1]
else:
return min(info_list2)[-1]

10
synapse/media/v1/thumbnailer.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -48,7 +48,7 @@ class Thumbnailer(object):
def scale(self, output_path, width, height, output_type):
"""Rescales the image to the given dimensions"""
scaled = self.image.resize((width, height), Image.BILINEAR)
scaled = self.image.resize((width, height), Image.ANTIALIAS)
return self.save_image(scaled, output_type, output_path)
def crop(self, output_path, width, height, output_type):
@ -65,7 +65,7 @@ class Thumbnailer(object):
if width * self.height > height * self.width:
scaled_height = (width * self.height) // self.width
scaled_image = self.image.resize(
(width, scaled_height), Image.BILINEAR
(width, scaled_height), Image.ANTIALIAS
)
crop_top = (scaled_height - height) // 2
crop_bottom = height + crop_top
@ -73,7 +73,7 @@ class Thumbnailer(object):
else:
scaled_width = (height * self.width) // self.height
scaled_image = self.image.resize(
(scaled_width, height), Image.BILINEAR
(scaled_width, height), Image.ANTIALIAS
)
crop_left = (scaled_width - width) // 2
crop_right = width + crop_left
@ -82,7 +82,7 @@ class Thumbnailer(object):
def save_image(self, output_image, output_type, output_path):
output_bytes_io = BytesIO()
output_image.save(output_bytes_io, self.FORMATS[output_type])
output_image.save(output_bytes_io, self.FORMATS[output_type], quality=70)
output_bytes = output_bytes_io.getvalue()
with open(output_path, "wb") as output_file:
output_file.write(output_bytes)

2
synapse/media/v1/upload_resource.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/notifier.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

80
synapse/python_dependencies.py Normal file
View file

@ -0,0 +1,80 @@
import logging
from distutils.version import LooseVersion
logger = logging.getLogger(__name__)
REQUIREMENTS = {
"syutil==0.0.2": ["syutil"],
"matrix_angular_sdk==0.6.0": ["syweb==0.6.0"],
"Twisted>=14.0.0": ["twisted>=14.0.0"],
"service_identity>=1.0.0": ["service_identity>=1.0.0"],
"pyopenssl>=0.14": ["OpenSSL>=0.14"],
"pyyaml": ["yaml"],
"pyasn1": ["pyasn1"],
"pynacl": ["nacl"],
"daemonize": ["daemonize"],
"py-bcrypt": ["bcrypt"],
"frozendict>=0.4": ["frozendict"],
"pillow": ["PIL"],
}
class MissingRequirementError(Exception):
pass
def check_requirements():
"""Checks that all the modules needed by synapse have been correctly
installed and are at the correct version"""
for dependency, module_requirements in REQUIREMENTS.items():
for module_requirement in module_requirements:
if ">=" in module_requirement:
module_name, required_version = module_requirement.split(">=")
version_test = ">="
elif "==" in module_requirement:
module_name, required_version = module_requirement.split("==")
version_test = "=="
else:
module_name = module_requirement
version_test = None
try:
module = __import__(module_name)
except ImportError:
logging.exception(
"Can't import %r which is part of %r",
module_name, dependency
)
raise MissingRequirementError(
"Can't import %r which is part of %r"
% (module_name, dependency)
)
version = getattr(module, "__version__", None)
file_path = getattr(module, "__file__", None)
logger.info(
"Using %r version %r from %r to satisfy %r",
module_name, version, file_path, dependency
)
if version_test == ">=":
if version is None:
raise MissingRequirementError(
"Version of %r isn't set as __version__ of module %r"
% (dependency, module_name)
)
if LooseVersion(version) < LooseVersion(required_version):
raise MissingRequirementError(
"Version of %r in %r is too old. %r < %r"
% (dependency, file_path, version, required_version)
)
elif version_test == "==":
if version is None:
raise MissingRequirementError(
"Version of %r isn't set as __version__ of module %r"
% (dependency, module_name)
)
if LooseVersion(version) != LooseVersion(required_version):
raise MissingRequirementError(
"Unexpected version of %r in %r. %r != %r"
% (dependency, file_path, version, required_version)
)

2
synapse/rest/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/rest/admin.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/rest/base.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/rest/directory.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

7
synapse/rest/events.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -44,8 +44,11 @@ class EventStreamRestServlet(RestServlet):
except ValueError:
raise SynapseError(400, "timeout must be in milliseconds.")
as_client_event = "raw" not in request.args
chunk = yield handler.get_stream(
auth_user.to_string(), pagin_config, timeout=timeout
auth_user.to_string(), pagin_config, timeout=timeout,
as_client_event=as_client_event
)
except:
logger.exception("Event stream failed")

7
synapse/rest/initial_sync.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -27,12 +27,15 @@ class InitialSyncRestServlet(RestServlet):
def on_GET(self, request):
user = yield self.auth.get_user_by_req(request)
with_feedback = "feedback" in request.args
as_client_event = "raw" not in request.args
pagination_config = PaginationConfig.from_request(request)
handler = self.handlers.message_handler
content = yield handler.snapshot_all_rooms(
user_id=user.to_string(),
pagin_config=pagination_config,
feedback=with_feedback)
feedback=with_feedback,
as_client_event=as_client_event
)
defer.returnValue((200, content))

2
synapse/rest/login.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/rest/presence.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/rest/profile.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/rest/register.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

7
synapse/rest/room.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -314,12 +314,15 @@ class RoomMessageListRestServlet(RestServlet):
request, default_limit=10,
)
with_feedback = "feedback" in request.args
as_client_event = "raw" not in request.args
handler = self.handlers.message_handler
msgs = yield handler.get_messages(
room_id=room_id,
user_id=user.to_string(),
pagin_config=pagination_config,
feedback=with_feedback)
feedback=with_feedback,
as_client_event=as_client_event
)
defer.returnValue((200, msgs))

2
synapse/rest/transactions.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/rest/voip.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

6
synapse/server.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -151,8 +151,8 @@ class BaseHomeServer(object):
object."""
return EventID.from_string(s)
def serialize_event(self, e):
return serialize_event(self, e)
def serialize_event(self, e, as_client_event=True):
return serialize_event(self, e, as_client_event)
def get_ip_from_request(self, request):
# May be an X-Forwarding-For header depending on config

2
synapse/state.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

8
synapse/storage/__init__.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -68,7 +68,7 @@ SCHEMAS = [
# Remember to update this number every time an incompatible change is made to
# database schema files, so the users will be informed on server restarts.
SCHEMA_VERSION = 10
SCHEMA_VERSION = 11
class _RollbackButIsFineException(Exception):
@ -146,9 +146,7 @@ class DataStore(RoomMemberStore, RoomStore,
elif event.type == EventTypes.Redaction:
self._store_redaction(txn, event)
outlier = False
if hasattr(event.internal_metadata, "outlier"):
outlier = event.internal_metadata.outlier
outlier = event.internal_metadata.is_outlier()
event_dict = {
k: v

52
synapse/storage/_base.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -64,7 +64,7 @@ class LoggingTransaction(object):
# Don't let logging failures stop SQL from working
pass
start = time.clock() * 1000
start = time.time() * 1000
try:
return self.txn.execute(
sql, *args, **kwargs
@ -73,7 +73,7 @@ class LoggingTransaction(object):
logger.exception("[SQL FAIL] {%s}", self.name)
raise
finally:
end = time.clock() * 1000
end = time.time() * 1000
sql_logger.debug("[SQL time] {%s} %f", self.name, end - start)
@ -93,7 +93,7 @@ class SQLBaseStore(object):
def inner_func(txn, *args, **kwargs):
with LoggingContext("runInteraction") as context:
current_context.copy_to(context)
start = time.clock() * 1000
start = time.time() * 1000
txn_id = SQLBaseStore._TXN_ID
# We don't really need these to be unique, so lets stop it from
@ -109,7 +109,7 @@ class SQLBaseStore(object):
logger.exception("[TXN FAIL] {%s}", name)
raise
finally:
end = time.clock() * 1000
end = time.time() * 1000
transaction_logger.debug(
"[TXN END] {%s} %f",
name, end - start
@ -479,23 +479,31 @@ class SQLBaseStore(object):
return self.runInteraction("_simple_max_id", func)
def _get_events(self, event_ids):
def _get_events(self, event_ids, check_redacted=True,
get_prev_content=False):
return self.runInteraction(
"_get_events", self._get_events_txn, event_ids
"_get_events", self._get_events_txn, event_ids,
check_redacted=check_redacted, get_prev_content=get_prev_content,
)
def _get_events_txn(self, txn, event_ids):
events = []
for e_id in event_ids:
ev = self._get_event_txn(txn, e_id)
def _get_events_txn(self, txn, event_ids, check_redacted=True,
get_prev_content=False):
if not event_ids:
return []
if ev:
events.append(ev)
events = [
self._get_event_txn(
txn, event_id,
check_redacted=check_redacted,
get_prev_content=get_prev_content
)
for event_id in event_ids
]
return events
return [e for e in events if e]
def _get_event_txn(self, txn, event_id, check_redacted=True,
get_prev_content=True):
get_prev_content=False):
sql = (
"SELECT internal_metadata, json, r.event_id FROM event_json as e "
"LEFT JOIN redactions as r ON e.event_id = r.redacts "
@ -512,6 +520,14 @@ class SQLBaseStore(object):
internal_metadata, js, redacted = res
return self._get_event_from_row_txn(
txn, internal_metadata, js, redacted,
check_redacted=check_redacted,
get_prev_content=get_prev_content,
)
def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted,
check_redacted=True, get_prev_content=False):
d = json.loads(js)
internal_metadata = json.loads(internal_metadata)
@ -533,11 +549,13 @@ class SQLBaseStore(object):
ev.unsigned["redacted_because"] = because
if get_prev_content and "replaces_state" in ev.unsigned:
ev.unsigned["prev_content"] = self._get_event_txn(
prev = self._get_event_txn(
txn,
ev.unsigned["replaces_state"],
get_prev_content=False,
).get_dict()["content"]
)
if prev:
ev.unsigned["prev_content"] = prev.get_dict()["content"]
return ev

2
synapse/storage/directory.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

26
synapse/storage/event_federation.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -32,39 +32,33 @@ class EventFederationStore(SQLBaseStore):
and backfilling from another server respectively.
"""
def get_auth_chain(self, event_id):
def get_auth_chain(self, event_ids):
return self.runInteraction(
"get_auth_chain",
self._get_auth_chain_txn,
event_id
event_ids
)
def _get_auth_chain_txn(self, txn, event_id):
results = self._get_auth_chain_ids_txn(txn, event_id)
def _get_auth_chain_txn(self, txn, event_ids):
results = self._get_auth_chain_ids_txn(txn, event_ids)
sql = "SELECT * FROM events WHERE event_id = ?"
rows = []
for ev_id in results:
c = txn.execute(sql, (ev_id,))
rows.extend(self.cursor_to_dict(c))
return self._get_events_txn(txn, results)
return self._parse_events_txn(txn, rows)
def get_auth_chain_ids(self, event_id):
def get_auth_chain_ids(self, event_ids):
return self.runInteraction(
"get_auth_chain_ids",
self._get_auth_chain_ids_txn,
event_id
event_ids
)
def _get_auth_chain_ids_txn(self, txn, event_id):
def _get_auth_chain_ids_txn(self, txn, event_ids):
results = set()
base_sql = (
"SELECT auth_id FROM event_auth WHERE %s"
)
front = set([event_id])
front = set(event_ids)
while front:
sql = base_sql % (
" OR ".join(["event_id=?"] * len(front)),

2
synapse/storage/feedback.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/storage/keys.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/storage/media_repository.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
synapse/storage/presence.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
# Copyright 2014, 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

Some files were not shown because too many files have changed in this diff Show more