MirrorHub/synapse
0
0
Fork 1
mirror of https://mau.dev/maunium/synapse.git synced 2024-12-14 16:13:50 +01:00
Code Issues Projects Releases Wiki Activity

Add type hints to matrix federation client / agent. (#8806)

Browse source
This commit is contained in:
Patrick Cloke 2020-11-25 07:07:21 -05:00 committed by GitHub
parent b08dc7effe
commit f38676d161
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 231 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/8806.misc Normal file
View file

@ -0,0 +1 @@
Add type hints to matrix federation client and agent.

2
mypy.ini
View file

@ -45,7 +45,9 @@ files =
synapse/handlers/saml_handler.py, synapse/handlers/saml_handler.py,
synapse/handlers/sync.py, synapse/handlers/sync.py,
synapse/handlers/ui_auth, synapse/handlers/ui_auth,
synapse/http/federation/matrix_federation_agent.py,
synapse/http/federation/well_known_resolver.py, synapse/http/federation/well_known_resolver.py,
synapse/http/matrixfederationclient.py,
synapse/http/server.py, synapse/http/server.py,
synapse/http/site.py, synapse/http/site.py,
synapse/logging, synapse/logging,

100
synapse/http/federation/matrix_federation_agent.py
View file

@ -12,21 +12,25 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import urllib import urllib.parse
from typing import List from typing import List, Optional
from netaddr import AddrFormatError, IPAddress from netaddr import AddrFormatError, IPAddress
from zope.interface import implementer from zope.interface import implementer
from twisted.internet import defer from twisted.internet import defer
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.internet.interfaces import IStreamClientEndpoint from twisted.internet.interfaces import (
from twisted.web.client import Agent, HTTPConnectionPool IProtocolFactory,
IReactorCore,
IStreamClientEndpoint,
)
from twisted.web.client import URI, Agent, HTTPConnectionPool
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
from twisted.web.iweb import IAgent, IAgentEndpointFactory from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer
from synapse.crypto.context_factory import FederationPolicyForHTTPS
from synapse.http.federation.srv_resolver import Server, SrvResolver from synapse.http.federation.srv_resolver import Server, SrvResolver
from synapse.http.federation.well_known_resolver import WellKnownResolver from synapse.http.federation.well_known_resolver import WellKnownResolver
from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.context import make_deferred_yieldable, run_in_background
@ -44,30 +48,30 @@ class MatrixFederationAgent:
Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.) Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.)
Args: Args:
reactor (IReactor): twisted reactor to use for underlying requests reactor: twisted reactor to use for underlying requests
tls_client_options_factory (FederationPolicyForHTTPS|None): tls_client_options_factory:
factory to use for fetching client tls options, or none to disable TLS. factory to use for fetching client tls options, or none to disable TLS.
user_agent (bytes): user_agent:
The user agent header to use for federation requests. The user agent header to use for federation requests.
_srv_resolver (SrvResolver|None): _srv_resolver:
SRVResolver impl to use for looking up SRV records. None to use a default SrvResolver implementation to use for looking up SRV records. None
implementation. to use a default implementation.
_well_known_resolver (WellKnownResolver|None): _well_known_resolver:
WellKnownResolver to use to perform well-known lookups. None to use a WellKnownResolver to use to perform well-known lookups. None to use a
default implementation. default implementation.
""" """
def __init__( def __init__(
self, self,
reactor, reactor: IReactorCore,
tls_client_options_factory, tls_client_options_factory: Optional[FederationPolicyForHTTPS],
user_agent, user_agent: bytes,
_srv_resolver=None, _srv_resolver: Optional[SrvResolver] = None,
_well_known_resolver=None, _well_known_resolver: Optional[WellKnownResolver] = None,
): ):
self._reactor = reactor self._reactor = reactor
self._clock = Clock(reactor) self._clock = Clock(reactor)
@ -99,15 +103,20 @@ class MatrixFederationAgent:
self._well_known_resolver = _well_known_resolver self._well_known_resolver = _well_known_resolver
@defer.inlineCallbacks @defer.inlineCallbacks
def request(self, method, uri, headers=None, bodyProducer=None): def request(
self,
method: bytes,
uri: bytes,
headers: Optional[Headers] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> defer.Deferred:
""" """
Args: Args:
method (bytes): HTTP method: GET/POST/etc method: HTTP method: GET/POST/etc
uri (bytes): Absolute URI to be retrieved uri: Absolute URI to be retrieved
headers (twisted.web.http_headers.Headers|None): headers:
HTTP headers to send with the request, or None to HTTP headers to send with the request, or None to send no extra headers.
send no extra headers. bodyProducer:
bodyProducer (twisted.web.iweb.IBodyProducer|None):
An object which can generate bytes to make up the An object which can generate bytes to make up the
body of this request (for example, the properly encoded contents of body of this request (for example, the properly encoded contents of
a file for a file upload). Or None if the request is to have a file for a file upload). Or None if the request is to have
@ -123,6 +132,9 @@ class MatrixFederationAgent:
# explicit port. # explicit port.
parsed_uri = urllib.parse.urlparse(uri) parsed_uri = urllib.parse.urlparse(uri)
# There must be a valid hostname.
assert parsed_uri.hostname
# If this is a matrix:// URI check if the server has delegated matrix # If this is a matrix:// URI check if the server has delegated matrix
# traffic using well-known delegation. # traffic using well-known delegation.
# #
@ -179,7 +191,12 @@ class MatrixHostnameEndpointFactory:
"""Factory for MatrixHostnameEndpoint for parsing to an Agent. """Factory for MatrixHostnameEndpoint for parsing to an Agent.
""" """
def __init__(self, reactor, tls_client_options_factory, srv_resolver): def __init__(
self,
reactor: IReactorCore,
tls_client_options_factory: Optional[FederationPolicyForHTTPS],
srv_resolver: Optional[SrvResolver],
):
self._reactor = reactor self._reactor = reactor
self._tls_client_options_factory = tls_client_options_factory self._tls_client_options_factory = tls_client_options_factory
@ -203,15 +220,20 @@ class MatrixHostnameEndpoint:
resolution (i.e. via SRV). Does not check for well-known delegation. resolution (i.e. via SRV). Does not check for well-known delegation.
Args: Args:
reactor (IReactor) reactor: twisted reactor to use for underlying requests
tls_client_options_factory (ClientTLSOptionsFactory|None): tls_client_options_factory:
factory to use for fetching client tls options, or none to disable TLS. factory to use for fetching client tls options, or none to disable TLS.
srv_resolver (SrvResolver): The SRV resolver to use srv_resolver: The SRV resolver to use
parsed_uri (twisted.web.client.URI): The parsed URI that we're wanting parsed_uri: The parsed URI that we're wanting to connect to.
to connect to.
""" """
def __init__(self, reactor, tls_client_options_factory, srv_resolver, parsed_uri): def __init__(
self,
reactor: IReactorCore,
tls_client_options_factory: Optional[FederationPolicyForHTTPS],
srv_resolver: SrvResolver,
parsed_uri: URI,
):
self._reactor = reactor self._reactor = reactor
self._parsed_uri = parsed_uri self._parsed_uri = parsed_uri
@ -231,13 +253,13 @@ class MatrixHostnameEndpoint:
self._srv_resolver = srv_resolver self._srv_resolver = srv_resolver
def connect(self, protocol_factory): def connect(self, protocol_factory: IProtocolFactory) -> defer.Deferred:
"""Implements IStreamClientEndpoint interface """Implements IStreamClientEndpoint interface
""" """
return run_in_background(self._do_connect, protocol_factory) return run_in_background(self._do_connect, protocol_factory)
async def _do_connect(self, protocol_factory): async def _do_connect(self, protocol_factory: IProtocolFactory) -> None:
first_exception = None first_exception = None
server_list = await self._resolve_server() server_list = await self._resolve_server()
@ -303,20 +325,20 @@ class MatrixHostnameEndpoint:
return [Server(host, 8448)] return [Server(host, 8448)]
def _is_ip_literal(host): def _is_ip_literal(host: bytes) -> bool:
"""Test if the given host name is either an IPv4 or IPv6 literal. """Test if the given host name is either an IPv4 or IPv6 literal.
Args: Args:
host (bytes) host: The host name to check
Returns: Returns:
bool True if the hostname is an IP address literal.
""" """
host = host.decode("ascii") host_str = host.decode("ascii")
try: try:
IPAddress(host) IPAddress(host_str)
return True return True
except AddrFormatError: except AddrFormatError:
return False return False

16
synapse/http/federation/well_known_resolver.py
View file

@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import random import random
import time import time
@ -21,10 +20,11 @@ from typing import Callable, Dict, Optional, Tuple
import attr import attr
from twisted.internet import defer from twisted.internet import defer
from twisted.internet.interfaces import IReactorTime
from twisted.web.client import RedirectAgent, readBody from twisted.web.client import RedirectAgent, readBody
from twisted.web.http import stringToDatetime from twisted.web.http import stringToDatetime
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
from twisted.web.iweb import IResponse from twisted.web.iweb import IAgent, IResponse
from synapse.logging.context import make_deferred_yieldable from synapse.logging.context import make_deferred_yieldable
from synapse.util import Clock, json_decoder from synapse.util import Clock, json_decoder
@ -81,11 +81,11 @@ class WellKnownResolver:
def __init__( def __init__(
self, self,
reactor, reactor: IReactorTime,
agent, agent: IAgent,
user_agent, user_agent: bytes,
well_known_cache=None, well_known_cache: Optional[TTLCache] = None,
had_well_known_cache=None, had_well_known_cache: Optional[TTLCache] = None,
): ):
self._reactor = reactor self._reactor = reactor
self._clock = Clock(reactor) self._clock = Clock(reactor)
@ -127,7 +127,7 @@ class WellKnownResolver:
with Measure(self._clock, "get_well_known"): with Measure(self._clock, "get_well_known"):
result, cache_period = await self._fetch_well_known( result, cache_period = await self._fetch_well_known(
server_name server_name
) # type: Tuple[Optional[bytes], float] ) # type: Optional[bytes], float
except _FetchWellKnownFailure as e: except _FetchWellKnownFailure as e:
if prev_result and e.temporary: if prev_result and e.temporary:

304
synapse/http/matrixfederationclient.py
View file

@ -17,8 +17,9 @@ import cgi
import logging import logging
import random import random
import sys import sys
import urllib import urllib.parse
from io import BytesIO from io import BytesIO
from typing import BinaryIO, Callable, Dict, List, Optional, Tuple, Union
import attr import attr
import treq import treq
@ -31,9 +32,10 @@ from twisted.internet import defer, protocol
from twisted.internet.error import DNSLookupError from twisted.internet.error import DNSLookupError
from twisted.internet.interfaces import IReactorPluggableNameResolver, IReactorTime from twisted.internet.interfaces import IReactorPluggableNameResolver, IReactorTime
from twisted.internet.task import _EPSILON, Cooperator from twisted.internet.task import _EPSILON, Cooperator
from twisted.python.failure import Failure
from twisted.web._newclient import ResponseDone from twisted.web._newclient import ResponseDone
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
from twisted.web.iweb import IResponse from twisted.web.iweb import IBodyProducer, IResponse
import synapse.metrics import synapse.metrics
import synapse.util.retryutils import synapse.util.retryutils
@ -54,6 +56,7 @@ from synapse.logging.opentracing import (
start_active_span, start_active_span,
tags, tags,
) )
from synapse.types import JsonDict
from synapse.util import json_decoder from synapse.util import json_decoder
from synapse.util.async_helpers import timeout_deferred from synapse.util.async_helpers import timeout_deferred
from synapse.util.metrics import Measure from synapse.util.metrics import Measure
@ -76,47 +79,44 @@ MAXINT = sys.maxsize
_next_id = 1 _next_id = 1
QueryArgs = Dict[str, Union[str, List[str]]]
@attr.s(slots=True, frozen=True) @attr.s(slots=True, frozen=True)
class MatrixFederationRequest: class MatrixFederationRequest:
method = attr.ib() method = attr.ib(type=str)
"""HTTP method """HTTP method
:type: str
""" """
path = attr.ib() path = attr.ib(type=str)
"""HTTP path """HTTP path
:type: str
""" """
destination = attr.ib() destination = attr.ib(type=str)
"""The remote server to send the HTTP request to. """The remote server to send the HTTP request to.
:type: str""" """
json = attr.ib(default=None) json = attr.ib(default=None, type=Optional[JsonDict])
"""JSON to send in the body. """JSON to send in the body.
:type: dict|None
""" """
json_callback = attr.ib(default=None) json_callback = attr.ib(default=None, type=Optional[Callable[[], JsonDict]])
"""A callback to generate the JSON. """A callback to generate the JSON.
:type: func|None
""" """
query = attr.ib(default=None) query = attr.ib(default=None, type=Optional[dict])
"""Query arguments. """Query arguments.
:type: dict|None
""" """
txn_id = attr.ib(default=None) txn_id = attr.ib(default=None, type=Optional[str])
"""Unique ID for this request (for logging) """Unique ID for this request (for logging)
:type: str|None
""" """
uri = attr.ib(init=False, type=bytes) uri = attr.ib(init=False, type=bytes)
"""The URI of this request """The URI of this request
""" """
def __attrs_post_init__(self): def __attrs_post_init__(self) -> None:
global _next_id global _next_id
txn_id = "%s-O-%s" % (self.method, _next_id) txn_id = "%s-O-%s" % (self.method, _next_id)
_next_id = (_next_id + 1) % (MAXINT - 1) _next_id = (_next_id + 1) % (MAXINT - 1)
@ -136,7 +136,7 @@ class MatrixFederationRequest:
) )
object.__setattr__(self, "uri", uri) object.__setattr__(self, "uri", uri)
def get_json(self): def get_json(self) -> Optional[JsonDict]:
if self.json_callback: if self.json_callback:
return self.json_callback() return self.json_callback()
return self.json return self.json
@ -148,7 +148,7 @@ async def _handle_json_response(
request: MatrixFederationRequest, request: MatrixFederationRequest,
response: IResponse, response: IResponse,
start_ms: int, start_ms: int,
): ) -> JsonDict:
""" """
Reads the JSON body of a response, with a timeout Reads the JSON body of a response, with a timeout
@ -160,7 +160,7 @@ async def _handle_json_response(
start_ms: Timestamp when request was made start_ms: Timestamp when request was made
Returns: Returns:
dict: parsed JSON response The parsed JSON response
""" """
try: try:
check_content_type_is_json(response.headers) check_content_type_is_json(response.headers)
@ -266,27 +266,29 @@ class MatrixFederationHttpClient:
self._cooperator = Cooperator(scheduler=schedule) self._cooperator = Cooperator(scheduler=schedule)
async def _send_request_with_optional_trailing_slash( async def _send_request_with_optional_trailing_slash(
self, request, try_trailing_slash_on_400=False, **send_request_args self,
): request: MatrixFederationRequest,
try_trailing_slash_on_400: bool = False,
**send_request_args
) -> IResponse:
"""Wrapper for _send_request which can optionally retry the request """Wrapper for _send_request which can optionally retry the request
upon receiving a combination of a 400 HTTP response code and a upon receiving a combination of a 400 HTTP response code and a
'M_UNRECOGNIZED' errcode. This is a workaround for Synapse <= v0.99.3 'M_UNRECOGNIZED' errcode. This is a workaround for Synapse <= v0.99.3
due to #3622. due to #3622.
Args: Args:
request (MatrixFederationRequest): details of request to be sent request: details of request to be sent
try_trailing_slash_on_400 (bool): Whether on receiving a 400 try_trailing_slash_on_400: Whether on receiving a 400
'M_UNRECOGNIZED' from the server to retry the request with a 'M_UNRECOGNIZED' from the server to retry the request with a
trailing slash appended to the request path. trailing slash appended to the request path.
send_request_args (Dict): A dictionary of arguments to pass to send_request_args: A dictionary of arguments to pass to `_send_request()`.
`_send_request()`.
Raises: Raises:
HttpResponseException: If we get an HTTP response code >= 300 HttpResponseException: If we get an HTTP response code >= 300
(except 429). (except 429).
Returns: Returns:
Dict: Parsed JSON response body. Parsed JSON response body.
""" """
try: try:
response = await self._send_request(request, **send_request_args) response = await self._send_request(request, **send_request_args)
@ -313,24 +315,26 @@ class MatrixFederationHttpClient:
async def _send_request( async def _send_request(
self, self,
request, request: MatrixFederationRequest,
retry_on_dns_fail=True, retry_on_dns_fail: bool = True,
timeout=None, timeout: Optional[int] = None,
long_retries=False, long_retries: bool = False,
ignore_backoff=False, ignore_backoff: bool = False,
backoff_on_404=False, backoff_on_404: bool = False,
): ) -> IResponse:
""" """
Sends a request to the given server. Sends a request to the given server.
Args: Args:
request (MatrixFederationRequest): details of request to be sent request: details of request to be sent
timeout (int|None): number of milliseconds to wait for the response headers retry_on_dns_fail: true if the request should be retied on DNS failures
timeout: number of milliseconds to wait for the response headers
(including connecting to the server), *for each attempt*. (including connecting to the server), *for each attempt*.
60s by default. 60s by default.
long_retries (bool): whether to use the long retry algorithm. long_retries: whether to use the long retry algorithm.
The regular retry algorithm makes 4 attempts, with intervals The regular retry algorithm makes 4 attempts, with intervals
[0.5s, 1s, 2s]. [0.5s, 1s, 2s].
@ -346,14 +350,13 @@ class MatrixFederationHttpClient:
NB: the long retry algorithm takes over 20 minutes to complete, with NB: the long retry algorithm takes over 20 minutes to complete, with
a default timeout of 60s! a default timeout of 60s!
ignore_backoff (bool): true to ignore the historical backoff data ignore_backoff: true to ignore the historical backoff data
and try the request anyway. and try the request anyway.
backoff_on_404 (bool): Back off if we get a 404 backoff_on_404: Back off if we get a 404
Returns: Returns:
twisted.web.client.Response: resolves with the HTTP Resolves with the HTTP response object on success.
response object on success.
Raises: Raises:
HttpResponseException: If we get an HTTP response code >= 300 HttpResponseException: If we get an HTTP response code >= 300
@ -404,7 +407,7 @@ class MatrixFederationHttpClient:
) )
# Inject the span into the headers # Inject the span into the headers
headers_dict = {} headers_dict = {} # type: Dict[bytes, List[bytes]]
inject_active_span_byte_dict(headers_dict, request.destination) inject_active_span_byte_dict(headers_dict, request.destination)
headers_dict[b"User-Agent"] = [self.version_string_bytes] headers_dict[b"User-Agent"] = [self.version_string_bytes]
@ -435,7 +438,7 @@ class MatrixFederationHttpClient:
data = encode_canonical_json(json) data = encode_canonical_json(json)
producer = QuieterFileBodyProducer( producer = QuieterFileBodyProducer(
BytesIO(data), cooperator=self._cooperator BytesIO(data), cooperator=self._cooperator
) ) # type: Optional[IBodyProducer]
else: else:
producer = None producer = None
auth_headers = self.build_auth_headers( auth_headers = self.build_auth_headers(
@ -524,14 +527,16 @@ class MatrixFederationHttpClient:
) )
body = None body = None
e = HttpResponseException(response.code, response_phrase, body) exc = HttpResponseException(
response.code, response_phrase, body
)
# Retry if the error is a 429 (Too Many Requests), # Retry if the error is a 429 (Too Many Requests),
# otherwise just raise a standard HttpResponseException # otherwise just raise a standard HttpResponseException
if response.code == 429: if response.code == 429:
raise RequestSendFailed(e, can_retry=True) from e raise RequestSendFailed(exc, can_retry=True) from exc
else: else:
raise e raise exc
break break
except RequestSendFailed as e: except RequestSendFailed as e:
@ -582,22 +587,27 @@ class MatrixFederationHttpClient:
return response return response
def build_auth_headers( def build_auth_headers(
self, destination, method, url_bytes, content=None, destination_is=None self,
): destination: Optional[bytes],
method: bytes,
url_bytes: bytes,
content: Optional[JsonDict] = None,
destination_is: Optional[bytes] = None,
) -> List[bytes]:
""" """
Builds the Authorization headers for a federation request Builds the Authorization headers for a federation request
Args: Args:
destination (bytes|None): The destination homeserver of the request. destination: The destination homeserver of the request.
May be None if the destination is an identity server, in which case May be None if the destination is an identity server, in which case
destination_is must be non-None. destination_is must be non-None.
method (bytes): The HTTP method of the request method: The HTTP method of the request
url_bytes (bytes): The URI path of the request url_bytes: The URI path of the request
content (object): The body of the request content: The body of the request
destination_is (bytes): As 'destination', but if the destination is an destination_is: As 'destination', but if the destination is an
identity server identity server
Returns: Returns:
list[bytes]: a list of headers to be added as "Authorization:" headers A list of headers to be added as "Authorization:" headers
""" """
request = { request = {
"method": method.decode("ascii"), "method": method.decode("ascii"),
@ -629,33 +639,32 @@ class MatrixFederationHttpClient:
async def put_json( async def put_json(
self, self,
destination, destination: str,
path, path: str,
args={}, args: Optional[QueryArgs] = None,
data={}, data: Optional[JsonDict] = None,
json_data_callback=None, json_data_callback: Optional[Callable[[], JsonDict]] = None,
long_retries=False, long_retries: bool = False,
timeout=None, timeout: Optional[int] = None,
ignore_backoff=False, ignore_backoff: bool = False,
backoff_on_404=False, backoff_on_404: bool = False,
try_trailing_slash_on_400=False, try_trailing_slash_on_400: bool = False,
): ) -> Union[JsonDict, list]:
""" Sends the specified json data using PUT """ Sends the specified json data using PUT
Args: Args:
destination (str): The remote server to send the HTTP request destination: The remote server to send the HTTP request to.
to. path: The HTTP path.
path (str): The HTTP path. args: query params
args (dict): query params data: A dict containing the data that will be used as
data (dict): A dict containing the data that will be used as
the request body. This will be encoded as JSON. the request body. This will be encoded as JSON.
json_data_callback (callable): A callable returning the dict to json_data_callback: A callable returning the dict to
use as the request body. use as the request body.
long_retries (bool): whether to use the long retry algorithm. See long_retries: whether to use the long retry algorithm. See
docs on _send_request for details. docs on _send_request for details.
timeout (int|None): number of milliseconds to wait for the response. timeout: number of milliseconds to wait for the response.
self._default_timeout (60s) by default. self._default_timeout (60s) by default.
Note that we may make several attempts to send the request; this Note that we may make several attempts to send the request; this
@ -663,19 +672,19 @@ class MatrixFederationHttpClient:
*each* attempt (including connection time) as well as the time spent *each* attempt (including connection time) as well as the time spent
reading the response body after a 200 response. reading the response body after a 200 response.
ignore_backoff (bool): true to ignore the historical backoff data ignore_backoff: true to ignore the historical backoff data
and try the request anyway. and try the request anyway.
backoff_on_404 (bool): True if we should count a 404 response as backoff_on_404: True if we should count a 404 response as
a failure of the server (and should therefore back off future a failure of the server (and should therefore back off future
requests). requests).
try_trailing_slash_on_400 (bool): True if on a 400 M_UNRECOGNIZED try_trailing_slash_on_400: True if on a 400 M_UNRECOGNIZED
response we should try appending a trailing slash to the end response we should try appending a trailing slash to the end
of the request. Workaround for #3622 in Synapse <= v0.99.3. This of the request. Workaround for #3622 in Synapse <= v0.99.3. This
will be attempted before backing off if backing off has been will be attempted before backing off if backing off has been
enabled. enabled.
Returns: Returns:
dict|list: Succeeds when we get a 2xx HTTP response. The Succeeds when we get a 2xx HTTP response. The
result will be the decoded JSON body. result will be the decoded JSON body.
Raises: Raises:
@ -721,29 +730,28 @@ class MatrixFederationHttpClient:
async def post_json( async def post_json(
self, self,
destination, destination: str,
path, path: str,
data={}, data: Optional[JsonDict] = None,
long_retries=False, long_retries: bool = False,
timeout=None, timeout: Optional[int] = None,
ignore_backoff=False, ignore_backoff: bool = False,
args={}, args: Optional[QueryArgs] = None,
): ) -> Union[JsonDict, list]:
""" Sends the specified json data using POST """ Sends the specified json data using POST
Args: Args:
destination (str): The remote server to send the HTTP request destination: The remote server to send the HTTP request to.
to.
path (str): The HTTP path. path: The HTTP path.
data (dict): A dict containing the data that will be used as data: A dict containing the data that will be used as
the request body. This will be encoded as JSON. the request body. This will be encoded as JSON.
long_retries (bool): whether to use the long retry algorithm. See long_retries: whether to use the long retry algorithm. See
docs on _send_request for details. docs on _send_request for details.
timeout (int|None): number of milliseconds to wait for the response. timeout: number of milliseconds to wait for the response.
self._default_timeout (60s) by default. self._default_timeout (60s) by default.
Note that we may make several attempts to send the request; this Note that we may make several attempts to send the request; this
@ -751,10 +759,10 @@ class MatrixFederationHttpClient:
*each* attempt (including connection time) as well as the time spent *each* attempt (including connection time) as well as the time spent
reading the response body after a 200 response. reading the response body after a 200 response.
ignore_backoff (bool): true to ignore the historical backoff data and ignore_backoff: true to ignore the historical backoff data and
try the request anyway. try the request anyway.
args (dict): query params args: query params
Returns: Returns:
dict|list: Succeeds when we get a 2xx HTTP response. The dict|list: Succeeds when we get a 2xx HTTP response. The
result will be the decoded JSON body. result will be the decoded JSON body.
@ -795,26 +803,25 @@ class MatrixFederationHttpClient:
async def get_json( async def get_json(
self, self,
destination, destination: str,
path, path: str,
args=None, args: Optional[QueryArgs] = None,
retry_on_dns_fail=True, retry_on_dns_fail: bool = True,
timeout=None, timeout: Optional[int] = None,
ignore_backoff=False, ignore_backoff: bool = False,
try_trailing_slash_on_400=False, try_trailing_slash_on_400: bool = False,
): ) -> Union[JsonDict, list]:
""" GETs some json from the given host homeserver and path """ GETs some json from the given host homeserver and path
Args: Args:
destination (str): The remote server to send the HTTP request destination: The remote server to send the HTTP request to.
to.
path (str): The HTTP path. path: The HTTP path.
args (dict|None): A dictionary used to create query strings, defaults to args: A dictionary used to create query strings, defaults to
None. None.
timeout (int|None): number of milliseconds to wait for the response. timeout: number of milliseconds to wait for the response.
self._default_timeout (60s) by default. self._default_timeout (60s) by default.
Note that we may make several attempts to send the request; this Note that we may make several attempts to send the request; this
@ -822,14 +829,14 @@ class MatrixFederationHttpClient:
*each* attempt (including connection time) as well as the time spent *each* attempt (including connection time) as well as the time spent
reading the response body after a 200 response. reading the response body after a 200 response.
ignore_backoff (bool): true to ignore the historical backoff data ignore_backoff: true to ignore the historical backoff data
and try the request anyway. and try the request anyway.
try_trailing_slash_on_400 (bool): True if on a 400 M_UNRECOGNIZED try_trailing_slash_on_400: True if on a 400 M_UNRECOGNIZED
response we should try appending a trailing slash to the end of response we should try appending a trailing slash to the end of
the request. Workaround for #3622 in Synapse <= v0.99.3. the request. Workaround for #3622 in Synapse <= v0.99.3.
Returns: Returns:
dict|list: Succeeds when we get a 2xx HTTP response. The Succeeds when we get a 2xx HTTP response. The
result will be the decoded JSON body. result will be the decoded JSON body.
Raises: Raises:
@ -870,24 +877,23 @@ class MatrixFederationHttpClient:
async def delete_json( async def delete_json(
self, self,
destination, destination: str,
path, path: str,
long_retries=False, long_retries: bool = False,
timeout=None, timeout: Optional[int] = None,
ignore_backoff=False, ignore_backoff: bool = False,
args={}, args: Optional[QueryArgs] = None,
): ) -> Union[JsonDict, list]:
"""Send a DELETE request to the remote expecting some json response """Send a DELETE request to the remote expecting some json response
Args: Args:
destination (str): The remote server to send the HTTP request destination: The remote server to send the HTTP request to.
to. path: The HTTP path.
path (str): The HTTP path.
long_retries (bool): whether to use the long retry algorithm. See long_retries: whether to use the long retry algorithm. See
docs on _send_request for details. docs on _send_request for details.
timeout (int|None): number of milliseconds to wait for the response. timeout: number of milliseconds to wait for the response.
self._default_timeout (60s) by default. self._default_timeout (60s) by default.
Note that we may make several attempts to send the request; this Note that we may make several attempts to send the request; this
@ -895,12 +901,12 @@ class MatrixFederationHttpClient:
*each* attempt (including connection time) as well as the time spent *each* attempt (including connection time) as well as the time spent
reading the response body after a 200 response. reading the response body after a 200 response.
ignore_backoff (bool): true to ignore the historical backoff data and ignore_backoff: true to ignore the historical backoff data and
try the request anyway. try the request anyway.
args (dict): query params args: query params
Returns: Returns:
dict|list: Succeeds when we get a 2xx HTTP response. The Succeeds when we get a 2xx HTTP response. The
result will be the decoded JSON body. result will be the decoded JSON body.
Raises: Raises:
@ -938,25 +944,25 @@ class MatrixFederationHttpClient:
async def get_file( async def get_file(
self, self,
destination, destination: str,
path, path: str,
output_stream, output_stream,
args={}, args: Optional[QueryArgs] = None,
retry_on_dns_fail=True, retry_on_dns_fail: bool = True,
max_size=None, max_size: Optional[int] = None,
ignore_backoff=False, ignore_backoff: bool = False,
): ) -> Tuple[int, Dict[bytes, List[bytes]]]:
"""GETs a file from a given homeserver """GETs a file from a given homeserver
Args: Args:
destination (str): The remote server to send the HTTP request to. destination: The remote server to send the HTTP request to.
path (str): The HTTP path to GET. path: The HTTP path to GET.
output_stream (file): File to write the response body to. output_stream: File to write the response body to.
args (dict): Optional dictionary used to create the query string. args: Optional dictionary used to create the query string.
ignore_backoff (bool): true to ignore the historical backoff data ignore_backoff: true to ignore the historical backoff data
and try the request anyway. and try the request anyway.
Returns: Returns:
tuple[int, dict]: Resolves with an (int,dict) tuple of Resolves with an (int,dict) tuple of
the file length and a dict of the response headers. the file length and a dict of the response headers.
Raises: Raises:
@ -1005,13 +1011,15 @@ class MatrixFederationHttpClient:
class _ReadBodyToFileProtocol(protocol.Protocol): class _ReadBodyToFileProtocol(protocol.Protocol):
def __init__(self, stream, deferred, max_size): def __init__(
self, stream: BinaryIO, deferred: defer.Deferred, max_size: Optional[int]
):
self.stream = stream self.stream = stream
self.deferred = deferred self.deferred = deferred
self.length = 0 self.length = 0
self.max_size = max_size self.max_size = max_size
def dataReceived(self, data): def dataReceived(self, data: bytes) -> None:
self.stream.write(data) self.stream.write(data)
self.length += len(data) self.length += len(data)
if self.max_size is not None and self.length >= self.max_size: if self.max_size is not None and self.length >= self.max_size:
@ -1025,14 +1033,16 @@ class _ReadBodyToFileProtocol(protocol.Protocol):
self.deferred = defer.Deferred() self.deferred = defer.Deferred()
self.transport.loseConnection() self.transport.loseConnection()
def connectionLost(self, reason): def connectionLost(self, reason: Failure) -> None:
if reason.check(ResponseDone): if reason.check(ResponseDone):
self.deferred.callback(self.length) self.deferred.callback(self.length)
else: else:
self.deferred.errback(reason) self.deferred.errback(reason)
def _readBodyToFile(response, stream, max_size): def _readBodyToFile(
response: IResponse, stream: BinaryIO, max_size: Optional[int]
) -> defer.Deferred:
d = defer.Deferred() d = defer.Deferred()
response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size)) response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size))
return d return d
@ -1049,13 +1059,13 @@ def _flatten_response_never_received(e):
return repr(e) return repr(e)
def check_content_type_is_json(headers): def check_content_type_is_json(headers: Headers) -> None:
""" """
Check that a set of HTTP headers have a Content-Type header, and that it Check that a set of HTTP headers have a Content-Type header, and that it
is application/json. is application/json.
Args: Args:
headers (twisted.web.http_headers.Headers): headers to check headers: headers to check
Raises: Raises:
RequestSendFailed: if the Content-Type header is missing or isn't JSON RequestSendFailed: if the Content-Type header is missing or isn't JSON
@ -1080,7 +1090,7 @@ def check_content_type_is_json(headers):
) )
def encode_query_args(args): def encode_query_args(args: Optional[QueryArgs]) -> bytes:
if args is None: if args is None:
return b"" return b""
@ -1088,8 +1098,8 @@ def encode_query_args(args):
for k, vs in args.items(): for k, vs in args.items():
if isinstance(vs, str): if isinstance(vs, str):
vs = [vs] vs = [vs]
encoded_args[k] = [v.encode("UTF-8") for v in vs] encoded_args[k] = [v.encode("utf8") for v in vs]
query_bytes = urllib.parse.urlencode(encoded_args, True) query_str = urllib.parse.urlencode(encoded_args, True)
return query_bytes.encode("utf8") return query_str.encode("utf8")

3
synapse/server.py
View file

@ -27,7 +27,8 @@ import logging
import os import os
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, cast from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, cast
import twisted import twisted.internet.base
import twisted.internet.tcp
from twisted.mail.smtp import sendmail from twisted.mail.smtp import sendmail
from twisted.web.iweb import IPolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS