Port http/ to Python 3 (#3771)

This commit is contained in:
Amber Brown 2018-09-06 00:10:47 +10:00 committed by GitHub
parent a6cf7d9d9a
commit 2d2828dcbc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 134 additions and 186 deletions

1
changelog.d/3771.misc Normal file
View file

@ -0,0 +1 @@
http/ is now ported to Python 3.

View file

@ -17,13 +17,14 @@ ignore =
[pep8] [pep8]
max-line-length = 90 max-line-length = 90
# W503 requires that binary operators be at the end, not start, of lines. Erik # W503 requires that binary operators be at the end, not start, of lines. Erik
# doesn't like it. E203 is contrary to PEP8. # doesn't like it. E203 is contrary to PEP8. E731 is silly.
ignore = W503,E203 ignore = W503,E203,E731
[flake8] [flake8]
# note that flake8 inherits the "ignore" settings from "pep8" (because it uses # note that flake8 inherits the "ignore" settings from "pep8" (because it uses
# pep8 to do those checks), but not the "max-line-length" setting # pep8 to do those checks), but not the "max-line-length" setting
max-line-length = 90 max-line-length = 90
ignore=W503,E203,E731
[isort] [isort]
line_length = 89 line_length = 89

View file

@ -13,7 +13,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import urllib
from six.moves import urllib
from prometheus_client import Counter from prometheus_client import Counter
@ -98,7 +99,7 @@ class ApplicationServiceApi(SimpleHttpClient):
def query_user(self, service, user_id): def query_user(self, service, user_id):
if service.url is None: if service.url is None:
defer.returnValue(False) defer.returnValue(False)
uri = service.url + ("/users/%s" % urllib.quote(user_id)) uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
response = None response = None
try: try:
response = yield self.get_json(uri, { response = yield self.get_json(uri, {
@ -119,7 +120,7 @@ class ApplicationServiceApi(SimpleHttpClient):
def query_alias(self, service, alias): def query_alias(self, service, alias):
if service.url is None: if service.url is None:
defer.returnValue(False) defer.returnValue(False)
uri = service.url + ("/rooms/%s" % urllib.quote(alias)) uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
response = None response = None
try: try:
response = yield self.get_json(uri, { response = yield self.get_json(uri, {
@ -153,7 +154,7 @@ class ApplicationServiceApi(SimpleHttpClient):
service.url, service.url,
APP_SERVICE_PREFIX, APP_SERVICE_PREFIX,
kind, kind,
urllib.quote(protocol) urllib.parse.quote(protocol)
) )
try: try:
response = yield self.get_json(uri, fields) response = yield self.get_json(uri, fields)
@ -188,7 +189,7 @@ class ApplicationServiceApi(SimpleHttpClient):
uri = "%s%s/thirdparty/protocol/%s" % ( uri = "%s%s/thirdparty/protocol/%s" % (
service.url, service.url,
APP_SERVICE_PREFIX, APP_SERVICE_PREFIX,
urllib.quote(protocol) urllib.parse.quote(protocol)
) )
try: try:
info = yield self.get_json(uri, {}) info = yield self.get_json(uri, {})
@ -228,7 +229,7 @@ class ApplicationServiceApi(SimpleHttpClient):
txn_id = str(txn_id) txn_id = str(txn_id)
uri = service.url + ("/transactions/%s" % uri = service.url + ("/transactions/%s" %
urllib.quote(txn_id)) urllib.parse.quote(txn_id))
try: try:
yield self.put_json( yield self.put_json(
uri=uri, uri=uri,

View file

@ -13,24 +13,25 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import urllib
from six import StringIO from six import text_type
from six.moves import urllib
import treq
from canonicaljson import encode_canonical_json, json from canonicaljson import encode_canonical_json, json
from prometheus_client import Counter from prometheus_client import Counter
from OpenSSL import SSL from OpenSSL import SSL
from OpenSSL.SSL import VERIFY_NONE from OpenSSL.SSL import VERIFY_NONE
from twisted.internet import defer, protocol, reactor, ssl, task from twisted.internet import defer, protocol, reactor, ssl
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.web._newclient import ResponseDone from twisted.web._newclient import ResponseDone
from twisted.web.client import ( from twisted.web.client import (
Agent, Agent,
BrowserLikeRedirectAgent, BrowserLikeRedirectAgent,
ContentDecoderAgent, ContentDecoderAgent,
FileBodyProducer as TwistedFileBodyProducer,
GzipDecoder, GzipDecoder,
HTTPConnectionPool, HTTPConnectionPool,
PartialDownloadError, PartialDownloadError,
@ -83,18 +84,20 @@ class SimpleHttpClient(object):
if hs.config.user_agent_suffix: if hs.config.user_agent_suffix:
self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,) self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,)
self.user_agent = self.user_agent.encode('ascii')
@defer.inlineCallbacks @defer.inlineCallbacks
def request(self, method, uri, *args, **kwargs): def request(self, method, uri, data=b'', headers=None):
# A small wrapper around self.agent.request() so we can easily attach # A small wrapper around self.agent.request() so we can easily attach
# counters to it # counters to it
outgoing_requests_counter.labels(method).inc() outgoing_requests_counter.labels(method).inc()
# log request but strip `access_token` (AS requests for example include this) # log request but strip `access_token` (AS requests for example include this)
logger.info("Sending request %s %s", method, redact_uri(uri)) logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii')))
try: try:
request_deferred = self.agent.request( request_deferred = treq.request(
method, uri, *args, **kwargs method, uri, agent=self.agent, data=data, headers=headers
) )
add_timeout_to_deferred( add_timeout_to_deferred(
request_deferred, 60, self.hs.get_reactor(), request_deferred, 60, self.hs.get_reactor(),
@ -105,14 +108,14 @@ class SimpleHttpClient(object):
incoming_responses_counter.labels(method, response.code).inc() incoming_responses_counter.labels(method, response.code).inc()
logger.info( logger.info(
"Received response to %s %s: %s", "Received response to %s %s: %s",
method, redact_uri(uri), response.code method, redact_uri(uri.encode('ascii')), response.code
) )
defer.returnValue(response) defer.returnValue(response)
except Exception as e: except Exception as e:
incoming_responses_counter.labels(method, "ERR").inc() incoming_responses_counter.labels(method, "ERR").inc()
logger.info( logger.info(
"Error sending request to %s %s: %s %s", "Error sending request to %s %s: %s %s",
method, redact_uri(uri), type(e).__name__, e.message method, redact_uri(uri.encode('ascii')), type(e).__name__, e.args[0]
) )
raise raise
@ -137,7 +140,8 @@ class SimpleHttpClient(object):
# TODO: Do we ever want to log message contents? # TODO: Do we ever want to log message contents?
logger.debug("post_urlencoded_get_json args: %s", args) logger.debug("post_urlencoded_get_json args: %s", args)
query_bytes = urllib.urlencode(encode_urlencode_args(args), True) query_bytes = urllib.parse.urlencode(
encode_urlencode_args(args), True).encode("utf8")
actual_headers = { actual_headers = {
b"Content-Type": [b"application/x-www-form-urlencoded"], b"Content-Type": [b"application/x-www-form-urlencoded"],
@ -148,15 +152,14 @@ class SimpleHttpClient(object):
response = yield self.request( response = yield self.request(
"POST", "POST",
uri.encode("ascii"), uri,
headers=Headers(actual_headers), headers=Headers(actual_headers),
bodyProducer=FileBodyProducer(StringIO(query_bytes)) data=query_bytes
) )
body = yield make_deferred_yieldable(readBody(response))
if 200 <= response.code < 300: if 200 <= response.code < 300:
defer.returnValue(json.loads(body)) body = yield make_deferred_yieldable(treq.json_content(response))
defer.returnValue(body)
else: else:
raise HttpResponseException(response.code, response.phrase, body) raise HttpResponseException(response.code, response.phrase, body)
@ -191,9 +194,9 @@ class SimpleHttpClient(object):
response = yield self.request( response = yield self.request(
"POST", "POST",
uri.encode("ascii"), uri,
headers=Headers(actual_headers), headers=Headers(actual_headers),
bodyProducer=FileBodyProducer(StringIO(json_str)) data=json_str
) )
body = yield make_deferred_yieldable(readBody(response)) body = yield make_deferred_yieldable(readBody(response))
@ -248,7 +251,7 @@ class SimpleHttpClient(object):
ValueError: if the response was not JSON ValueError: if the response was not JSON
""" """
if len(args): if len(args):
query_bytes = urllib.urlencode(args, True) query_bytes = urllib.parse.urlencode(args, True)
uri = "%s?%s" % (uri, query_bytes) uri = "%s?%s" % (uri, query_bytes)
json_str = encode_canonical_json(json_body) json_str = encode_canonical_json(json_body)
@ -262,9 +265,9 @@ class SimpleHttpClient(object):
response = yield self.request( response = yield self.request(
"PUT", "PUT",
uri.encode("ascii"), uri,
headers=Headers(actual_headers), headers=Headers(actual_headers),
bodyProducer=FileBodyProducer(StringIO(json_str)) data=json_str
) )
body = yield make_deferred_yieldable(readBody(response)) body = yield make_deferred_yieldable(readBody(response))
@ -293,7 +296,7 @@ class SimpleHttpClient(object):
HttpResponseException on a non-2xx HTTP response. HttpResponseException on a non-2xx HTTP response.
""" """
if len(args): if len(args):
query_bytes = urllib.urlencode(args, True) query_bytes = urllib.parse.urlencode(args, True)
uri = "%s?%s" % (uri, query_bytes) uri = "%s?%s" % (uri, query_bytes)
actual_headers = { actual_headers = {
@ -304,7 +307,7 @@ class SimpleHttpClient(object):
response = yield self.request( response = yield self.request(
"GET", "GET",
uri.encode("ascii"), uri,
headers=Headers(actual_headers), headers=Headers(actual_headers),
) )
@ -339,7 +342,7 @@ class SimpleHttpClient(object):
response = yield self.request( response = yield self.request(
"GET", "GET",
url.encode("ascii"), url,
headers=Headers(actual_headers), headers=Headers(actual_headers),
) )
@ -434,12 +437,12 @@ class CaptchaServerHttpClient(SimpleHttpClient):
@defer.inlineCallbacks @defer.inlineCallbacks
def post_urlencoded_get_raw(self, url, args={}): def post_urlencoded_get_raw(self, url, args={}):
query_bytes = urllib.urlencode(encode_urlencode_args(args), True) query_bytes = urllib.parse.urlencode(encode_urlencode_args(args), True)
response = yield self.request( response = yield self.request(
"POST", "POST",
url.encode("ascii"), url,
bodyProducer=FileBodyProducer(StringIO(query_bytes)), data=query_bytes,
headers=Headers({ headers=Headers({
b"Content-Type": [b"application/x-www-form-urlencoded"], b"Content-Type": [b"application/x-www-form-urlencoded"],
b"User-Agent": [self.user_agent], b"User-Agent": [self.user_agent],
@ -510,7 +513,7 @@ def encode_urlencode_args(args):
def encode_urlencode_arg(arg): def encode_urlencode_arg(arg):
if isinstance(arg, unicode): if isinstance(arg, text_type):
return arg.encode('utf-8') return arg.encode('utf-8')
elif isinstance(arg, list): elif isinstance(arg, list):
return [encode_urlencode_arg(i) for i in arg] return [encode_urlencode_arg(i) for i in arg]
@ -542,26 +545,3 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory):
def creatorForNetloc(self, hostname, port): def creatorForNetloc(self, hostname, port):
return self return self
class FileBodyProducer(TwistedFileBodyProducer):
"""Workaround for https://twistedmatrix.com/trac/ticket/8473
We override the pauseProducing and resumeProducing methods in twisted's
FileBodyProducer so that they do not raise exceptions if the task has
already completed.
"""
def pauseProducing(self):
try:
super(FileBodyProducer, self).pauseProducing()
except task.TaskDone:
# task has already completed
pass
def resumeProducing(self):
try:
super(FileBodyProducer, self).resumeProducing()
except task.NotPaused:
# task was not paused (probably because it had already completed)
pass

View file

@ -17,19 +17,19 @@ import cgi
import logging import logging
import random import random
import sys import sys
import urllib
from six import string_types from six import PY3, string_types
from six.moves.urllib import parse as urlparse from six.moves import urllib
from canonicaljson import encode_canonical_json, json import treq
from canonicaljson import encode_canonical_json
from prometheus_client import Counter from prometheus_client import Counter
from signedjson.sign import sign_json from signedjson.sign import sign_json
from twisted.internet import defer, protocol, reactor from twisted.internet import defer, protocol, reactor
from twisted.internet.error import DNSLookupError from twisted.internet.error import DNSLookupError
from twisted.web._newclient import ResponseDone from twisted.web._newclient import ResponseDone
from twisted.web.client import Agent, HTTPConnectionPool, readBody from twisted.web.client import Agent, HTTPConnectionPool
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
import synapse.metrics import synapse.metrics
@ -58,13 +58,18 @@ incoming_responses_counter = Counter("synapse_http_matrixfederationclient_respon
MAX_LONG_RETRIES = 10 MAX_LONG_RETRIES = 10
MAX_SHORT_RETRIES = 3 MAX_SHORT_RETRIES = 3
if PY3:
MAXINT = sys.maxsize
else:
MAXINT = sys.maxint
class MatrixFederationEndpointFactory(object): class MatrixFederationEndpointFactory(object):
def __init__(self, hs): def __init__(self, hs):
self.tls_client_options_factory = hs.tls_client_options_factory self.tls_client_options_factory = hs.tls_client_options_factory
def endpointForURI(self, uri): def endpointForURI(self, uri):
destination = uri.netloc destination = uri.netloc.decode('ascii')
return matrix_federation_endpoint( return matrix_federation_endpoint(
reactor, destination, timeout=10, reactor, destination, timeout=10,
@ -93,26 +98,32 @@ class MatrixFederationHttpClient(object):
) )
self.clock = hs.get_clock() self.clock = hs.get_clock()
self._store = hs.get_datastore() self._store = hs.get_datastore()
self.version_string = hs.version_string self.version_string = hs.version_string.encode('ascii')
self._next_id = 1 self._next_id = 1
def _create_url(self, destination, path_bytes, param_bytes, query_bytes): def _create_url(self, destination, path_bytes, param_bytes, query_bytes):
return urlparse.urlunparse( return urllib.parse.urlunparse(
("matrix", destination, path_bytes, param_bytes, query_bytes, "") (b"matrix", destination, path_bytes, param_bytes, query_bytes, b"")
) )
@defer.inlineCallbacks @defer.inlineCallbacks
def _request(self, destination, method, path, def _request(self, destination, method, path,
body_callback, headers_dict={}, param_bytes=b"", json=None, json_callback=None,
query_bytes=b"", retry_on_dns_fail=True, param_bytes=b"",
query=None, retry_on_dns_fail=True,
timeout=None, long_retries=False, timeout=None, long_retries=False,
ignore_backoff=False, ignore_backoff=False,
backoff_on_404=False): backoff_on_404=False):
""" Creates and sends a request to the given server """
Creates and sends a request to the given server.
Args: Args:
destination (str): The remote server to send the HTTP request to. destination (str): The remote server to send the HTTP request to.
method (str): HTTP method method (str): HTTP method
path (str): The HTTP path path (str): The HTTP path
json (dict or None): JSON to send in the body.
json_callback (func or None): A callback to generate the JSON.
query (dict or None): Query arguments.
ignore_backoff (bool): true to ignore the historical backoff data ignore_backoff (bool): true to ignore the historical backoff data
and try the request anyway. and try the request anyway.
backoff_on_404 (bool): Back off if we get a 404 backoff_on_404 (bool): Back off if we get a 404
@ -146,22 +157,29 @@ class MatrixFederationHttpClient(object):
ignore_backoff=ignore_backoff, ignore_backoff=ignore_backoff,
) )
destination = destination.encode("ascii") headers_dict = {}
path_bytes = path.encode("ascii") path_bytes = path.encode("ascii")
with limiter: if query:
headers_dict[b"User-Agent"] = [self.version_string] query_bytes = encode_query_args(query)
headers_dict[b"Host"] = [destination] else:
query_bytes = b""
url_bytes = self._create_url( headers_dict = {
destination, path_bytes, param_bytes, query_bytes "User-Agent": [self.version_string],
) "Host": [destination],
}
with limiter:
url = self._create_url(
destination.encode("ascii"), path_bytes, param_bytes, query_bytes
).decode('ascii')
txn_id = "%s-O-%s" % (method, self._next_id) txn_id = "%s-O-%s" % (method, self._next_id)
self._next_id = (self._next_id + 1) % (sys.maxint - 1) self._next_id = (self._next_id + 1) % (MAXINT - 1)
outbound_logger.info( outbound_logger.info(
"{%s} [%s] Sending request: %s %s", "{%s} [%s] Sending request: %s %s",
txn_id, destination, method, url_bytes txn_id, destination, method, url
) )
# XXX: Would be much nicer to retry only at the transaction-layer # XXX: Would be much nicer to retry only at the transaction-layer
@ -171,23 +189,33 @@ class MatrixFederationHttpClient(object):
else: else:
retries_left = MAX_SHORT_RETRIES retries_left = MAX_SHORT_RETRIES
http_url_bytes = urlparse.urlunparse( http_url = urllib.parse.urlunparse(
("", "", path_bytes, param_bytes, query_bytes, "") (b"", b"", path_bytes, param_bytes, query_bytes, b"")
) ).decode('ascii')
log_result = None log_result = None
try: try:
while True: while True:
producer = None
if body_callback:
producer = body_callback(method, http_url_bytes, headers_dict)
try: try:
request_deferred = self.agent.request( if json_callback:
json = json_callback()
if json:
data = encode_canonical_json(json)
headers_dict["Content-Type"] = ["application/json"]
self.sign_request(
destination, method, http_url, headers_dict, json
)
else:
data = None
self.sign_request(destination, method, http_url, headers_dict)
request_deferred = treq.request(
method, method,
url_bytes, url,
Headers(headers_dict), headers=Headers(headers_dict),
producer data=data,
agent=self.agent,
) )
add_timeout_to_deferred( add_timeout_to_deferred(
request_deferred, request_deferred,
@ -218,7 +246,7 @@ class MatrixFederationHttpClient(object):
txn_id, txn_id,
destination, destination,
method, method,
url_bytes, url,
_flatten_response_never_received(e), _flatten_response_never_received(e),
) )
@ -252,7 +280,7 @@ class MatrixFederationHttpClient(object):
# :'( # :'(
# Update transactions table? # Update transactions table?
with logcontext.PreserveLoggingContext(): with logcontext.PreserveLoggingContext():
body = yield readBody(response) body = yield treq.content(response)
raise HttpResponseException( raise HttpResponseException(
response.code, response.phrase, body response.code, response.phrase, body
) )
@ -297,11 +325,11 @@ class MatrixFederationHttpClient(object):
auth_headers = [] auth_headers = []
for key, sig in request["signatures"][self.server_name].items(): for key, sig in request["signatures"][self.server_name].items():
auth_headers.append(bytes( auth_headers.append((
"X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % (
self.server_name, key, sig, self.server_name, key, sig,
)).encode('ascii')
) )
))
headers_dict[b"Authorization"] = auth_headers headers_dict[b"Authorization"] = auth_headers
@ -347,24 +375,14 @@ class MatrixFederationHttpClient(object):
""" """
if not json_data_callback: if not json_data_callback:
def json_data_callback(): json_data_callback = lambda: data
return data
def body_callback(method, url_bytes, headers_dict):
json_data = json_data_callback()
self.sign_request(
destination, method, url_bytes, headers_dict, json_data
)
producer = _JsonProducer(json_data)
return producer
response = yield self._request( response = yield self._request(
destination, destination,
"PUT", "PUT",
path, path,
body_callback=body_callback, json_callback=json_data_callback,
headers_dict={"Content-Type": ["application/json"]}, query=args,
query_bytes=encode_query_args(args),
long_retries=long_retries, long_retries=long_retries,
timeout=timeout, timeout=timeout,
ignore_backoff=ignore_backoff, ignore_backoff=ignore_backoff,
@ -376,8 +394,8 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers) check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext(): with logcontext.PreserveLoggingContext():
body = yield readBody(response) body = yield treq.json_content(response)
defer.returnValue(json.loads(body)) defer.returnValue(body)
@defer.inlineCallbacks @defer.inlineCallbacks
def post_json(self, destination, path, data={}, long_retries=False, def post_json(self, destination, path, data={}, long_retries=False,
@ -410,20 +428,12 @@ class MatrixFederationHttpClient(object):
Fails with ``FederationDeniedError`` if this destination Fails with ``FederationDeniedError`` if this destination
is not on our federation whitelist is not on our federation whitelist
""" """
def body_callback(method, url_bytes, headers_dict):
self.sign_request(
destination, method, url_bytes, headers_dict, data
)
return _JsonProducer(data)
response = yield self._request( response = yield self._request(
destination, destination,
"POST", "POST",
path, path,
query_bytes=encode_query_args(args), query=args,
body_callback=body_callback, json=data,
headers_dict={"Content-Type": ["application/json"]},
long_retries=long_retries, long_retries=long_retries,
timeout=timeout, timeout=timeout,
ignore_backoff=ignore_backoff, ignore_backoff=ignore_backoff,
@ -434,9 +444,9 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers) check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext(): with logcontext.PreserveLoggingContext():
body = yield readBody(response) body = yield treq.json_content(response)
defer.returnValue(json.loads(body)) defer.returnValue(body)
@defer.inlineCallbacks @defer.inlineCallbacks
def get_json(self, destination, path, args=None, retry_on_dns_fail=True, def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
@ -471,16 +481,11 @@ class MatrixFederationHttpClient(object):
logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
def body_callback(method, url_bytes, headers_dict):
self.sign_request(destination, method, url_bytes, headers_dict)
return None
response = yield self._request( response = yield self._request(
destination, destination,
"GET", "GET",
path, path,
query_bytes=encode_query_args(args), query=args,
body_callback=body_callback,
retry_on_dns_fail=retry_on_dns_fail, retry_on_dns_fail=retry_on_dns_fail,
timeout=timeout, timeout=timeout,
ignore_backoff=ignore_backoff, ignore_backoff=ignore_backoff,
@ -491,9 +496,9 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers) check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext(): with logcontext.PreserveLoggingContext():
body = yield readBody(response) body = yield treq.json_content(response)
defer.returnValue(json.loads(body)) defer.returnValue(body)
@defer.inlineCallbacks @defer.inlineCallbacks
def delete_json(self, destination, path, long_retries=False, def delete_json(self, destination, path, long_retries=False,
@ -523,13 +528,11 @@ class MatrixFederationHttpClient(object):
Fails with ``FederationDeniedError`` if this destination Fails with ``FederationDeniedError`` if this destination
is not on our federation whitelist is not on our federation whitelist
""" """
response = yield self._request( response = yield self._request(
destination, destination,
"DELETE", "DELETE",
path, path,
query_bytes=encode_query_args(args), query=args,
headers_dict={"Content-Type": ["application/json"]},
long_retries=long_retries, long_retries=long_retries,
timeout=timeout, timeout=timeout,
ignore_backoff=ignore_backoff, ignore_backoff=ignore_backoff,
@ -540,9 +543,9 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers) check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext(): with logcontext.PreserveLoggingContext():
body = yield readBody(response) body = yield treq.json_content(response)
defer.returnValue(json.loads(body)) defer.returnValue(body)
@defer.inlineCallbacks @defer.inlineCallbacks
def get_file(self, destination, path, output_stream, args={}, def get_file(self, destination, path, output_stream, args={},
@ -569,26 +572,11 @@ class MatrixFederationHttpClient(object):
Fails with ``FederationDeniedError`` if this destination Fails with ``FederationDeniedError`` if this destination
is not on our federation whitelist is not on our federation whitelist
""" """
encoded_args = {}
for k, vs in args.items():
if isinstance(vs, string_types):
vs = [vs]
encoded_args[k] = [v.encode("UTF-8") for v in vs]
query_bytes = urllib.urlencode(encoded_args, True)
logger.debug("Query bytes: %s Retry DNS: %s", query_bytes, retry_on_dns_fail)
def body_callback(method, url_bytes, headers_dict):
self.sign_request(destination, method, url_bytes, headers_dict)
return None
response = yield self._request( response = yield self._request(
destination, destination,
"GET", "GET",
path, path,
query_bytes=query_bytes, query=args,
body_callback=body_callback,
retry_on_dns_fail=retry_on_dns_fail, retry_on_dns_fail=retry_on_dns_fail,
ignore_backoff=ignore_backoff, ignore_backoff=ignore_backoff,
) )
@ -639,30 +627,6 @@ def _readBodyToFile(response, stream, max_size):
return d return d
class _JsonProducer(object):
""" Used by the twisted http client to create the HTTP body from json
"""
def __init__(self, jsn):
self.reset(jsn)
def reset(self, jsn):
self.body = encode_canonical_json(jsn)
self.length = len(self.body)
def startProducing(self, consumer):
consumer.write(self.body)
return defer.succeed(None)
def pauseProducing(self):
pass
def stopProducing(self):
pass
def resumeProducing(self):
pass
def _flatten_response_never_received(e): def _flatten_response_never_received(e):
if hasattr(e, "reasons"): if hasattr(e, "reasons"):
reasons = ", ".join( reasons = ", ".join(
@ -693,7 +657,7 @@ def check_content_type_is_json(headers):
"No Content-Type header" "No Content-Type header"
) )
c_type = c_type[0] # only the first header c_type = c_type[0].decode('ascii') # only the first header
val, options = cgi.parse_header(c_type) val, options = cgi.parse_header(c_type)
if val != "application/json": if val != "application/json":
raise RuntimeError( raise RuntimeError(
@ -711,6 +675,6 @@ def encode_query_args(args):
vs = [vs] vs = [vs]
encoded_args[k] = [v.encode("UTF-8") for v in vs] encoded_args[k] = [v.encode("UTF-8") for v in vs]
query_bytes = urllib.urlencode(encoded_args, True) query_bytes = urllib.parse.urlencode(encoded_args, True)
return query_bytes return query_bytes.encode('utf8')

View file

@ -204,14 +204,14 @@ class SynapseRequest(Request):
self.start_time = time.time() self.start_time = time.time()
self.request_metrics = RequestMetrics() self.request_metrics = RequestMetrics()
self.request_metrics.start( self.request_metrics.start(
self.start_time, name=servlet_name, method=self.method, self.start_time, name=servlet_name, method=self.method.decode('ascii'),
) )
self.site.access_logger.info( self.site.access_logger.info(
"%s - %s - Received request: %s %s", "%s - %s - Received request: %s %s",
self.getClientIP(), self.getClientIP(),
self.site.site_tag, self.site.site_tag,
self.method, self.method.decode('ascii'),
self.get_redacted_uri() self.get_redacted_uri()
) )

View file

@ -40,6 +40,7 @@ REQUIREMENTS = {
"pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
"service_identity>=1.0.0": ["service_identity>=1.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"],
"Twisted>=17.1.0": ["twisted>=17.1.0"], "Twisted>=17.1.0": ["twisted>=17.1.0"],
"treq>=15.1": ["treq>=15.1"],
# We use crypto.get_elliptic_curve which is only supported in >=0.15 # We use crypto.get_elliptic_curve which is only supported in >=0.15
"pyopenssl>=0.15": ["OpenSSL>=0.15"], "pyopenssl>=0.15": ["OpenSSL>=0.15"],