# # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright 2014-2016 OpenMarket Ltd # Copyright (C) 2023 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # See the GNU Affero General Public License for more details: # <https://www.gnu.org/licenses/agpl-3.0.html>. # # Originally licensed under the Apache License, Version 2.0: # <http://www.apache.org/licenses/LICENSE-2.0>. # # [This file includes modifications made by New Vector Limited] # # import json import urllib from pprint import pformat from typing import Optional from twisted.internet import defer, reactor from twisted.web.client import Agent, readBody from twisted.web.http_headers import Headers class HttpClient: """Interface for talking json over http""" def put_json(self, url, data): """Sends the specifed json data using PUT Args: url (str): The URL to PUT data to. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. """ def get_json(self, url, args=None): """Gets some json from the given host homeserver and path Args: url (str): The URL to GET data from. args (dict): A dictionary used to create query strings, defaults to None. **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. """ class TwistedHttpClient(HttpClient): """Wrapper around the twisted HTTP client api. Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the requests. """ def __init__(self): self.agent = Agent(reactor) @defer.inlineCallbacks def put_json(self, url, data): response = yield self._create_put_request( url, data, headers_dict={"Content-Type": ["application/json"]} ) body = yield readBody(response) defer.returnValue((response.code, body)) @defer.inlineCallbacks def get_json(self, url, args=None): if args: # generates a list of strings of form "k=v". qs = urllib.urlencode(args, True) url = "%s?%s" % (url, qs) response = yield self._create_get_request(url) body = yield readBody(response) defer.returnValue(json.loads(body)) def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None): """Wrapper of _create_request to issue a PUT request""" headers_dict = headers_dict or {} if "Content-Type" not in headers_dict: raise defer.error(RuntimeError("Must include Content-Type header for PUTs")) return self._create_request( "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict ) def _create_get_request(self, url, headers_dict: Optional[dict] = None): """Wrapper of _create_request to issue a GET request""" return self._create_request("GET", url, headers_dict=headers_dict or {}) @defer.inlineCallbacks def do_request( self, method, url, data=None, qparams=None, jsonreq=True, headers: Optional[dict] = None, ): headers = headers or {} if qparams: url = "%s?%s" % (url, urllib.urlencode(qparams, True)) if jsonreq: prod = _JsonProducer(data) headers["Content-Type"] = ["application/json"] else: prod = _RawProducer(data) if method in ["POST", "PUT"]: response = yield self._create_request( method, url, producer=prod, headers_dict=headers ) else: response = yield self._create_request(method, url) body = yield readBody(response) defer.returnValue(json.loads(body)) @defer.inlineCallbacks def _create_request( self, method, url, producer=None, headers_dict: Optional[dict] = None ): """Creates and sends a request to the given url""" headers_dict = headers_dict or {} headers_dict["User-Agent"] = ["Synapse Cmd Client"] retries_left = 5 print("%s to %s with headers %s" % (method, url, headers_dict)) if self.verbose and producer: if "password" in producer.data: temp = producer.data["password"] producer.data["password"] = "[REDACTED]" print(json.dumps(producer.data, indent=4)) producer.data["password"] = temp else: print(json.dumps(producer.data, indent=4)) while True: try: response = yield self.agent.request( method, url.encode("UTF8"), Headers(headers_dict), producer ) break except Exception as e: print("uh oh: %s" % e) if retries_left: yield self.sleep(2 ** (5 - retries_left)) retries_left -= 1 else: raise e if self.verbose: print("Status %s %s" % (response.code, response.phrase)) print(pformat(list(response.headers.getAllRawHeaders()))) defer.returnValue(response) def sleep(self, seconds): d = defer.Deferred() reactor.callLater(seconds, d.callback, seconds) return d class _RawProducer: def __init__(self, data): self.data = data self.body = data self.length = len(self.body) def startProducing(self, consumer): consumer.write(self.body) return defer.succeed(None) def pauseProducing(self): pass def stopProducing(self): pass class _JsonProducer: """Used by the twisted http client to create the HTTP body from json""" def __init__(self, jsn): self.data = jsn self.body = json.dumps(jsn).encode("utf8") self.length = len(self.body) def startProducing(self, consumer): consumer.write(self.body) return defer.succeed(None) def pauseProducing(self): pass def stopProducing(self): pass