maxmustermann/godot
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #38944 from Wavesonics/http-gzip

Browse source 
HttpRequest now handles gzipping response bodies
This commit is contained in:
Fabio Alessandrelli 2020-09-07 17:03:19 +02:00 committed by GitHub
parent 8eb295cc54 6584db1538
commit 2cb6b2ac6f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 272 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
core/io/compression.cpp
View file

@ -180,8 +180,95 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
ERR_FAIL_V(-1);
}
/**
This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies.
This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
*/
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
int ret;
uint8_t *dst = nullptr;
int out_mark = 0;
z_stream strm;
ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
// This function only supports GZip and Deflate
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
// Initialize the stream
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
int err = inflateInit2(&strm, window_bits);
ERR_FAIL_COND_V(err != Z_OK, -1);
// Setup the stream inputs
strm.next_in = (Bytef *)p_src;
strm.avail_in = p_src_size;
// Ensure the destination buffer is empty
p_dst_vect->resize(0);
// decompress until deflate stream ends or end of file
do {
// Add another chunk size to the output buffer
// This forces a copy of the whole buffer
p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
// Get pointer to the actual output buffer
dst = p_dst_vect->ptrw();
// Set the stream to the new output stream
// Since it was copied, we need to reset the stream to the new buffer
strm.next_out = &(dst[out_mark]);
strm.avail_out = gzip_chunk;
// run inflate() on input until output buffer is full and needs to be resized
// or input runs out
do {
ret = inflate(&strm, Z_SYNC_FLUSH);
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR;
[[fallthrough]];
case Z_DATA_ERROR:
case Z_MEM_ERROR:
case Z_STREAM_ERROR:
WARN_PRINT(strm.msg);
(void)inflateEnd(&strm);
p_dst_vect->resize(0);
return ret;
}
} while (strm.avail_out > 0 && strm.avail_in > 0);
out_mark += gzip_chunk;
// Encorce max output size
if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
(void)inflateEnd(&strm);
p_dst_vect->resize(0);
return Z_BUF_ERROR;
}
} while (ret != Z_STREAM_END);
// If all done successfully, resize the output if it's larger than the actual output
if (ret == Z_STREAM_END && (unsigned long)p_dst_vect->size() > strm.total_out) {
p_dst_vect->resize(strm.total_out);
}
// clean up and return
(void)inflateEnd(&strm);
return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}
int Compression::zlib_level = Z_DEFAULT_COMPRESSION;
int Compression::gzip_level = Z_DEFAULT_COMPRESSION;
int Compression::zstd_level = 3;
bool Compression::zstd_long_distance_matching = false;
int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT
int Compression::gzip_chunk = 16384;

3
core/io/compression.h
View file

@ -32,6 +32,7 @@
#define COMPRESSION_H
#include "core/typedefs.h"
#include "core/vector.h"
class Compression {
public:
@ -40,6 +41,7 @@ public:
static int zstd_level;
static bool zstd_long_distance_matching;
static int zstd_window_log_size;
static int gzip_chunk;
enum Mode {
MODE_FASTLZ,
@ -51,6 +53,7 @@ public:
static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
Compression() {}
};

18
core/variant_call.cpp
View file

@ -712,6 +712,23 @@ struct _VariantCall {
r_ret = decompressed;
}
static void _call_PackedByteArray_decompress_dynamic(Variant &r_ret, Variant &p_self, const Variant **p_args) {
Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
PackedByteArray decompressed;
int max_output_size = (int)(*p_args[0]);
Compression::Mode mode = (Compression::Mode)(int)(*p_args[1]);
int result = Compression::decompress_dynamic(&decompressed, max_output_size, ba->array.ptr(), ba->array.size(), mode);
if (result == OK) {
r_ret = decompressed;
} else {
decompressed.clear();
r_ret = decompressed;
ERR_FAIL_MSG("Decompression failed.");
}
}
static void _call_PackedByteArray_hex_encode(Variant &r_ret, Variant &p_self, const Variant **p_args) {
Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
if (ba->array.size() == 0) {
@ -2177,6 +2194,7 @@ void register_variant_methods() {
ADDFUNC0R(PACKED_BYTE_ARRAY, STRING, PackedByteArray, hex_encode, varray());
ADDFUNC1R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, compress, INT, "compression_mode", varray(0));
ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress, INT, "buffer_size", INT, "compression_mode", varray(0));
ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress_dynamic, INT, "max_output_size", INT, "compression_mode", varray(0));
ADDFUNC0R(PACKED_INT32_ARRAY, INT, PackedInt32Array, size, varray());
ADDFUNC0R(PACKED_INT32_ARRAY, BOOL, PackedInt32Array, empty, varray());

31
doc/classes/HTTPRequest.xml
View file

@ -64,6 +64,11 @@
add_child(texture_rect)
texture_rect.texture = texture
[/codeblock]
[b]Gzipped response bodies[/b]
HttpRequest will automatically handle decompression of response bodies.
A "Accept-Encoding" header will be automatically added to each of your requests, unless one is already specified.
Any response with a "Content-Encoding: gzip" header will automatically be decompressed and delivered to you as a uncompressed bytes.
[b]Note:[/b] When performing HTTP requests from a project exported to HTML5, keep in mind the remote server may not allow requests from foreign origins due to [url=https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS]CORS[/url]. If you host the server in question, you should modify its backend to allow requests from foreign origins by adding the [code]Access-Control-Allow-Origin: *[/code] HTTP header.
</description>
<tutorials>
@ -119,10 +124,28 @@
[b]Note:[/b] The [code]request_data[/code] parameter is ignored if [code]method[/code] is [constant HTTPClient.METHOD_GET]. This is because GET methods can't contain request data. As a workaround, you can pass request data as a query string in the URL. See [method String.http_escape] for an example.
</description>
</method>
<method name="request_raw">
<return type="int" enum="Error">
</return>
<argument index="0" name="url" type="String">
</argument>
<argument index="1" name="custom_headers" type="PackedStringArray" default="PackedStringArray( )">
</argument>
<argument index="2" name="ssl_validate_domain" type="bool" default="true">
</argument>
<argument index="3" name="method" type="int" enum="HTTPClient.Method" default="0">
</argument>
<argument index="4" name="request_data_raw" type="PackedByteArray" default="PackedByteArray()">
</argument>
<description>
Creates request on the underlying [HTTPClient] using a raw array of bytes for the request body. If there is no configuration errors, it tries to connect using [method HTTPClient.connect_to_host] and passes parameters onto [method HTTPClient.request].
Returns [constant OK] if request is successfully created. (Does not imply that the server has responded), [constant ERR_UNCONFIGURED] if not in the tree, [constant ERR_BUSY] if still processing previous request, [constant ERR_INVALID_PARAMETER] if given string is not a valid URL format, or [constant ERR_CANT_CONNECT] if not using thread and the [HTTPClient] cannot connect to host.
</description>
</method>
</methods>
<members>
<member name="body_size_limit" type="int" setter="set_body_size_limit" getter="get_body_size_limit" default="-1">
Maximum allowed size for response bodies.
Maximum allowed size for response bodies. If the response body is compressed, this will be used as the maximum allowed size for the decompressed body.
</member>
<member name="download_chunk_size" type="int" setter="set_download_chunk_size" getter="get_download_chunk_size" default="4096">
The size of the buffer used and maximum bytes to read per iteration. See [member HTTPClient.read_chunk_size].
@ -139,6 +162,12 @@
<member name="use_threads" type="bool" setter="set_use_threads" getter="is_using_threads" default="false">
If [code]true[/code], multithreading is used to improve performance.
</member>
<member name="accept_gzip" type="bool" setter="set_accept_gzip" getter="is_accepting_gzip" default="true">
If [code]true[/code], this header will be added to each request: [code]Accept-Encoding: gzip, deflate[/code] telling servers that it's okay to compress response bodies.
Any Reponse body declaring a [code]Content-Encoding[/code] of either [code]gzip[/code] or [code]deflate[/code] will then be automatically decompressed, and the uncompressed bytes will be delivered via [code]request_completed[/code].
If the user has specified their own [code]Accept-Encoding[/code] header, then no header will be added regaurdless of [code]accept_gzip[/code].
If [code]false[/code] no header will be added, and no decompression will be performed on response bodies. The raw bytes of the response body will be returned via [code]request_completed[/code].
</member>
</members>
<signals>
<signal name="request_completed">

14
doc/classes/PackedByteArray.xml
View file

@ -57,6 +57,20 @@
Returns a new [PackedByteArray] with the data decompressed. Set [code]buffer_size[/code] to the size of the uncompressed data. Set the compression mode using one of [enum File.CompressionMode]'s constants.
</description>
</method>
<method name="decompress_dynamic">
<return type="PackedByteArray">
</return>
<argument index="0" name="max_output_size" type="int" default="0">
</argument>
<argument index="1" name="compression_mode" type="int" default="0">
</argument>
<description>
Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum File.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b]
This method is potentially slower than [code]decompress[/code], as it may have to re-allocate it's output buffer multiple times while decompressing, where as [code]decompress[/code] knows it's output buffer size from the begining.
GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [code]max_output_size[/code]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that ammount in bytes, then an error will be returned.
</description>
</method>
<method name="empty">
<return type="bool">
</return>

113
scene/main/http_request.cpp
View file

@ -29,6 +29,8 @@
/*************************************************************************/
#include "http_request.h"
#include "core/io/compression.h"
#include "core/ustring.h"
void HTTPRequest::_redirect_request(const String &p_new_url) {
}
@ -82,7 +84,51 @@ Error HTTPRequest::_parse_url(const String &p_url) {
return OK;
}
bool HTTPRequest::has_header(const PackedStringArray &p_headers, const String &p_header_name) {
bool exists = false;
String lower_case_header_name = p_header_name.to_lower();
for (int i = 0; i < p_headers.size() && !exists; i++) {
String sanitized = p_headers[i].strip_edges().to_lower();
if (sanitized.begins_with(lower_case_header_name)) {
exists = true;
}
}
return exists;
}
String HTTPRequest::get_header_value(const PackedStringArray &p_headers, const String &p_header_name) {
String value = "";
String lowwer_case_header_name = p_header_name.to_lower();
for (int i = 0; i < p_headers.size(); i++) {
if (p_headers[i].find(":", 0) >= 0) {
Vector<String> parts = p_headers[i].split(":", false, 1);
if (parts[0].strip_edges().to_lower() == lowwer_case_header_name) {
value = parts[1].strip_edges();
break;
}
}
}
return value;
}
Error HTTPRequest::request(const String &p_url, const Vector<String> &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const String &p_request_data) {
// Copy the string into a raw buffer
Vector<uint8_t> raw_data;
CharString charstr = p_request_data.utf8();
size_t len = charstr.length();
raw_data.resize(len);
uint8_t *w = raw_data.ptrw();
copymem(w, charstr.ptr(), len);
return request_raw(p_url, p_custom_headers, p_ssl_validate_domain, p_method, raw_data);
}
Error HTTPRequest::request_raw(const String &p_url, const Vector<String> &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const Vector<uint8_t> &p_request_data_raw) {
ERR_FAIL_COND_V(!is_inside_tree(), ERR_UNCONFIGURED);
ERR_FAIL_COND_V_MSG(requesting, ERR_BUSY, "HTTPRequest is processing a request. Wait for completion or cancel it before attempting a new one.");
@ -102,7 +148,14 @@ Error HTTPRequest::request(const String &p_url, const Vector<String> &p_custom_h
headers = p_custom_headers;
request_data = p_request_data;
if (accept_gzip) {
// If the user has specified a different Accept-Encoding, don't overwrite it
if (!has_header(headers, "Accept-Encoding")) {
headers.push_back("Accept-Encoding: gzip, deflate");
}
}
request_data = p_request_data_raw;
requesting = true;
@ -288,7 +341,7 @@ bool HTTPRequest::_update_connection() {
} else {
// Did not request yet, do request
Error err = client->request(method, request_string, headers, request_data);
Error err = client->request_raw(method, request_string, headers, request_data);
if (err != OK) {
call_deferred("_request_done", RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
return true;
@ -382,9 +435,47 @@ bool HTTPRequest::_update_connection() {
ERR_FAIL_V(false);
}
void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &headers, const PackedByteArray &p_data) {
void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) {
cancel_request();
emit_signal("request_completed", p_status, p_code, headers, p_data);
// Determine if the request body is compressed
bool is_compressed;
String content_encoding = get_header_value(p_headers, "Content-Encoding").to_lower();
Compression::Mode mode;
if (content_encoding == "gzip") {
mode = Compression::Mode::MODE_GZIP;
is_compressed = true;
} else if (content_encoding == "deflate") {
mode = Compression::Mode::MODE_DEFLATE;
is_compressed = true;
} else {
is_compressed = false;
}
const PackedByteArray *data = NULL;
if (accept_gzip && is_compressed && p_data.size() > 0) {
// Decompress request body
PackedByteArray *decompressed = memnew(PackedByteArray);
int result = Compression::decompress_dynamic(decompressed, body_size_limit, p_data.ptr(), p_data.size(), mode);
if (result == OK) {
data = decompressed;
} else if (result == -5) {
WARN_PRINT("Decompressed size of HTTP response body exceeded body_size_limit");
p_status = RESULT_BODY_SIZE_LIMIT_EXCEEDED;
// Just return the raw data if we failed to decompress it
data = &p_data;
} else {
WARN_PRINT("Failed to decompress HTTP response body");
p_status = RESULT_BODY_DECOMPRESS_FAILED;
// Just return the raw data if we failed to decompress it
data = &p_data;
}
} else {
data = &p_data;
}
emit_signal("request_completed", p_status, p_code, p_headers, *data);
}
void HTTPRequest::_notification(int p_what) {
@ -415,6 +506,14 @@ bool HTTPRequest::is_using_threads() const {
return use_threads;
}
void HTTPRequest::set_accept_gzip(bool p_gzip) {
accept_gzip = p_gzip;
}
bool HTTPRequest::is_accepting_gzip() const {
return accept_gzip;
}
void HTTPRequest::set_body_size_limit(int p_bytes) {
ERR_FAIL_COND(get_http_client_status() != HTTPClient::STATUS_DISCONNECTED);
@ -481,6 +580,7 @@ void HTTPRequest::_timeout() {
void HTTPRequest::_bind_methods() {
ClassDB::bind_method(D_METHOD("request", "url", "custom_headers", "ssl_validate_domain", "method", "request_data"), &HTTPRequest::request, DEFVAL(PackedStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(String()));
ClassDB::bind_method(D_METHOD("request_raw", "url", "custom_headers", "ssl_validate_domain", "method", "request_data_raw"), &HTTPRequest::request_raw, DEFVAL(PackedStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(PackedByteArray()));
ClassDB::bind_method(D_METHOD("cancel_request"), &HTTPRequest::cancel_request);
ClassDB::bind_method(D_METHOD("get_http_client_status"), &HTTPRequest::get_http_client_status);
@ -488,6 +588,9 @@ void HTTPRequest::_bind_methods() {
ClassDB::bind_method(D_METHOD("set_use_threads", "enable"), &HTTPRequest::set_use_threads);
ClassDB::bind_method(D_METHOD("is_using_threads"), &HTTPRequest::is_using_threads);
ClassDB::bind_method(D_METHOD("set_accept_gzip", "enable"), &HTTPRequest::set_accept_gzip);
ClassDB::bind_method(D_METHOD("is_accepting_gzip"), &HTTPRequest::is_accepting_gzip);
ClassDB::bind_method(D_METHOD("set_body_size_limit", "bytes"), &HTTPRequest::set_body_size_limit);
ClassDB::bind_method(D_METHOD("get_body_size_limit"), &HTTPRequest::get_body_size_limit);
@ -512,6 +615,7 @@ void HTTPRequest::_bind_methods() {
ADD_PROPERTY(PropertyInfo(Variant::STRING, "download_file", PROPERTY_HINT_FILE), "set_download_file", "get_download_file");
ADD_PROPERTY(PropertyInfo(Variant::INT, "download_chunk_size", PROPERTY_HINT_RANGE, "256,16777216"), "set_download_chunk_size", "get_download_chunk_size");
ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_threads"), "set_use_threads", "is_using_threads");
ADD_PROPERTY(PropertyInfo(Variant::BOOL, "accept_gzip"), "set_accept_gzip", "is_accepting_gzip");
ADD_PROPERTY(PropertyInfo(Variant::INT, "body_size_limit", PROPERTY_HINT_RANGE, "-1,2000000000"), "set_body_size_limit", "get_body_size_limit");
ADD_PROPERTY(PropertyInfo(Variant::INT, "max_redirects", PROPERTY_HINT_RANGE, "-1,64"), "set_max_redirects", "get_max_redirects");
ADD_PROPERTY(PropertyInfo(Variant::INT, "timeout", PROPERTY_HINT_RANGE, "0,86400"), "set_timeout", "get_timeout");
@ -544,6 +648,7 @@ HTTPRequest::HTTPRequest() {
got_response = false;
validate_ssl = false;
use_ssl = false;
accept_gzip = true;
response_code = 0;
request_sent = false;
requesting = false;

13
scene/main/http_request.h
View file

@ -50,6 +50,7 @@ public:
RESULT_SSL_HANDSHAKE_ERROR,
RESULT_NO_RESPONSE,
RESULT_BODY_SIZE_LIMIT_EXCEEDED,
RESULT_BODY_DECOMPRESS_FAILED,
RESULT_REQUEST_FAILED,
RESULT_DOWNLOAD_FILE_CANT_OPEN,
RESULT_DOWNLOAD_FILE_WRITE_ERROR,
@ -68,12 +69,13 @@ private:
bool validate_ssl;
bool use_ssl;
HTTPClient::Method method;
String request_data;
Vector<uint8_t> request_data;
bool request_sent;
Ref<HTTPClient> client;
PackedByteArray body;
volatile bool use_threads;
bool accept_gzip;
bool got_response;
int response_code;
@ -102,12 +104,15 @@ private:
Error _parse_url(const String &p_url);
Error _request();
bool has_header(const PackedStringArray &p_headers, const String &p_header_name);
String get_header_value(const PackedStringArray &p_headers, const String &header_name);
volatile bool thread_done;
volatile bool thread_request_quit;
Thread *thread;
void _request_done(int p_status, int p_code, const PackedStringArray &headers, const PackedByteArray &p_data);
void _request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data);
static void _thread_func(void *p_userdata);
protected:
@ -116,12 +121,16 @@ protected:
public:
Error request(const String &p_url, const Vector<String> &p_custom_headers = Vector<String>(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const String &p_request_data = ""); //connects to a full url and perform request
Error request_raw(const String &p_url, const Vector<String> &p_custom_headers = Vector<String>(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const Vector<uint8_t> &p_request_data_raw = Vector<uint8_t>()); //connects to a full url and perform request
void cancel_request();
HTTPClient::Status get_http_client_status() const;
void set_use_threads(bool p_use);
bool is_using_threads() const;
void set_accept_gzip(bool p_gzip);
bool is_accepting_gzip() const;
void set_download_file(const String &p_file);
String get_download_file() const;