ircd::server: Checkpoint preliminary chunk vectoring; state, options.

include/ircd/server/request.h

@@ -19,6 +19,7 @@ namespace ircd::server
 
 	size_t size(const in &);
 	size_t size(const out &);
+	size_t size_chunks(const in &);
 
 	void submit(const hostport &, request &);
 	bool cancel(request &);
@@ -66,6 +67,14 @@ struct ircd::server::in
 	/// constructed mutable_buffer). The allocated buffer will eventually be
 	/// placed here; any existing buffer will be discarded.
 	unique_buffer<mutable_buffer> dynamic;
+
+	/// Dynamic can also be used when receiving a chunked encoded message where
+	/// the length is not initially known. In that case, we create a buffer for
+	/// each chunk and append it to this vector. When the message is finished,
+	/// a final contiguous buffer is created in dynamic and the message is
+	/// copied there; this vector is cleared and content points there instead.
+	/// An option can be set in request::opts to skip the last step.
+	std::vector<unique_buffer<mutable_buffer>> chunks;
 };
 
 /// This is a handle for being a client to another server. This handle will
@@ -119,6 +128,19 @@ struct ircd::server::request::opts
 	/// content-length value. If the remote sends more content, the behavior
 	/// is the same as if specifying an in.content buffer of this size.
 	size_t content_length_maxalloc {256_MiB};
+
+	/// Only applies when using dynamic content allocation when the message is
+	/// received with chunked encoding. By default, chunks are saved in
+	/// individual buffers and copied to a final contiguous buffer. To skip the
+	/// contiguous allocation + copy and maintain the individual buffers,
+	/// set this option to false.
+	bool contiguous_content {true};
+
+	/// Only applies when using dynamic content allocation with a chunked
+	/// encoded response. This will hint the chunk vector. Ideally it can be
+	/// set to the number of chunks expected in a response to avoid growth of
+	/// that vector ... if you somehow know what that is going to be.
+	uint16_t chunks_reserve {4};
 };
 
 inline
@@ -181,6 +203,16 @@ noexcept
 	assert(!tag);
 }
 
+inline size_t
+ircd::server::size_chunks(const in &in)
+{
+	return std::accumulate(begin(in.chunks), end(in.chunks), size_t(0), []
+	(auto ret, const auto &buffer)
+	{
+		return ret += size(buffer);
+	});
+}
+
 inline size_t
 ircd::server::size(const in &in)
 {

include/ircd/server/tag.h

@@ -30,8 +30,9 @@ struct ircd::server::tag
 	{
 		size_t written {0};
 		size_t head_read {0};          // includes head terminator
-		size_t content_read {0};
+		size_t content_read {0};       // total content read after head
 		size_t content_length {0};     // fixed; or grows monotonic for chunked enc
+		size_t chunk_read {0};         // content read after last chunk head
 		size_t chunk_length {0};       // -1 for chunk header mode
 		http::code status {(http::code)0};
 	}

ircd/server.cc

@@ -2250,7 +2250,25 @@ ircd::server::tag::read_head(const const_buffer &buffer,
 	// will return anything beyond this message as overrun and indicate done.
 	if(head.transfer_encoding == "chunked")
 	{
-		assert(!dynamic);
+		if(dynamic)
+		{
+			assert(req.opt);
+			req.in.chunks.reserve(req.opt->chunks_reserve);
+		}
+
+
+	///TODO: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+	if(dynamic)
+	{
+		assert(req.opt);
+		const size_t alloc_size
+		{
+			40_MiB
+		};
+
+		req.in.dynamic = unique_buffer<mutable_buffer>{alloc_size};
+		req.in.content = req.in.dynamic;
+	}
 
 		const const_buffer chunk
 		{
@@ -2440,7 +2458,7 @@ ircd::server::tag::read_chunk_head(const const_buffer &buffer,
 	state.chunk_length = chunk.size + size(terminator);
 
 	// Now we check how much chunk was received beyond the head
-	const size_t &chunk_read
+	const auto &chunk_read
 	{
 		std::min(state.chunk_length, beyond_head_length)
 	};
@@ -2506,16 +2524,29 @@ ircd::server::tag::read_chunk_content(const const_buffer &buffer,
 		std::min(size(buffer), remaining)
 	};
 
+	// Increment the read counters for this chunk and all chunks.
+	state.chunk_read += addl_content_read;
 	state.content_read += addl_content_read;
+	assert(state.chunk_read <= state.content_read);
+
 	if(state.content_read == state.content_length)
 	{
+		// This branch is taken at the completion of a chunk. The size
+		// all the buffers is rolled back to hide the terminator so it's
+		// either ignored or overwritten so it doesn't leak to the user.
 		static const string_view terminator{"\r\n"};
 		assert(state.content_length >= size(terminator));
 		state.content_length -= size(terminator);
 		state.content_read -= size(terminator);
 
-		if(state.chunk_length == 2)
+		assert(state.chunk_length >= 2);
+		assert(state.chunk_read == state.chunk_length);
+		state.chunk_length -= size(terminator);
+		state.chunk_read -= size(terminator);
+
+		if(state.chunk_length == 0)
 		{
+			assert(state.chunk_read == 0);
 			assert(!done);
 			done = true;
 			req.in.content = mutable_buffer{data(req.in.content), state.content_length};
@@ -2529,7 +2560,12 @@ ircd::server::tag::read_chunk_content(const const_buffer &buffer,
 		req.in.progress(buffer, const_buffer{data(content), state.content_read});
 
 	if(state.content_read == state.content_length)
+	{
+		assert(state.chunk_read == state.chunk_length);
+		assert(state.chunk_read <= state.content_read);
 		state.chunk_length = size_t(-1);
+		state.chunk_read = 0;
+	}
 
 	return {};
 }