ircd::fs::aio: Preliminary io_submit queue to leverage disk controller optimizations.

include/ircd/fs/aio.h

@@ -49,6 +49,7 @@ struct ircd::fs::aio::stats
 {
 	uint32_t requests {0};             ///< count of requests created
 	uint32_t complete {0};             ///< count of requests completed
+	uint32_t submits {0};              ///< count of io_submit calls
 	uint32_t handles {0};              ///< count of event_fd callbacks
 	uint32_t events {0};               ///< count of events from io_getevents
 	uint32_t cancel {0};               ///< count of requests canceled
@@ -70,6 +71,8 @@ struct ircd::fs::aio::stats
 	uint32_t max_requests {0};         ///< maximum observed pending requests
 	uint32_t max_reads {0};            ///< maximum observed pending reads
 	uint32_t max_writes {0};           ///< maximum observed pending write
+
+	uint32_t maxed_submits {0};        ///< number of submits at threshold
 };
 
 struct ircd::fs::aio::init

ircd/aio.cc

@@ -237,18 +237,13 @@ try
 	assert(ctx::current);
 	assert(waiter == ctx::current);
 
-	struct iocb *const cbs[]
-	{
-		static_cast<iocb *>(this)
-	};
-
 	const size_t submitted_bytes
 	{
 		bytes(iovec())
 	};
 
 	// Submit to kernel
-	syscall<SYS_io_submit>(context->idp, 1, &cbs);
+	context->submit(*this);
 
 	// Update stats for submission phase
 	stats.bytes_requests += submitted_bytes;
@@ -261,6 +256,8 @@ try
 	while(retval == std::numeric_limits<ssize_t>::min())
 		ctx::wait();
 
+	assert(retval <= ssize_t(submitted_bytes));
+
 	// Update stats for completion phase.
 	stats.bytes_complete += submitted_bytes;
 	stats.complete++;
@@ -393,6 +390,54 @@ ircd::fs::aio::kernel::wait()
 	return true;
 }
 
+void
+ircd::fs::aio::kernel::submit(request &request)
+noexcept try
+{
+	static const size_t threshold(16);
+	thread_local std::array<iocb *, MAX_EVENTS> queue;
+	thread_local size_t count;
+	thread_local size_t sem[2];
+
+	assert(request.aio_data == uintptr_t(&request));
+	queue.at(count++) = static_cast<iocb *>(&request);
+
+	if(count >= threshold)
+	{
+		syscall<SYS_io_submit>(context->idp, count, queue.data());
+		++stats.maxed_submits;
+		++stats.submits;
+		count = 0;
+		return;
+	}
+
+	++sem[0];
+	ircd::post([]
+	{
+		if(sem[0] > ++sem[1])
+			return;
+
+		if(!count)
+			return;
+
+		syscall<SYS_io_submit>(context->idp, count, queue.data());
+		stats.maxed_submits += count >= threshold;
+		++stats.submits;
+		count = 0;
+	});
+}
+catch(const std::exception &e)
+{
+	log::critical
+	{
+		"AIO(%p) submit: %s",
+		this,
+		e.what()
+	};
+
+	throw;
+}
+
 void
 ircd::fs::aio::kernel::set_handle()
 {
@@ -495,7 +540,9 @@ noexcept try
 		*reinterpret_cast<aio::request *>(event.data)
 	};
 
-	assert(reinterpret_cast<iocb *>(event.obj) == static_cast<iocb *>(&request));
+	auto *const iocb(reinterpret_cast<struct ::iocb *>(event.obj));
+	assert(iocb == static_cast<struct ::iocb *>(&request));
+	assert(reinterpret_cast<aio::request *>(iocb->aio_data) == &request);
 	assert(event.res2 >= 0);
 	assert(event.res == -1 || event.res2 == 0);
 

ircd/aio.h

@@ -53,6 +53,9 @@ struct ircd::fs::aio::kernel
 	void handle(const boost::system::error_code &, const size_t) noexcept;
 	void set_handle();
 
+	void submit(request &) noexcept;
+
+	// Control panel
 	bool wait();
 	bool interrupt();
 

modules/console.cc

@@ -895,6 +895,14 @@ console_cmd__aio(opt &out, const string_view &line)
 	    << std::setw(9) << std::right << s.events
 	    << std::endl;
 
+	out << std::setw(12) << std::left << "submits"
+	    << std::setw(9) << std::right << s.submits
+	    << std::endl;
+
+	out << std::setw(12) << std::left << "submits max"
+	    << std::setw(9) << std::right << s.maxed_submits
+	    << std::endl;
+
 	return true;
 }