ircd::ios: ABI simplify; fix double-indirection, AGU load, code size on context switch.

include/ircd/ios/descriptor.h

@@ -29,14 +29,14 @@ struct ircd::ios::descriptor
 
 	static uint64_t ids;
 
-	static void *default_allocator(handler &, const size_t &);
+	static void *default_allocator(handler &, const size_t);
-	static void default_deallocator(handler &, void *const &, const size_t &) noexcept;
+	static void default_deallocator(handler &, void *, const size_t) noexcept;
 
 	string_view name;
 	uint64_t id {++ids};
 	std::unique_ptr<struct stats> stats;
-	void *(*allocator)(handler &, const size_t &);
+	void *(*allocator)(handler &, const size_t);
-	void (*deallocator)(handler &, void *const &, const size_t &);
+	void (*deallocator)(handler &, void *, const size_t);
 	std::vector<std::array<uint64_t, 2>> history; // epoch, cycles
 	uint8_t history_pos {0};
 	bool continuation {false};
@@ -72,10 +72,10 @@ struct ircd::ios::descriptor::stats
 	item alloc_bytes;
 	item frees;
 	item free_bytes;
-	item slice_total;
 	item slice_last;
-	item latency_total;
+	item slice_total;
 	item latency_last;
+	item latency_total;
 
 	stats(descriptor &);
 	stats() = delete;
@@ -87,8 +87,8 @@ struct ircd::ios::descriptor::stats
 [[gnu::hot]]
 inline void
 ircd::ios::descriptor::default_deallocator(handler &handler,
-                                           void *const &ptr,
+                                           void *const ptr,
-                                           const size_t &size)
+                                           const size_t size)
 noexcept
 {
 	#ifdef __clang__
@@ -101,7 +101,7 @@ noexcept
 [[gnu::hot]]
 inline void *
 ircd::ios::descriptor::default_allocator(handler &handler,
-                                         const size_t &size)
+                                         const size_t size)
 {
 	return ::operator new(size);
 }

include/ircd/ios/handler.h

@@ -28,13 +28,13 @@ struct ircd::ios::handler
 	static thread_local handler *current;
 	static thread_local uint64_t epoch;
 
-	static void enqueue(handler *const &) noexcept;
+	static void enqueue(handler *) noexcept;
-	static void *allocate(handler *const &, const size_t &);
+	static void *allocate(handler *, const size_t);
-	static void deallocate(handler *const &, void *const &, const size_t &) noexcept;
+	static void deallocate(handler *, void *, const size_t) noexcept;
-	static bool continuation(handler *const &) noexcept;
+	static bool continuation(handler *) noexcept;
-	static void enter(handler *const &) noexcept;
+	static void enter(handler *) noexcept;
-	static void leave(handler *const &) noexcept;
+	static void leave(handler *) noexcept;
-	static bool fault(handler *const &) noexcept;
+	static bool fault(handler *) noexcept;
 
 	ios::descriptor *descriptor {nullptr};
 	uint64_t ts {0}; // last tsc sample; for profiling each phase
@@ -91,9 +91,9 @@ const
 
 [[gnu::hot]]
 inline void
-ircd::ios::handler::deallocate(handler *const &handler,
+ircd::ios::handler::deallocate(handler *const handler,
-                               void *const &ptr,
+                               void *const ptr,
-                               const size_t &size)
+                               const size_t size)
 noexcept
 {
 	assert(handler && handler->descriptor);
@@ -110,8 +110,8 @@ noexcept
 
 [[gnu::hot]]
 inline void *
-ircd::ios::handler::allocate(handler *const &handler,
+ircd::ios::handler::allocate(handler *const handler,
-                             const size_t &size)
+                             const size_t size)
 {
 	assert(handler && handler->descriptor);
 	auto &descriptor(*handler->descriptor);
@@ -127,7 +127,7 @@ ircd::ios::handler::allocate(handler *const &handler,
 
 [[gnu::hot]]
 inline void
-ircd::ios::handler::enqueue(handler *const &handler)
+ircd::ios::handler::enqueue(handler *const handler)
 noexcept
 {
 	assert(handler && handler->descriptor);
@@ -152,7 +152,7 @@ noexcept
 
 [[gnu::hot]]
 inline bool
-ircd::ios::handler::continuation(handler *const &handler)
+ircd::ios::handler::continuation(handler *const handler)
 noexcept
 {
 	assert(handler && handler->descriptor);

include/ircd/ios/ios.h

@@ -30,6 +30,7 @@ namespace ircd::ios
 	extern asio::executor user, main;
 	extern std::thread::id main_thread_id;
 	extern thread_local bool is_main_thread;
+	extern bool user_available, main_available;
 
 	bool available() noexcept;
 	const uint64_t &epoch() noexcept;
@@ -64,3 +65,11 @@ noexcept
 {
 	return handler::epoch;
 }
+
+inline bool
+__attribute__((always_inline))
+ircd::ios::available()
+noexcept
+{
+	return main_available;
+}

ircd/fs_aio.cc

@@ -689,7 +689,7 @@ ircd::fs::aio::system::handle_descriptor
 	// appears to excessively allocate and deallocate 120 bytes; this
 	// is a simple asynchronous operation, we can do better (and perhaps
 	// even better than this below).
-	[](ios::handler &handler, const size_t &size) -> void *
+	[](ios::handler &handler, const size_t size) -> void *
 	{
 		assert(ircd::fs::aio::system);
 		auto &system(*ircd::fs::aio::system);
@@ -705,7 +705,7 @@ ircd::fs::aio::system::handle_descriptor
 	},
 
 	// no deallocation; satisfied by class member unique_ptr
-	[](ios::handler &handler, void *const &ptr, const size_t &size) {},
+	[](ios::handler &handler, void *const ptr, const size_t size) {},
 
 	// continuation
 	true,

ircd/ios.cc

@@ -39,6 +39,14 @@ ircd::ios::primary;
 decltype(ircd::ios::main)
 ircd::ios::main;
 
+/// Indicates the user asio::executor is initialized.
+decltype(ircd::ios::user_available)
+ircd::ios::user_available;
+
+/// Indicates the main asio::executor is initialized.
+decltype(ircd::ios::main_available)
+ircd::ios::main_available;
+
 decltype(ircd::boost_version_api)
 ircd::boost_version_api
 {
@@ -71,12 +79,14 @@ ircd::ios::init(asio::executor &&user)
 
 	// Save a reference handle to the user's executor.
 	ios::user = user;
+	ios::user_available = bool(ios::user);
 
 	// Create our strand instance.
 	ios::primary.emplace(static_cast<asio::io_context &>(user.context()));
 
 	// Set the reference handle to our executor.
 	ios::main = *ios::primary;
+	ios::main_available = bool(ios::main);
 }
 
 [[using gnu: cold]]
@@ -180,13 +190,6 @@ ircd::ios::forked_parent()
 	#endif
 }
 
-bool
-ircd::ios::available()
-noexcept
-{
-	return bool(main);
-}
-
 //
 // emption
 //
@@ -480,13 +483,6 @@ ircd::ios::descriptor::stats::stats(descriptor &d)
 		{ "name", stats_name(d, "free_bytes") },
 	},
 }
-,slice_total
-{
-	value + items++,
-	{
-		{ "name", stats_name(d, "slice_total") },
-	},
-}
 ,slice_last
 {
 	value + items++,
@@ -494,11 +490,11 @@ ircd::ios::descriptor::stats::stats(descriptor &d)
 		{ "name", stats_name(d, "slice_last") },
 	},
 }
-,latency_total
+,slice_total
 {
 	value + items++,
 	{
-		{ "name", stats_name(d, "latency_total") },
+		{ "name", stats_name(d, "slice_total") },
 	},
 }
 ,latency_last
@@ -508,6 +504,13 @@ ircd::ios::descriptor::stats::stats(descriptor &d)
 		{ "name", stats_name(d, "latency_last") },
 	},
 }
+,latency_total
+{
+	value + items++,
+	{
+		{ "name", stats_name(d, "latency_total") },
+	},
+}
 {
 	assert(items <= (sizeof(value) / sizeof(value[0])));
 }
@@ -531,7 +534,7 @@ ircd::ios::handler::epoch;
 
 [[gnu::cold]]
 bool
-ircd::ios::handler::fault(handler *const &handler)
+ircd::ios::handler::fault(handler *const handler)
 noexcept
 {
 	assert(handler && handler->descriptor);
@@ -568,7 +571,7 @@ noexcept
 
 [[gnu::hot]]
 void
-ircd::ios::handler::leave(handler *const &handler)
+ircd::ios::handler::leave(handler *const handler)
 noexcept
 {
 	assert(handler && handler->descriptor);
@@ -619,19 +622,14 @@ noexcept
 
 [[gnu::hot]]
 void
-ircd::ios::handler::enter(handler *const &handler)
+ircd::ios::handler::enter(handler *const handler)
 noexcept
 {
-	assert(!handler::current);
-	handler::current = handler;
-	++handler::epoch;
-
 	assert(handler && handler->descriptor);
 	auto &descriptor(*handler->descriptor);
 
 	assert(descriptor.stats);
 	auto &stats(*descriptor.stats);
-	++stats.calls;
 
 	const auto last_ts
 	{
@@ -640,6 +638,11 @@ noexcept
 
 	stats.latency_last = handler->ts - last_ts;
 	stats.latency_total += stats.latency_last;
+	++stats.calls;
+
+	assert(!handler::current);
+	handler::current = handler;
+	++handler::epoch;
 
 	if constexpr(profile::logging)
 		log::logf
@@ -696,7 +699,6 @@ ircd::ios::dispatch::dispatch(descriptor &descriptor,
 	latch.wait();
 }
 
-[[gnu::hot]]
 ircd::ios::dispatch::dispatch(descriptor &descriptor,
                               defer_t,
                               std::function<void ()> function)
@@ -725,7 +727,6 @@ ircd::ios::dispatch::dispatch(descriptor &descriptor,
 	};
 }
 
-[[gnu::hot]]
 ircd::ios::dispatch::dispatch(descriptor &descriptor,
                               std::function<void ()> function)
 {