ircd::ios: ABI simplify; fix double-indirection, AGU load, code size on context switch.
This commit is contained in:
parent
82482278fb
commit
14f55f6110
|
@ -29,14 +29,14 @@ struct ircd::ios::descriptor
|
|||
|
||||
static uint64_t ids;
|
||||
|
||||
static void *default_allocator(handler &, const size_t &);
|
||||
static void default_deallocator(handler &, void *const &, const size_t &) noexcept;
|
||||
static void *default_allocator(handler &, const size_t);
|
||||
static void default_deallocator(handler &, void *, const size_t) noexcept;
|
||||
|
||||
string_view name;
|
||||
uint64_t id {++ids};
|
||||
std::unique_ptr<struct stats> stats;
|
||||
void *(*allocator)(handler &, const size_t &);
|
||||
void (*deallocator)(handler &, void *const &, const size_t &);
|
||||
void *(*allocator)(handler &, const size_t);
|
||||
void (*deallocator)(handler &, void *, const size_t);
|
||||
std::vector<std::array<uint64_t, 2>> history; // epoch, cycles
|
||||
uint8_t history_pos {0};
|
||||
bool continuation {false};
|
||||
|
@ -72,10 +72,10 @@ struct ircd::ios::descriptor::stats
|
|||
item alloc_bytes;
|
||||
item frees;
|
||||
item free_bytes;
|
||||
item slice_total;
|
||||
item slice_last;
|
||||
item latency_total;
|
||||
item slice_total;
|
||||
item latency_last;
|
||||
item latency_total;
|
||||
|
||||
stats(descriptor &);
|
||||
stats() = delete;
|
||||
|
@ -87,8 +87,8 @@ struct ircd::ios::descriptor::stats
|
|||
[[gnu::hot]]
|
||||
inline void
|
||||
ircd::ios::descriptor::default_deallocator(handler &handler,
|
||||
void *const &ptr,
|
||||
const size_t &size)
|
||||
void *const ptr,
|
||||
const size_t size)
|
||||
noexcept
|
||||
{
|
||||
#ifdef __clang__
|
||||
|
@ -101,7 +101,7 @@ noexcept
|
|||
[[gnu::hot]]
|
||||
inline void *
|
||||
ircd::ios::descriptor::default_allocator(handler &handler,
|
||||
const size_t &size)
|
||||
const size_t size)
|
||||
{
|
||||
return ::operator new(size);
|
||||
}
|
||||
|
|
|
@ -28,13 +28,13 @@ struct ircd::ios::handler
|
|||
static thread_local handler *current;
|
||||
static thread_local uint64_t epoch;
|
||||
|
||||
static void enqueue(handler *const &) noexcept;
|
||||
static void *allocate(handler *const &, const size_t &);
|
||||
static void deallocate(handler *const &, void *const &, const size_t &) noexcept;
|
||||
static bool continuation(handler *const &) noexcept;
|
||||
static void enter(handler *const &) noexcept;
|
||||
static void leave(handler *const &) noexcept;
|
||||
static bool fault(handler *const &) noexcept;
|
||||
static void enqueue(handler *) noexcept;
|
||||
static void *allocate(handler *, const size_t);
|
||||
static void deallocate(handler *, void *, const size_t) noexcept;
|
||||
static bool continuation(handler *) noexcept;
|
||||
static void enter(handler *) noexcept;
|
||||
static void leave(handler *) noexcept;
|
||||
static bool fault(handler *) noexcept;
|
||||
|
||||
ios::descriptor *descriptor {nullptr};
|
||||
uint64_t ts {0}; // last tsc sample; for profiling each phase
|
||||
|
@ -91,9 +91,9 @@ const
|
|||
|
||||
[[gnu::hot]]
|
||||
inline void
|
||||
ircd::ios::handler::deallocate(handler *const &handler,
|
||||
void *const &ptr,
|
||||
const size_t &size)
|
||||
ircd::ios::handler::deallocate(handler *const handler,
|
||||
void *const ptr,
|
||||
const size_t size)
|
||||
noexcept
|
||||
{
|
||||
assert(handler && handler->descriptor);
|
||||
|
@ -110,8 +110,8 @@ noexcept
|
|||
|
||||
[[gnu::hot]]
|
||||
inline void *
|
||||
ircd::ios::handler::allocate(handler *const &handler,
|
||||
const size_t &size)
|
||||
ircd::ios::handler::allocate(handler *const handler,
|
||||
const size_t size)
|
||||
{
|
||||
assert(handler && handler->descriptor);
|
||||
auto &descriptor(*handler->descriptor);
|
||||
|
@ -127,7 +127,7 @@ ircd::ios::handler::allocate(handler *const &handler,
|
|||
|
||||
[[gnu::hot]]
|
||||
inline void
|
||||
ircd::ios::handler::enqueue(handler *const &handler)
|
||||
ircd::ios::handler::enqueue(handler *const handler)
|
||||
noexcept
|
||||
{
|
||||
assert(handler && handler->descriptor);
|
||||
|
@ -152,7 +152,7 @@ noexcept
|
|||
|
||||
[[gnu::hot]]
|
||||
inline bool
|
||||
ircd::ios::handler::continuation(handler *const &handler)
|
||||
ircd::ios::handler::continuation(handler *const handler)
|
||||
noexcept
|
||||
{
|
||||
assert(handler && handler->descriptor);
|
||||
|
|
|
@ -30,6 +30,7 @@ namespace ircd::ios
|
|||
extern asio::executor user, main;
|
||||
extern std::thread::id main_thread_id;
|
||||
extern thread_local bool is_main_thread;
|
||||
extern bool user_available, main_available;
|
||||
|
||||
bool available() noexcept;
|
||||
const uint64_t &epoch() noexcept;
|
||||
|
@ -64,3 +65,11 @@ noexcept
|
|||
{
|
||||
return handler::epoch;
|
||||
}
|
||||
|
||||
inline bool
|
||||
__attribute__((always_inline))
|
||||
ircd::ios::available()
|
||||
noexcept
|
||||
{
|
||||
return main_available;
|
||||
}
|
||||
|
|
|
@ -689,7 +689,7 @@ ircd::fs::aio::system::handle_descriptor
|
|||
// appears to excessively allocate and deallocate 120 bytes; this
|
||||
// is a simple asynchronous operation, we can do better (and perhaps
|
||||
// even better than this below).
|
||||
[](ios::handler &handler, const size_t &size) -> void *
|
||||
[](ios::handler &handler, const size_t size) -> void *
|
||||
{
|
||||
assert(ircd::fs::aio::system);
|
||||
auto &system(*ircd::fs::aio::system);
|
||||
|
@ -705,7 +705,7 @@ ircd::fs::aio::system::handle_descriptor
|
|||
},
|
||||
|
||||
// no deallocation; satisfied by class member unique_ptr
|
||||
[](ios::handler &handler, void *const &ptr, const size_t &size) {},
|
||||
[](ios::handler &handler, void *const ptr, const size_t size) {},
|
||||
|
||||
// continuation
|
||||
true,
|
||||
|
|
53
ircd/ios.cc
53
ircd/ios.cc
|
@ -39,6 +39,14 @@ ircd::ios::primary;
|
|||
decltype(ircd::ios::main)
|
||||
ircd::ios::main;
|
||||
|
||||
/// Indicates the user asio::executor is initialized.
|
||||
decltype(ircd::ios::user_available)
|
||||
ircd::ios::user_available;
|
||||
|
||||
/// Indicates the main asio::executor is initialized.
|
||||
decltype(ircd::ios::main_available)
|
||||
ircd::ios::main_available;
|
||||
|
||||
decltype(ircd::boost_version_api)
|
||||
ircd::boost_version_api
|
||||
{
|
||||
|
@ -71,12 +79,14 @@ ircd::ios::init(asio::executor &&user)
|
|||
|
||||
// Save a reference handle to the user's executor.
|
||||
ios::user = user;
|
||||
ios::user_available = bool(ios::user);
|
||||
|
||||
// Create our strand instance.
|
||||
ios::primary.emplace(static_cast<asio::io_context &>(user.context()));
|
||||
|
||||
// Set the reference handle to our executor.
|
||||
ios::main = *ios::primary;
|
||||
ios::main_available = bool(ios::main);
|
||||
}
|
||||
|
||||
[[using gnu: cold]]
|
||||
|
@ -180,13 +190,6 @@ ircd::ios::forked_parent()
|
|||
#endif
|
||||
}
|
||||
|
||||
bool
|
||||
ircd::ios::available()
|
||||
noexcept
|
||||
{
|
||||
return bool(main);
|
||||
}
|
||||
|
||||
//
|
||||
// emption
|
||||
//
|
||||
|
@ -480,13 +483,6 @@ ircd::ios::descriptor::stats::stats(descriptor &d)
|
|||
{ "name", stats_name(d, "free_bytes") },
|
||||
},
|
||||
}
|
||||
,slice_total
|
||||
{
|
||||
value + items++,
|
||||
{
|
||||
{ "name", stats_name(d, "slice_total") },
|
||||
},
|
||||
}
|
||||
,slice_last
|
||||
{
|
||||
value + items++,
|
||||
|
@ -494,11 +490,11 @@ ircd::ios::descriptor::stats::stats(descriptor &d)
|
|||
{ "name", stats_name(d, "slice_last") },
|
||||
},
|
||||
}
|
||||
,latency_total
|
||||
,slice_total
|
||||
{
|
||||
value + items++,
|
||||
{
|
||||
{ "name", stats_name(d, "latency_total") },
|
||||
{ "name", stats_name(d, "slice_total") },
|
||||
},
|
||||
}
|
||||
,latency_last
|
||||
|
@ -508,6 +504,13 @@ ircd::ios::descriptor::stats::stats(descriptor &d)
|
|||
{ "name", stats_name(d, "latency_last") },
|
||||
},
|
||||
}
|
||||
,latency_total
|
||||
{
|
||||
value + items++,
|
||||
{
|
||||
{ "name", stats_name(d, "latency_total") },
|
||||
},
|
||||
}
|
||||
{
|
||||
assert(items <= (sizeof(value) / sizeof(value[0])));
|
||||
}
|
||||
|
@ -531,7 +534,7 @@ ircd::ios::handler::epoch;
|
|||
|
||||
[[gnu::cold]]
|
||||
bool
|
||||
ircd::ios::handler::fault(handler *const &handler)
|
||||
ircd::ios::handler::fault(handler *const handler)
|
||||
noexcept
|
||||
{
|
||||
assert(handler && handler->descriptor);
|
||||
|
@ -568,7 +571,7 @@ noexcept
|
|||
|
||||
[[gnu::hot]]
|
||||
void
|
||||
ircd::ios::handler::leave(handler *const &handler)
|
||||
ircd::ios::handler::leave(handler *const handler)
|
||||
noexcept
|
||||
{
|
||||
assert(handler && handler->descriptor);
|
||||
|
@ -619,19 +622,14 @@ noexcept
|
|||
|
||||
[[gnu::hot]]
|
||||
void
|
||||
ircd::ios::handler::enter(handler *const &handler)
|
||||
ircd::ios::handler::enter(handler *const handler)
|
||||
noexcept
|
||||
{
|
||||
assert(!handler::current);
|
||||
handler::current = handler;
|
||||
++handler::epoch;
|
||||
|
||||
assert(handler && handler->descriptor);
|
||||
auto &descriptor(*handler->descriptor);
|
||||
|
||||
assert(descriptor.stats);
|
||||
auto &stats(*descriptor.stats);
|
||||
++stats.calls;
|
||||
|
||||
const auto last_ts
|
||||
{
|
||||
|
@ -640,6 +638,11 @@ noexcept
|
|||
|
||||
stats.latency_last = handler->ts - last_ts;
|
||||
stats.latency_total += stats.latency_last;
|
||||
++stats.calls;
|
||||
|
||||
assert(!handler::current);
|
||||
handler::current = handler;
|
||||
++handler::epoch;
|
||||
|
||||
if constexpr(profile::logging)
|
||||
log::logf
|
||||
|
@ -696,7 +699,6 @@ ircd::ios::dispatch::dispatch(descriptor &descriptor,
|
|||
latch.wait();
|
||||
}
|
||||
|
||||
[[gnu::hot]]
|
||||
ircd::ios::dispatch::dispatch(descriptor &descriptor,
|
||||
defer_t,
|
||||
std::function<void ()> function)
|
||||
|
@ -725,7 +727,6 @@ ircd::ios::dispatch::dispatch(descriptor &descriptor,
|
|||
};
|
||||
}
|
||||
|
||||
[[gnu::hot]]
|
||||
ircd::ios::dispatch::dispatch(descriptor &descriptor,
|
||||
std::function<void ()> function)
|
||||
{
|
||||
|
|
Loading…
Reference in New Issue