// Matrix Construct // // Copyright (C) Matrix Construct Developers, Authors & Contributors // Copyright (C) 2016-2018 Jason Volk // // Permission to use, copy, modify, and/or distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice is present in all copies. The // full license for this software is available in the LICENSE file. /// "main" thread for IRCd; the one the main context landed on. decltype(ircd::ios::main_thread_id) ircd::ios::main_thread_id; decltype(ircd::ios::user) ircd::ios::user; decltype(ircd::boost_version_api) ircd::boost_version_api { "boost", info::versions::API, BOOST_VERSION, { BOOST_VERSION / 100000, BOOST_VERSION / 100 % 1000, BOOST_VERSION % 100, } }; decltype(ircd::boost_version_abi) ircd::boost_version_abi { "boost", info::versions::ABI //TODO: get this }; // // init // void ircd::ios::init(asio::io_context &user) { // Sample the ID of this thread. Since this is the first transfer of // control to libircd after static initialization we have nothing to // consider a main thread yet. We need something set for many assertions // to pass until ircd::main() is entered which will reset this to where // ios.run() is really running. main_thread_id = std::this_thread::get_id(); // Set a reference to the user's ios_service ios::user = &user; } // // descriptor // template<> decltype(ircd::util::instance_list::allocator) ircd::util::instance_list::allocator {}; template<> decltype(ircd::util::instance_list::list) ircd::util::instance_list::list { allocator }; decltype(ircd::ios::descriptor::ids) ircd::ios::descriptor::ids; // // descriptor::descriptor // ircd::ios::descriptor::descriptor(const string_view &name, const decltype(allocator) &allocator, const decltype(deallocator) &deallocator, const bool &continuation) :name { name } ,stats { std::make_unique() } ,allocator { allocator?: default_allocator } ,deallocator { deallocator?: default_deallocator } ,continuation { continuation } { } ircd::ios::descriptor::~descriptor() noexcept { } [[gnu::hot]] void ircd::ios::descriptor::default_deallocator(handler &handler, void *const &ptr, const size_t &size) noexcept { #ifdef __clang__ ::operator delete(ptr); #else ::operator delete(ptr, size); #endif } [[gnu::hot]] void * ircd::ios::descriptor::default_allocator(handler &handler, const size_t &size) { return ::operator new(size); } // // descriptor::stats // ircd::ios::descriptor::stats::stats() { } ircd::ios::descriptor::stats::~stats() noexcept { } struct ircd::ios::descriptor::stats & ircd::ios::descriptor::stats::operator+=(const stats &o) & { queued += o.queued; calls += o.calls; faults += o.faults; allocs += o.allocs; alloc_bytes += o.alloc_bytes; frees += o.frees; free_bytes += o.free_bytes; slice_total += o.slice_total; slice_last += o.slice_last; latency_total += o.latency_total; latency_last += o.latency_last; return *this; } // // handler // decltype(ircd::ios::handler::current) thread_local ircd::ios::handler::current; decltype(ircd::ios::handler::epoch) thread_local ircd::ios::handler::epoch; [[gnu::cold]] bool ircd::ios::handler::fault(handler *const &handler) noexcept { assert(handler && handler->descriptor); auto &descriptor(*handler->descriptor); assert(descriptor.stats); auto &stats(*descriptor.stats); ++stats.faults; bool ret(false); // leave() isn't called if we return false so the tsc counter // needs to be tied off here instead. if(!ret) { stats.slice_last = cycles() - handler->ts; stats.slice_total += stats.slice_last; assert(handler::current == handler); handler::current = nullptr; } return ret; } [[gnu::hot]] void ircd::ios::handler::leave(handler *const &handler) noexcept { assert(handler && handler->descriptor); auto &descriptor(*handler->descriptor); assert(descriptor.stats); auto &stats(*descriptor.stats); const auto slice_stop { cycles() }; // NOTE: will fail without constant_tsc; // NOTE: may fail without nonstop_tsc after OS suspend mode assert(slice_stop >= handler->ts); stats.slice_last = slice_stop - handler->ts; stats.slice_total += stats.slice_last; assert(handler::current == handler); handler::current = nullptr; } [[gnu::hot]] void ircd::ios::handler::enter(handler *const &handler) noexcept { assert(!handler::current); assert(handler && handler->descriptor); auto &descriptor(*handler->descriptor); assert(descriptor.stats); auto &stats(*descriptor.stats); handler::current = handler; ++handler::epoch; ++stats.calls; const auto last_ts { std::exchange(handler->ts, cycles()) }; stats.latency_last = handler->ts - last_ts; stats.latency_total += stats.latency_last; } [[gnu::hot]] bool ircd::ios::handler::continuation(handler *const &handler) noexcept { assert(handler && handler->descriptor); auto &descriptor(*handler->descriptor); return descriptor.continuation; } [[gnu::hot]] void ircd::ios::handler::deallocate(handler *const &handler, void *const &ptr, const size_t &size) noexcept { assert(handler && handler->descriptor); auto &descriptor(*handler->descriptor); descriptor.deallocator(*handler, ptr, size); assert(descriptor.stats); auto &stats(*descriptor.stats); stats.free_bytes += size; ++stats.frees; } [[gnu::hot]] void * ircd::ios::handler::allocate(handler *const &handler, const size_t &size) { assert(handler && handler->descriptor); auto &descriptor(*handler->descriptor); assert(descriptor.stats); auto &stats(*descriptor.stats); stats.alloc_bytes += size; ++stats.allocs; return descriptor.allocator(*handler, size); } // // ios.h // namespace ircd::ios { extern descriptor post_desc; extern descriptor defer_desc; extern descriptor dispatch_desc; } void ircd::ios::forking() { get().notify_fork(asio::execution_context::fork_prepare); } void ircd::ios::forked_child() { get().notify_fork(asio::execution_context::fork_child); } void ircd::ios::forked_parent() { get().notify_fork(asio::execution_context::fork_parent); } // // dispatch // decltype(ircd::ios::dispatch_desc) ircd::ios::dispatch_desc { "ircd::ios dispatch" }; [[gnu::hot]] ircd::ios::dispatch::dispatch(std::function function) :dispatch { dispatch_desc, std::move(function) } { } ircd::ios::dispatch::dispatch(synchronous_t, const std::function &function) :dispatch { dispatch_desc, synchronous, std::move(function) } { } ircd::ios::dispatch::dispatch(descriptor &descriptor, synchronous_t) :dispatch { dispatch_desc, synchronous, [] { } } { } ircd::ios::dispatch::dispatch(descriptor &descriptor, synchronous_t, const std::function &function) { const ctx::uninterruptible::nothrow ui; ctx::latch latch(1); dispatch(descriptor, [&function, &latch] { const unwind uw{[&latch] { latch.count_down(); }}; function(); }); latch.wait(); } [[gnu::hot]] ircd::ios::dispatch::dispatch(descriptor &descriptor, std::function function) { boost::asio::dispatch(get(), handle(descriptor, std::move(function))); } // // defer // decltype(ircd::ios::defer_desc) ircd::ios::defer_desc { "ircd::ios defer", descriptor::default_allocator, descriptor::default_deallocator, true, // continuation }; [[gnu::hot]] ircd::ios::defer::defer(std::function function) :defer { defer_desc, std::move(function) } { } ircd::ios::defer::defer(synchronous_t, const std::function &function) :defer { defer_desc, synchronous, function } { } ircd::ios::defer::defer(descriptor &descriptor, synchronous_t) :defer { defer_desc, synchronous, [] { } } { } ircd::ios::defer::defer(descriptor &descriptor, synchronous_t, const std::function &function) { const ctx::uninterruptible::nothrow ui; ctx::latch latch(1); defer(descriptor, [&function, &latch] { const unwind uw{[&latch] { latch.count_down(); }}; function(); }); latch.wait(); } [[gnu::hot]] ircd::ios::defer::defer(descriptor &descriptor, std::function function) { boost::asio::defer(get(), handle(descriptor, std::move(function))); } // // post // decltype(ircd::ios::post_desc) ircd::ios::post_desc { "ircd::ios post" }; [[gnu::hot]] ircd::ios::post::post(std::function function) :post { post_desc, std::move(function) } { } ircd::ios::post::post(synchronous_t, const std::function &function) :post { post_desc, synchronous, function } { } ircd::ios::post::post(descriptor &descriptor, synchronous_t) :post { descriptor, synchronous, [] { } } { } ircd::ios::post::post(descriptor &descriptor, synchronous_t, const std::function &function) { const ctx::uninterruptible::nothrow ui; ctx::latch latch(1); post(descriptor, [&function, &latch] { const unwind uw{[&latch] { latch.count_down(); }}; function(); }); latch.wait(); } [[gnu::hot]] ircd::ios::post::post(descriptor &descriptor, std::function function) { boost::asio::post(get(), handle(descriptor, std::move(function))); } [[gnu::hot]] boost::asio::io_context & ircd::ios::get() noexcept { assert(user); return *user; } [[gnu::hot]] bool ircd::ios::available() noexcept { return bool(user); }