diff --git a/include/ircd/fs/aio.h b/include/ircd/fs/aio.h index 00aee9dff..a8962467d 100644 --- a/include/ircd/fs/aio.h +++ b/include/ircd/fs/aio.h @@ -49,6 +49,7 @@ struct ircd::fs::aio::stats { uint32_t requests {0}; ///< count of requests created uint32_t complete {0}; ///< count of requests completed + uint32_t submits {0}; ///< count of io_submit calls uint32_t handles {0}; ///< count of event_fd callbacks uint32_t events {0}; ///< count of events from io_getevents uint32_t cancel {0}; ///< count of requests canceled @@ -70,6 +71,8 @@ struct ircd::fs::aio::stats uint32_t max_requests {0}; ///< maximum observed pending requests uint32_t max_reads {0}; ///< maximum observed pending reads uint32_t max_writes {0}; ///< maximum observed pending write + + uint32_t maxed_submits {0}; ///< number of submits at threshold }; struct ircd::fs::aio::init diff --git a/ircd/aio.cc b/ircd/aio.cc index 044e14a88..82ce7504f 100644 --- a/ircd/aio.cc +++ b/ircd/aio.cc @@ -237,18 +237,13 @@ try assert(ctx::current); assert(waiter == ctx::current); - struct iocb *const cbs[] - { - static_cast(this) - }; - const size_t submitted_bytes { bytes(iovec()) }; // Submit to kernel - syscall(context->idp, 1, &cbs); + context->submit(*this); // Update stats for submission phase stats.bytes_requests += submitted_bytes; @@ -261,6 +256,8 @@ try while(retval == std::numeric_limits::min()) ctx::wait(); + assert(retval <= ssize_t(submitted_bytes)); + // Update stats for completion phase. stats.bytes_complete += submitted_bytes; stats.complete++; @@ -393,6 +390,54 @@ ircd::fs::aio::kernel::wait() return true; } +void +ircd::fs::aio::kernel::submit(request &request) +noexcept try +{ + static const size_t threshold(16); + thread_local std::array queue; + thread_local size_t count; + thread_local size_t sem[2]; + + assert(request.aio_data == uintptr_t(&request)); + queue.at(count++) = static_cast(&request); + + if(count >= threshold) + { + syscall(context->idp, count, queue.data()); + ++stats.maxed_submits; + ++stats.submits; + count = 0; + return; + } + + ++sem[0]; + ircd::post([] + { + if(sem[0] > ++sem[1]) + return; + + if(!count) + return; + + syscall(context->idp, count, queue.data()); + stats.maxed_submits += count >= threshold; + ++stats.submits; + count = 0; + }); +} +catch(const std::exception &e) +{ + log::critical + { + "AIO(%p) submit: %s", + this, + e.what() + }; + + throw; +} + void ircd::fs::aio::kernel::set_handle() { @@ -495,7 +540,9 @@ noexcept try *reinterpret_cast(event.data) }; - assert(reinterpret_cast(event.obj) == static_cast(&request)); + auto *const iocb(reinterpret_cast(event.obj)); + assert(iocb == static_cast(&request)); + assert(reinterpret_cast(iocb->aio_data) == &request); assert(event.res2 >= 0); assert(event.res == -1 || event.res2 == 0); diff --git a/ircd/aio.h b/ircd/aio.h index b162ca290..4911d53ae 100644 --- a/ircd/aio.h +++ b/ircd/aio.h @@ -53,6 +53,9 @@ struct ircd::fs::aio::kernel void handle(const boost::system::error_code &, const size_t) noexcept; void set_handle(); + void submit(request &) noexcept; + + // Control panel bool wait(); bool interrupt(); diff --git a/modules/console.cc b/modules/console.cc index f02f754fb..727c2a737 100644 --- a/modules/console.cc +++ b/modules/console.cc @@ -895,6 +895,14 @@ console_cmd__aio(opt &out, const string_view &line) << std::setw(9) << std::right << s.events << std::endl; + out << std::setw(12) << std::left << "submits" + << std::setw(9) << std::right << s.submits + << std::endl; + + out << std::setw(12) << std::left << "submits max" + << std::setw(9) << std::right << s.maxed_submits + << std::endl; + return true; }