From ad41fb69a96fbe2eaefee729cd118f4d0085fcb2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 14 Oct 2020 01:34:23 -0700 Subject: [PATCH] ircd::ctx: Support user-supplied stacks. --- include/ircd/ctx/context.h | 5 + include/ircd/ctx/stack.h | 6 +- ircd/ctx.cc | 211 ++++++++++++++++++++----------------- ircd/ctx.h | 5 +- 4 files changed, 129 insertions(+), 98 deletions(-) diff --git a/include/ircd/ctx/context.h b/include/ircd/ctx/context.h index 6bca9b0fa..2d7c1345f 100644 --- a/include/ircd/ctx/context.h +++ b/include/ircd/ctx/context.h @@ -64,6 +64,11 @@ struct ircd::ctx::context ctx *detach(); // other calls undefined after this call // Note: Constructing with DETACH flag makes any further use of this object undefined. + context(const string_view &name, + const mutable_buffer &stack, + const flags &, + function); + context(const string_view &name, const size_t &stack_size, const flags &, diff --git a/include/ircd/ctx/stack.h b/include/ircd/ctx/stack.h index 5d4d4ce50..cd0bdc3c7 100644 --- a/include/ircd/ctx/stack.h +++ b/include/ircd/ctx/stack.h @@ -15,13 +15,15 @@ namespace ircd::ctx struct ircd::ctx::stack { + struct allocator; + mutable_buffer buf; // complete allocation - uintptr_t base {0}; // assigned when spawned + uintptr_t base {0}; // base frame pointer size_t max {0}; // User given stack size size_t at {0}; // Updated for profiling at sleep size_t peak {0}; // Updated for profiling; maximum - stack(const size_t &max); + stack(const mutable_buffer &) noexcept; static const stack &get(const ctx &) noexcept; static stack &get(ctx &) noexcept; diff --git a/ircd/ctx.cc b/ircd/ctx.cc index 4fc698b79..b115a6804 100644 --- a/ircd/ctx.cc +++ b/ircd/ctx.cc @@ -63,9 +63,14 @@ ircd::ctx::ctx::ios_handler &ios_desc }; +/// Points to the next context to spawn (internal use) +[[gnu::visibility("internal")]] +decltype(ircd::ctx::ctx::spawning) +ircd::ctx::ctx::spawning; + /// Internal context struct ctor ircd::ctx::ctx::ctx(const string_view &name, - const size_t &stack_max, + const ircd::ctx::stack &stack, const context::flags &flags) :name { @@ -81,7 +86,7 @@ ircd::ctx::ctx::ctx(const string_view &name, } ,stack { - stack_max + stack } { } @@ -106,9 +111,14 @@ ircd::ctx::ctx::spawn(context::function func) boost::coroutines::no_stack_unwind, }; + const scope_restore spawning + { + ircd::ctx::ctx::spawning, this + }; + auto bound { - std::bind(&ctx::operator(), this, ph::_1, ph::_2, std::move(func)) + std::bind(&ctx::operator(), this, ph::_1, std::move(func)) }; auto *const parent_context @@ -142,7 +152,6 @@ ircd::ctx::ctx::spawn(context::function func) void IRCD_CTX_STACK_PROTECT ircd::ctx::ctx::operator()(boost::asio::yield_context yc, - const mutable_buffer &stack_buf, const std::function func) noexcept try { @@ -150,7 +159,6 @@ noexcept try ircd::ctx::current = this; this->yc = &yc; notes = 1; - stack.buf = stack_buf; stack.base = uintptr_t(__builtin_frame_address(0)); const unwind atexit{[this] { @@ -956,36 +964,6 @@ noexcept } #endif -////////////////////////////////////////////////////////////////////////////// -// -// ctx/stack.h -// - -[[gnu::hot]] -ircd::ctx::stack & -ircd::ctx::stack::get(ctx &ctx) -noexcept -{ - return ctx.stack; -} - -[[gnu::hot]] -const ircd::ctx::stack & -ircd::ctx::stack::get(const ctx &ctx) -noexcept -{ - return ctx.stack; -} - -// -// stack::stack -// - -ircd::ctx::stack::stack(const size_t &max) -:max{max} -{ -} - /////////////////////////////////////////////////////////////////////////////// // // ctx/continuation.h @@ -1174,12 +1152,23 @@ ircd::ctx::context::context(const string_view &name, const size_t &stack_sz, const flags &flags, function func) +:context +{ + name, {nullptr, DEFAULT_STACK_SIZE}, flags, std::move(func) +} +{ +} + +ircd::ctx::context::context(const string_view &name, + const mutable_buffer &stack, + const flags &flags, + function func) :c { std::make_unique ( name, - stack_sz, + stack, !current? flags | POST : flags ) } @@ -3087,42 +3076,77 @@ noexcept return c.node; } -/////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// // -// (internal) stack_allocator +// ctx/stack.h // -namespace ircd::ctx +[[gnu::hot]] +ircd::ctx::stack & +ircd::ctx::stack::get(ctx &ctx) +noexcept { - struct stack_allocator; + return ctx.stack; } +[[gnu::hot]] +const ircd::ctx::stack & +ircd::ctx::stack::get(const ctx &ctx) +noexcept +{ + return ctx.stack; +} + +// +// stack::stack +// + +ircd::ctx::stack::stack(const mutable_buffer &buf) +noexcept +:buf +{ + buf +} +,max +{ + ircd::size(buf) +} +{ +} + +// +// stack::allocator +// + struct [[gnu::visibility("hidden")]] -ircd::ctx::stack_allocator +ircd::ctx::stack::allocator { using stack_context = boost::coroutines::stack_context; mutable_buffer &buf; + bool owner {false}; void allocate(stack_context &, size_t size); void deallocate(stack_context &); }; void -ircd::ctx::stack_allocator::allocate(stack_context &c, - size_t size) +ircd::ctx::stack::allocator::allocate(stack_context &c, + size_t size) { - assert(size >= traits_type::minimum_size()); - assert(traits_type::is_unbounded() || (traits_type::maximum_size() >= size)); - static const auto &alignment { info::page_size }; - unique_mutable_buffer buf + unique_mutable_buffer umb { - size, alignment + null(this->buf)? size: 0, alignment + }; + + const mutable_buffer &buf + { + umb? umb: this->buf }; c.size = ircd::size(buf); @@ -3132,26 +3156,26 @@ ircd::ctx::stack_allocator::allocate(stack_context &c, c.valgrind_stack_id = VALGRIND_STACK_REGISTER(c.sp, ircd::data(buf)); #endif - this->buf = buf.release(); + this->owner = bool(umb); + this->buf = umb? umb.release(): this->buf; } void -ircd::ctx::stack_allocator::deallocate(stack_context &c) +ircd::ctx::stack::allocator::deallocate(stack_context &c) { assert(c.sp); - assert(traits_type::minimum_size() <= c.size); - assert(traits_type::is_unbounded() || (traits_type::maximum_size() >= c.size)); #if defined(BOOST_USE_VALGRIND) VALGRIND_STACK_DEREGISTER(c.valgrind_stack_id) #endif - void *const base + const auto base { - static_cast(c.sp) - c.size + (reinterpret_cast(c.sp) - c.size) + & boolmask(owner) }; - std::free(base); + std::free(reinterpret_cast(base)); } /////////////////////////////////////////////////////////////////////////////// @@ -3174,54 +3198,20 @@ boost::asio::detail::spawn_data weak_ptr coro_; Function function_; Handler handler_; - ircd::mutable_buffer stack_; + ircd::ctx::ctx *ctrl; template spawn_data(H&& handler, bool call_handler, F&& function) :function_(std::move(function)) ,handler_(std::move(handler)) + ,ctrl{ircd::ctx::ctx::spawning} { assert(call_handler); + assert(ctrl); } }; -template -struct [[gnu::visibility("hidden")]] -boost::asio::detail::spawn_helper -< - boost::asio::executor_binder>, - Function -> -{ - using Handler = boost::asio::executor_binder>; - using callee_type = typename basic_yield_context::callee_type; - - void operator()() // push - { - ircd::ctx::stack_allocator allocator - { - data_->stack_ - }; - - const auto coro - { - std::make_shared - ( - coro_entry_point{data_}, - attributes_, - allocator - ) - }; - - data_->coro_ = coro; - (*coro)(); - } - - shared_ptr> data_; - boost::coroutines::attributes attributes_; -}; - template struct [[gnu::visibility("hidden")]] boost::asio::detail::coro_entry_point @@ -3237,7 +3227,7 @@ boost::asio::detail::coro_entry_point { const auto data { - data_ + this->data }; basic_yield_context yc @@ -3245,11 +3235,44 @@ boost::asio::detail::coro_entry_point data->coro_, ca, data->handler_ }; - (data->function_)(yc, data->stack_); + (data->function_)(yc); (data->handler_)(); } + shared_ptr> data; +}; + +// Hooks the first push phase of coroutine spawn to supply our own stack +// allocator. +template +struct [[gnu::visibility("hidden")]] +boost::asio::detail::spawn_helper +< + boost::asio::executor_binder>, + Function +> +{ + using Handler = boost::asio::executor_binder>; + using callee_type = typename basic_yield_context::callee_type; + + void operator()() // push + { + const auto coro + { + std::make_shared + ( + coro_entry_point{data_}, + attributes_, + ircd::ctx::stack::allocator{data_->ctrl->stack.buf} + ) + }; + + data_->coro_ = coro; + (*coro)(); + } + shared_ptr> data_; + boost::coroutines::attributes attributes_; }; /////////////////////////////////////////////////////////////////////////////// diff --git a/ircd/ctx.h b/ircd/ctx.h index 9c38c1d84..51063376f 100644 --- a/ircd/ctx.h +++ b/ircd/ctx.h @@ -33,6 +33,7 @@ struct ircd::ctx::ctx static uint64_t id_ctr; // monotonic static ios::descriptor ios_desc; static ios::handler ios_handler; + static ctx *spawning; uint64_t id {++id_ctr}; // Unique runtime ID string_view name; // User given name (optional) @@ -61,11 +62,11 @@ struct ircd::ctx::ctx bool wait(); // yield context to ios queue (returns on this resume) void jump(); // jump to context directly (returns on your resume) - void operator()(boost::asio::yield_context, const mutable_buffer &, const std::function) noexcept; + void operator()(boost::asio::yield_context, const std::function) noexcept; void spawn(context::function func); ctx(const string_view &name = ""_sv, - const size_t &stack_max = DEFAULT_STACK_SIZE, + const ircd::ctx::stack & = mutable_buffer{nullptr, DEFAULT_STACK_SIZE}, const context::flags & = (context::flags)0U); ctx(ctx &&) = delete;