ircd::ctx: Factor shared_ptr out of promise/future.

include/ircd/ctx/future.h

@@ -18,7 +18,6 @@ namespace ircd::ctx
 	template<class T = void> class future;
 	template<> class future<void>;
 	template<class... T> struct scoped_future;
-
 	enum class future_status;
 
 	template<class T,
@@ -34,16 +33,16 @@ enum class ircd::ctx::future_status
 };
 
 template<class T>
-class ircd::ctx::future
+struct ircd::ctx::future
 {
-	std::shared_ptr<shared_state<T>> st;
+	shared_state<T> st;
 
   public:
 	using value_type                             = typename shared_state<T>::value_type;
 	using pointer_type                           = typename shared_state<T>::pointer_type;
 	using reference_type                         = typename shared_state<T>::reference_type;
 
-	bool valid() const                           { return bool(st);                                }
+	bool valid() const                           { return !invalid(st);                            }
 	bool operator!() const                       { return !valid();                                }
 	operator bool() const                        { return valid();                                 }
 
@@ -58,19 +57,24 @@ class ircd::ctx::future
 	T get();
 	operator T()                                 { return get();                                   }
 
-	future();
+	future() = default;
 	future(promise<T> &promise);
+	future(future &&) noexcept;
+	future(const future &) = delete;
+	future &operator=(future &&) noexcept;
+	future &operator=(const future &) = delete;
+	~future() noexcept;
 };
 
 template<>
-class ircd::ctx::future<void>
+struct ircd::ctx::future<void>
 {
-	std::shared_ptr<shared_state<void>> st;
+	shared_state<void> st;
 
   public:
 	using value_type                             = typename shared_state<void>::value_type;
 
-	bool valid() const                           { return bool(st);                                }
+	bool valid() const                           { return !invalid(st);                            }
 	bool operator!() const                       { return !valid();                                }
 	operator bool() const                        { return valid();                                 }
 
@@ -82,8 +86,13 @@ class ircd::ctx::future<void>
 	template<class duration> future_status wait(const duration &d) const;
 	void wait() const;
 
-	future();
+	future() = default;
 	future(promise<void> &promise);
+	future(future &&) noexcept;
+	future(const future &) = delete;
+	future &operator=(future &&) noexcept;
+	future &operator=(const future &) = delete;
+	~future() noexcept;
 };
 
 template<class... T>
@@ -112,28 +121,77 @@ noexcept
 		this->wait();
 }
 
-inline
-ircd::ctx::future<void>::future()
-:st{nullptr}
-{
-}
-
 template<class T>
-ircd::ctx::future<T>::future()
+ircd::ctx::future<T>::future(promise<T> &promise)
-:st{nullptr}
 {
+	assert(!promise.st);
+	st.p = &promise;
+	update(st);
+	assert(promise.st);
 }
 
 inline
 ircd::ctx::future<void>::future(promise<void> &promise)
-:st{promise.get_state().share()}
 {
+	assert(!promise.st);
+	st.p = &promise;
+	update(st);
+	assert(promise.st);
 }
 
 template<class T>
-ircd::ctx::future<T>::future(promise<T> &promise)
+ircd::ctx::future<T>::future(future<T> &&o)
-:st{promise.get_state().share()}
+noexcept
+:st{std::move(o.st)}
 {
+	update(st);
+	o.st.p = nullptr;
+}
+
+inline
+ircd::ctx::future<void>::future(future<void> &&o)
+noexcept
+:st{std::move(o.st)}
+{
+	update(st);
+	o.st.p = nullptr;
+}
+
+template<class T>
+ircd::ctx::future<T> &
+ircd::ctx::future<T>::operator=(future<T> &&o)
+noexcept
+{
+	this->~future();
+	st = std::move(o.st);
+	update(st);
+	o.st.p = nullptr;
+	return *this;
+}
+
+inline ircd::ctx::future<void> &
+ircd::ctx::future<void>::operator=(future<void> &&o)
+noexcept
+{
+	this->~future();
+	st = std::move(o.st);
+	update(st);
+	o.st.p = nullptr;
+	return *this;
+}
+
+template<class T>
+ircd::ctx::future<T>::~future()
+noexcept
+{
+	invalidate(st);
+}
+
+inline
+ircd::ctx::future<void>::~future()
+noexcept
+{
+	invalidate(st);
 }
 
 template<class T>
@@ -142,17 +200,10 @@ ircd::ctx::future<T>::get()
 {
 	wait();
 
-	if(unlikely(bool(st->eptr)))
+	if(unlikely(bool(st.eptr)))
-		std::rethrow_exception(st->eptr);
+		std::rethrow_exception(st.eptr);
 
-	return st->val;
+	return st.val;
-}
-
-inline void
-ircd::ctx::future<void>::wait()
-const
-{
-	this->wait_until(steady_clock::time_point::max());
 }
 
 template<class T>
@@ -163,9 +214,17 @@ const
 	this->wait_until(steady_clock::time_point::max());
 }
 
+inline void
+ircd::ctx::future<void>::wait()
+const
+{
+	this->wait_until(steady_clock::time_point::max());
+}
+
+template<class T>
 template<class duration>
 ircd::ctx::future_status
-ircd::ctx::future<void>::wait(const duration &d)
+ircd::ctx::future<T>::wait(const duration &d)
 const
 {
 	return this->wait_until(steady_clock::now() + d);
@@ -173,17 +232,7 @@ const
 
 template<class duration>
 ircd::ctx::future_status
-ircd::ctx::future<void>::wait(const duration &d,
+ircd::ctx::future<void>::wait(const duration &d)
-                              std::nothrow_t)
-const
-{
-	return this->wait_until(steady_clock::now() + d, std::nothrow);
-}
-
-template<class T>
-template<class duration>
-ircd::ctx::future_status
-ircd::ctx::future<T>::wait(const duration &d)
 const
 {
 	return this->wait_until(steady_clock::now() + d);
@@ -199,6 +248,15 @@ const
 	return this->wait_until(steady_clock::now() + d, std::nothrow);
 }
 
+template<class duration>
+ircd::ctx::future_status
+ircd::ctx::future<void>::wait(const duration &d,
+                              std::nothrow_t)
+const
+{
+	return this->wait_until(steady_clock::now() + d, std::nothrow);
+}
+
 template<class T>
 template<class time_point>
 ircd::ctx::future_status
@@ -208,6 +266,14 @@ const
 	return ircd::ctx::wait_until(*this, tp);
 }
 
+template<class time_point>
+ircd::ctx::future_status
+ircd::ctx::future<void>::wait_until(const time_point &tp)
+const
+{
+	return ircd::ctx::wait_until(*this, tp);
+}
+
 template<class T>
 template<class time_point>
 ircd::ctx::future_status
@@ -218,14 +284,6 @@ const
 	return ircd::ctx::wait_until(*this, tp, std::nothrow);
 }
 
-template<class time_point>
-ircd::ctx::future_status
-ircd::ctx::future<void>::wait_until(const time_point &tp)
-const
-{
-	return ircd::ctx::wait_until(*this, tp);
-}
-
 template<class time_point>
 ircd::ctx::future_status
 ircd::ctx::future<void>::wait_until(const time_point &tp,
@@ -261,15 +319,16 @@ ircd::ctx::wait_until(const future<T> &f,
 {
 	const auto wfun([&f]() -> bool
 	{
-		return f.st->finished;
+		return !pending(f.st);
 	});
 
-	if(unlikely(!f.valid()))
+	if(unlikely(invalid(f.st)))
-		throw no_state();
+		throw no_state{};
 
-	if(unlikely(!f.st->cond.wait_until(tp, wfun)))
+	if(unlikely(!f.st.cond.wait_until(tp, wfun)))
 		return future_status::timeout;
 
-	return likely(wfun())? future_status::ready:
+	return likely(wfun())?
+		future_status::ready:
 		future_status::deferred;
 }

include/ircd/ctx/promise.h

@@ -13,20 +13,31 @@
 
 namespace ircd::ctx
 {
+	template<class T = void> class promise;
+	template<> class promise<void>;
+
+	template<class T> class future;
+	template<> class future<void>;
+
 	IRCD_EXCEPTION(ircd::ctx::error, future_error)
 	IRCD_EXCEPTION(future_error, no_state)
 	IRCD_EXCEPTION(future_error, broken_promise)
 	IRCD_EXCEPTION(future_error, future_already_retrieved)
 	IRCD_EXCEPTION(future_error, promise_already_satisfied)
 
-	template<class T = void> class promise;
+	template<class T> size_t refcount(const shared_state<T> &);
-	template<> class promise<void>;
+	template<class T> void update(shared_state<T> &s);
+	template<class T> void invalidate(shared_state<T> &);
+	template<class T> void remove(shared_state<T> &, promise<T> &);
+	template<class T> void update(promise<T> &new_, promise<T> &old);
+	template<class T> void append(promise<T> &new_, promise<T> &old);
 }
 
 template<class T>
-class ircd::ctx::promise
+struct ircd::ctx::promise
 {
-	std::shared_ptr<shared_state<T>> st;
+	shared_state<T> *st {nullptr};               // Reference to the state resident in future
+	mutable promise *next {nullptr};             // Promise fwdlist to support copy semantics
 
   public:
 	using value_type                             = typename shared_state<T>::value_type;
@@ -34,108 +45,149 @@ class ircd::ctx::promise
 	using reference_type                         = typename shared_state<T>::reference_type;
 
 	bool valid() const                           { return bool(st);                                }
-	bool finished() const                        { return !valid() || st->finished;                }
 	bool operator!() const                       { return !valid();                                }
 	operator bool() const                        { return valid();                                 }
 
-	const shared_state<T> &get_state() const     { return *st;                                     }
+	const shared_state<T> &get_state() const     { assert(valid()); return *st;                    }
-	shared_state<T> &get_state()                 { return *st;                                     }
+	shared_state<T> &get_state()                 { assert(valid()); return *st;                    }
 
 	void set_exception(std::exception_ptr eptr);
 	void set_value(const T &val);
 	void set_value(T&& val);
 
-	promise();
+	promise() = default;
-	promise(promise &&o) noexcept = default;
+	promise(promise &&o) noexcept;
 	promise(const promise &);
 	promise &operator=(const promise &) = delete;
-	promise &operator=(promise &&) noexcept = default;
+	promise &operator=(promise &&) noexcept;
 	~promise() noexcept;
 };
 
 template<>
-class ircd::ctx::promise<void>
+struct ircd::ctx::promise<void>
 {
-	std::shared_ptr<shared_state<void>> st;
+	shared_state<void> *st {nullptr};            // Reference to the state resident in future
+	mutable promise *next {nullptr};             // Promise fwdlist to support copy semantics
 
   public:
 	using value_type                             = typename shared_state<void>::value_type;
 
 	bool valid() const                           { return bool(st);                                }
-	bool finished() const                        { return !valid() || st->finished;                }
 	bool operator!() const                       { return !valid();                                }
 	operator bool() const                        { return valid();                                 }
 
-	const shared_state<void> &get_state() const  { return *st;                                     }
+	const shared_state<void> &get_state() const  { assert(valid()); return *st;                    }
-	shared_state<void> &get_state()              { return *st;                                     }
+	shared_state<void> &get_state()              { assert(valid()); return *st;                    }
 
 	void set_exception(std::exception_ptr eptr);
 	void set_value();
 
-	promise();
+	promise() = default;
-	promise(promise &&o) noexcept = default;
+	promise(promise &&o) noexcept;
 	promise(const promise &);
 	promise &operator=(const promise &) = delete;
-	promise &operator=(promise &&) noexcept = default;
+	promise &operator=(promise &&) noexcept;
 	~promise() noexcept;
 };
 
-inline
-ircd::ctx::promise<void>::promise()
-:st{std::make_shared<shared_state<void>>()}
-{
-	++st->promise_refcnt;
-	assert(st->promise_refcnt == 1);
-}
-
 template<class T>
-ircd::ctx::promise<T>::promise()
+ircd::ctx::promise<T>::promise(promise<T> &&o)
-:st{std::make_shared<shared_state<T>>()}
+noexcept
+:st{std::move(o.st)}
+,next{std::move(o.next)}
 {
-	++st->promise_refcnt;
+	if(st)
-	assert(st->promise_refcnt == 1);
+	{
+		update(*this, o);
+		o.st = nullptr;
+	}
 }
 
 inline
-ircd::ctx::promise<void>::promise(const promise<void> &o)
+ircd::ctx::promise<void>::promise(promise<void> &&o)
-:st{o.st}
+noexcept
+:st{std::move(o.st)}
+,next{std::move(o.next)}
 {
-	if(valid())
+	if(st)
 	{
-		++st->promise_refcnt;
+		update(*this, o);
-		assert(st->promise_refcnt > 1);
+		o.st = nullptr;
 	}
 }
 
 template<class T>
 ircd::ctx::promise<T>::promise(const promise<T> &o)
 :st{o.st}
+,next{nullptr}
 {
-	if(valid())
+	append(*this, const_cast<promise<T> &>(o));
-	{
-		++st->promise_refcnt;
-		assert(st->promise_refcnt > 1);
-	}
 }
 
 inline
-ircd::ctx::promise<void>::~promise()
+ircd::ctx::promise<void>::promise(const promise<void> &o)
+:st{o.st}
+,next{nullptr}
+{
+	append(*this, const_cast<promise<void> &>(o));
+}
+
+template<class T>
+ircd::ctx::promise<T> &
+ircd::ctx::promise<T>::operator=(promise<T> &&o)
 noexcept
 {
-	if(valid())
+	this->~promise();
-		if(!--st->promise_refcnt)
+	st = std::move(o.st);
-			if(!st->finished && !st.unique())
+	next = std::move(o.next);
-				set_exception(std::make_exception_ptr(broken_promise()));
+	if(!st)
+		return *this;
+
+	update(*this, o);
+	o.st = nullptr;
+	return *this;
+}
+
+inline
+ircd::ctx::promise<void> &
+ircd::ctx::promise<void>::operator=(promise<void> &&o)
+noexcept
+{
+	this->~promise();
+	st = std::move(o.st);
+	next = std::move(o.next);
+	if(!st)
+		return *this;
+
+	update(*this, o);
+	o.st = nullptr;
+	return *this;
 }
 
 template<class T>
 ircd::ctx::promise<T>::~promise()
 noexcept
 {
-	if(valid())
+	if(!valid())
-		if(!--st->promise_refcnt)
+		return;
-			if(!st->finished && !st.unique())
+
+	if(refcount(*st) == 1)
 		set_exception(std::make_exception_ptr(broken_promise()));
+	else
+		remove(*st, *this);
+}
+
+inline
+ircd::ctx::promise<void>::~promise()
+noexcept
+{
+	if(!valid())
+		return;
+
+	if(refcount(*st) == 1)
+		set_exception(std::make_exception_ptr(broken_promise()));
+	else
+		remove(*st, *this);
 }
 
 template<class T>
@@ -143,18 +195,11 @@ void
 ircd::ctx::promise<T>::set_value(T&& val)
 {
 	assert(valid());
-	assert(!finished());
+	assert(pending(*st));
+	auto *const st{this->st};
+	invalidate(*st);
+	set_ready(*st);
 	st->val = std::move(val);
-	st->finished = true;
-	st->cond.notify_all();
-}
-
-inline void
-ircd::ctx::promise<void>::set_value()
-{
-	assert(valid());
-	assert(!finished());
-	st->finished = true;
 	st->cond.notify_all();
 }
 
@@ -163,19 +208,22 @@ void
 ircd::ctx::promise<T>::set_value(const T &val)
 {
 	assert(valid());
-	assert(!finished());
+	assert(pending(*st));
+	auto *const st{this->st};
+	invalidate(*st);
+	set_ready(*st);
 	st->val = val;
-	st->finished = true;
 	st->cond.notify_all();
 }
 
 inline void
-ircd::ctx::promise<void>::set_exception(std::exception_ptr eptr)
+ircd::ctx::promise<void>::set_value()
 {
 	assert(valid());
-	assert(!finished());
+	assert(pending(*st));
-	st->eptr = std::move(eptr);
+	auto *const st{this->st};
-	st->finished = true;
+	invalidate(*st);
+	set_ready(*st);
 	st->cond.notify_all();
 }
 
@@ -184,8 +232,116 @@ void
 ircd::ctx::promise<T>::set_exception(std::exception_ptr eptr)
 {
 	assert(valid());
-	assert(!finished());
+	assert(pending(*st));
+	auto *const st{this->st};
+	invalidate(*st);
+	set_ready(*st);
 	st->eptr = std::move(eptr);
-	st->finished = true;
 	st->cond.notify_all();
 }
+
+inline void
+ircd::ctx::promise<void>::set_exception(std::exception_ptr eptr)
+{
+	assert(valid());
+	assert(pending(*st));
+	auto *const st{this->st};
+	invalidate(*st);
+	set_ready(*st);
+	st->eptr = std::move(eptr);
+	st->cond.notify_all();
+}
+
+template<class T>
+void
+ircd::ctx::append(promise<T> &new_,
+                  promise<T> &old)
+{
+	if(!old.next)
+	{
+		old.next = &new_;
+		return;
+	}
+
+	promise<T> *next{old.next};
+	for(; next->next; next = next->next);
+	next->next = &new_;
+}
+
+template<class T>
+void
+ircd::ctx::update(promise<T> &new_,
+                  promise<T> &old)
+{
+	assert(old.st);
+	auto &st{*old.st};
+	if(!pending(st))
+		return;
+
+	if(st.p == &old)
+	{
+		st.p = &new_;
+		return;
+	}
+
+	promise<T> *last{st.p};
+	for(promise<T> *next{last->next}; next; last = next, next = last->next)
+		if(next == &old)
+		{
+			last->next = &new_;
+			break;
+		}
+}
+
+template<class T>
+void
+ircd::ctx::remove(shared_state<T> &st,
+                  promise<T> &p)
+{
+	if(!pending(st))
+		return;
+
+	if(st.p == &p)
+	{
+		st.p = p.next;
+		return;
+	}
+
+	promise<T> *last{st.p};
+	for(promise<T> *next{last->next}; next; last = next, next = last->next)
+		if(next == &p)
+		{
+			last->next = p.next;
+			break;
+		}
+}
+
+template<class T>
+void
+ircd::ctx::invalidate(shared_state<T> &st)
+{
+	if(pending(st))
+		for(promise<T> *p{st.p}; p; p = p->next)
+			p->st = nullptr;
+}
+
+template<class T>
+void
+ircd::ctx::update(shared_state<T> &st)
+{
+	if(pending(st))
+		for(promise<T> *p{st.p}; p; p = p->next)
+			p->st = &st;
+}
+
+template<class T>
+size_t
+ircd::ctx::refcount(const shared_state<T> &st)
+{
+	size_t ret{0};
+	if(pending(st))
+		for(const promise<T> *p{st.p}; p; p = p->next)
+			++ret;
+
+	return ret;
+}

include/ircd/ctx/shared_state.h

@@ -16,66 +16,67 @@ namespace ircd::ctx
 	struct shared_state_base;
 	template<class T = void> struct shared_state;
 	template<> struct shared_state<void>;
+
+	template<class T> struct promise;
+	template<> struct promise<void>;
+
+	template<class T> bool invalid(const shared_state<T> &);
+	template<class T> bool pending(const shared_state<T> &);
+	template<class T> bool ready(const shared_state<T> &);
+	template<class T> void set_ready(shared_state<T> &);
 }
 
 struct ircd::ctx::shared_state_base
 {
-	dock cond;
+	mutable dock cond;
 	std::exception_ptr eptr;
-	uint promise_refcnt {0};
-	bool finished {false};
 };
 
 template<class T>
 struct ircd::ctx::shared_state
 :shared_state_base
-,std::enable_shared_from_this<shared_state<T>>
 {
 	using value_type      = T;
 	using pointer_type    = T *;
 	using reference_type  = T &;
 
+	promise<T> *p {nullptr};
 	T val;
-
-	std::shared_ptr<const shared_state<T>> share() const;
-	std::shared_ptr<shared_state<T>> share();
 };
 
 template<>
 struct ircd::ctx::shared_state<void>
 :shared_state_base
-,std::enable_shared_from_this<shared_state<void>>
 {
 	using value_type      = void;
 
-	std::shared_ptr<const shared_state<void>> share() const;
+	promise<void> *p {nullptr};
-	std::shared_ptr<shared_state<void>> share();
 };
 
-inline std::shared_ptr<ircd::ctx::shared_state<void>>
+template<class T>
-ircd::ctx::shared_state<void>::share()
+void
+ircd::ctx::set_ready(shared_state<T> &st)
 {
-	return this->shared_from_this();
+	st.p = reinterpret_cast<promise<T> *>(uintptr_t(0x42));
 }
 
 template<class T>
-std::shared_ptr<ircd::ctx::shared_state<T>>
+bool
-ircd::ctx::shared_state<T>::share()
+ircd::ctx::ready(const shared_state<T> &st)
 {
-	return this->shared_from_this();
+	return st.p == reinterpret_cast<const promise<T> *>(uintptr_t(0x42));
-}
-
-inline std::shared_ptr<const ircd::ctx::shared_state<void>>
-ircd::ctx::shared_state<void>::share()
-const
-{
-	return this->shared_from_this();
 }
 
 template<class T>
-std::shared_ptr<const ircd::ctx::shared_state<T>>
+bool
-ircd::ctx::shared_state<T>::share()
+ircd::ctx::pending(const shared_state<T> &st)
-const
 {
-	return this->shared_from_this();
+	return st.p > reinterpret_cast<const promise<T> *>(uintptr_t(0x42));
+}
+
+template<class T>
+bool
+ircd::ctx::invalid(const shared_state<T> &st)
+{
+	return st.p == nullptr;
 }

ircd/server.cc

@@ -2331,6 +2331,9 @@ template<class... args>
 void
 ircd::server::tag::set_exception(args&&... a)
 {
+	if(abandoned())
+		return;
+
 	set_exception(std::make_exception_ptr(std::forward<args>(a)...));
 }
 
@@ -2348,7 +2351,7 @@ bool
 ircd::server::tag::abandoned()
 const
 {
-	return p.finished();
+	return !p.valid();
 }
 
 bool