ircd: Break down client shutdown; improve various shutdown refusals.

include/ircd/client.h

@@ -19,8 +19,6 @@ namespace ircd
 	char *read(client &, char *&start, char *const &stop);
 	parse::read_closure read_closure(client &);
 
-	void close_all_clients();
-
 	std::shared_ptr<client> add_client(std::shared_ptr<socket>);  // Creates a client.
 }
 
@@ -42,6 +40,10 @@ struct ircd::client
 	static struct conf default_conf;
 	static ctx::pool context;
 
+	static void interrupt_all();
+	static void close_all();
+	static void wait_all();
+
 	struct conf *conf {&default_conf};
 	unique_buffer<mutable_buffer> head_buffer;
 	unique_buffer<mutable_buffer> content_buffer;
@@ -106,6 +108,8 @@ struct ircd::client::settings
 struct ircd::client::init
 {
 	void interrupt();
+	void close();
+	void wait();
 
 	init();
 	~init() noexcept;

ircd/client.cc

@@ -12,6 +12,8 @@
 
 namespace ircd
 {
+	ctx::dock dock;
+
 	template<class... args> std::shared_ptr<client> make_client(args&&...);
 }
 
@@ -49,45 +51,110 @@ ircd::client::init::init()
 	context.add(settings.pool_size);
 }
 
-void
-ircd::client::init::interrupt()
-{
-	if(context.active() || !client::list.empty())
-		log::warning("Interrupting %zu requests; dropping %zu requests; closing %zu clients...",
-		             context.active(),
-		             context.pending(),
-		             client::list.size());
-
-	context.interrupt();
-	close_all_clients();
-}
-
 ircd::client::init::~init()
 noexcept
 {
 	interrupt();
+	close();
+	wait();
+	assert(client::list.empty());
+}
 
-	if(context.active())
-		log::warning("Joining %zu active of %zu remaining request contexts...",
-		             context.active(),
-		             context.size());
-	else
-		log::debug("Waiting for %zu request contexts to join...",
-		           context.size());
+void
+ircd::client::init::interrupt()
+{
+	interrupt_all();
+}
 
-	context.join();
+void
+ircd::client::init::close()
+{
+	close_all();
+}
 
-	if(unlikely(!client::list.empty()))
-	{
-		log::error("%zu clients are unterminated...", client::list.size());
-		assert(client::list.empty());
-	}
+void
+ircd::client::init::wait()
+{
+	wait_all();
 }
 
 //
 // util
 //
 
+void
+ircd::client::interrupt_all()
+{
+	if(context.active())
+		log::warning
+		{
+			"Interrupting %zu requests; dropping %zu requests...",
+			context.active(),
+			context.pending()
+		};
+
+	context.interrupt();
+}
+
+void
+ircd::client::close_all()
+{
+	if(!client::list.empty())
+		log::debug
+		{
+			"Closing %zu clients", client::list.size()
+		};
+
+	auto it(begin(client::list));
+	while(it != end(client::list))
+	{
+		auto c(shared_from(**it)); ++it; try
+		{
+			c->close(net::dc::RST, [c](const auto &e)
+			{
+				dock.notify_one();
+			});
+		}
+		catch(const std::exception &e)
+		{
+			log::warning
+			{
+				"Error disconnecting client @%p: %s", c.get(), e.what()
+			};
+		}
+	}
+}
+
+void
+ircd::client::wait_all()
+{
+	if(context.active())
+		log::warning
+		{
+			"Joining %zu active of %zu remaining request contexts...",
+			context.active(),
+			context.size()
+		};
+	else
+		log::debug
+		{
+			"Waiting for %zu request contexts to join...",
+			context.size()
+		};
+
+	context.join();
+	while(!client::list.empty())
+	{
+		if(dock.wait_for(seconds(2)) == ctx::cv_status::no_timeout)
+			continue;
+
+		log::warning
+		{
+			"Waiting for %zu clients to close...", client::list.size()
+		};
+	}
+}
+
 ircd::parse::read_closure
 ircd::read_closure(client &client)
 {
@@ -122,6 +189,17 @@ ircd::read(client &client,
 std::shared_ptr<ircd::client>
 ircd::add_client(std::shared_ptr<socket> s)
 {
+	if(unlikely(ircd::runlevel != ircd::runlevel::RUN))
+	{
+		log::warning
+		{
+			"Refusing to add new client in runlevel %s", reflect(ircd::runlevel)
+		};
+
+		net::close(*s, net::dc::RST, net::close_ignore);
+		return {};
+	}
+
 	const auto client
 	{
 		make_client(std::move(s))
@@ -138,24 +216,6 @@ ircd::make_client(args&&... a)
 	return std::make_shared<client>(std::forward<args>(a)...);
 }
 
-void
-ircd::close_all_clients()
-{
-	auto it(begin(client::list));
-	while(it != end(client::list))
-	{
-		auto *const client(*it);
-		++it; try
-		{
-			client->close(net::dc::RST, net::close_ignore);
-		}
-		catch(const std::exception &e)
-		{
-			log::warning("Error disconnecting client @%p: %s", client, e.what());
-		}
-	}
-}
-
 ircd::ipport
 ircd::local(const client &client)
 {

ircd/ircd.cc

@@ -211,6 +211,9 @@ try
 		_server_.interrupt();
 		_client_.interrupt();
 		_server_.close();
+		_client_.close();
+		_server_.wait();
+		_client_.wait();
 	}};
 
 	// When the call to wait() below completes, IRCd exits from the RUN state

ircd/server.cc

@@ -13,7 +13,6 @@
 namespace ircd::server
 {
 	// Internal state
-	bool ready;         // like an /etc/nologin to prevent actions when false.
 	ctx::dock dock;     // internal semaphore
 
 	// Internal util
@@ -215,7 +214,6 @@ ircd::server::accumulate_peers(F&& closure)
 
 ircd::server::init::init()
 {
-	ready = true;
 }
 
 ircd::server::init::~init()
@@ -235,7 +233,6 @@ ircd::server::init::wait()
 void
 ircd::server::init::close()
 {
-	ready = false;
 	close_all();
 }
 
@@ -305,7 +302,7 @@ void
 ircd::server::submit(const hostport &hostport,
                      request &request)
 {
-	if(unlikely(!server::ready))
+	if(unlikely(ircd::runlevel != ircd::runlevel::RUN))
 		throw unavailable
 		{
 			"Unable to fulfill requests at this time."
@@ -428,7 +425,7 @@ void
 ircd::server::peer::submit(request &request)
 try
 {
-	if(!err_check() || unlikely(!server::ready))
+	if(!err_check() || unlikely(ircd::runlevel != ircd::runlevel::RUN))
 		throw unavailable
 		{
 			"Peer is unable to take any requests: %s", err_msg()
@@ -1118,7 +1115,7 @@ ircd::server::link::cancel_uncommitted(std::exception_ptr eptr)
 bool
 ircd::server::link::open(const net::open_opts &open_opts)
 {
-	assert(server::ready);
+	assert(ircd::runlevel == ircd::runlevel::RUN);
 
 	if(op_init)
 		return false;