From 0f7e17a5197d8953a5d6b176996100585257f82a Mon Sep 17 00:00:00 2001
From: Jason Volk <jason@zemos.net>
Date: Wed, 31 Oct 2018 15:03:32 -0700
Subject: [PATCH] ircd::db: Add resume from error interface w/ console cmd.

---
 include/ircd/db/database/database.h |  1 +
 ircd/db.cc                          | 37 +++++++++++++++++++++++++++++
 modules/console.cc                  | 29 ++++++++++++++++++++++
 3 files changed, 67 insertions(+)

diff --git a/include/ircd/db/database/database.h b/include/ircd/db/database/database.h
index 9b9723bfa..b78a94af9 100644
--- a/include/ircd/db/database/database.h
+++ b/include/ircd/db/database/database.h
@@ -39,6 +39,7 @@ namespace ircd::db
 	void setopt(database &, const string_view &key, const string_view &val);
 	void fdeletions(database &, const bool &enable, const bool &force = false);
 	uint64_t checkpoint(database &);
+	void resume(database &);
 	void check(database &);
 	void compact(database &, const compactor & = {});
 	void sort(database &, const bool &blocking = true);
diff --git a/ircd/db.cc b/ircd/db.cc
index 0dffcff69..88913e388 100644
--- a/ircd/db.cc
+++ b/ircd/db.cc
@@ -373,6 +373,32 @@ ircd::db::check(database &d)
 	};
 }
 
+void
+ircd::db::resume(database &d)
+{
+	assert(d.d);
+	const ctx::uninterruptible::nothrow ui;
+	const std::lock_guard<decltype(write_mutex)> lock{write_mutex};
+	log::debug
+	{
+		log, "'%s': Attempting to resume @%lu",
+		name(d),
+		sequence(d)
+	};
+
+	throw_on_error
+	{
+		d.d->Resume()
+	};
+
+	log::info
+	{
+		log, "'%s': Resumed normal operation at sequence number %lu.",
+		name(d),
+		sequence(d)
+	};
+}
+
 /// Writes a snapshot of this database to the directory specified. The
 /// snapshot consists of hardlinks to the bulk data files of this db, but
 /// copies the other stuff that usually gets corrupted. The directory can
@@ -2354,6 +2380,17 @@ noexcept
 		reflect(reason),
 		status->ToString()
 	};
+
+	// This is a legitimate when we want to use it. If the error is not
+	// suppressed the DB will enter read-only mode and will require a
+	// call to db::resume() to clear the error (i.e by admin at console).
+	const bool ignore
+	{
+		false
+	};
+
+	if(ignore)
+		*status = rocksdb::Status::OK();
 }
 
 void
diff --git a/modules/console.cc b/modules/console.cc
index 677054915..74c0a571b 100644
--- a/modules/console.cc
+++ b/modules/console.cc
@@ -1323,6 +1323,35 @@ catch(const std::out_of_range &e)
 	return true;
 }
 
+bool
+console_cmd__db__resume(opt &out, const string_view &line)
+try
+{
+	const params param{line, " ",
+	{
+		"dbname",
+	}};
+
+	const auto dbname
+	{
+		param.at("dbname")
+	};
+
+	auto &database
+	{
+		db::database::get(dbname)
+	};
+
+	resume(database);
+	out << "resumed database " << dbname << std::endl;
+	return true;
+}
+catch(const std::out_of_range &e)
+{
+	out << "No open database by that name" << std::endl;
+	return true;
+}
+
 bool
 console_cmd__db__ticker(opt &out, const string_view &line)
 try