ircd::db: Add resume from error interface w/ console cmd.

include/ircd/db/database/database.h

@@ -39,6 +39,7 @@ namespace ircd::db
 	void setopt(database &, const string_view &key, const string_view &val);
 	void fdeletions(database &, const bool &enable, const bool &force = false);
 	uint64_t checkpoint(database &);
+	void resume(database &);
 	void check(database &);
 	void compact(database &, const compactor & = {});
 	void sort(database &, const bool &blocking = true);

ircd/db.cc

@@ -373,6 +373,32 @@ ircd::db::check(database &d)
 	};
 }
 
+void
+ircd::db::resume(database &d)
+{
+	assert(d.d);
+	const ctx::uninterruptible::nothrow ui;
+	const std::lock_guard<decltype(write_mutex)> lock{write_mutex};
+	log::debug
+	{
+		log, "'%s': Attempting to resume @%lu",
+		name(d),
+		sequence(d)
+	};
+
+	throw_on_error
+	{
+		d.d->Resume()
+	};
+
+	log::info
+	{
+		log, "'%s': Resumed normal operation at sequence number %lu.",
+		name(d),
+		sequence(d)
+	};
+}
+
 /// Writes a snapshot of this database to the directory specified. The
 /// snapshot consists of hardlinks to the bulk data files of this db, but
 /// copies the other stuff that usually gets corrupted. The directory can
@@ -2354,6 +2380,17 @@ noexcept
 		reflect(reason),
 		status->ToString()
 	};
+
+	// This is a legitimate when we want to use it. If the error is not
+	// suppressed the DB will enter read-only mode and will require a
+	// call to db::resume() to clear the error (i.e by admin at console).
+	const bool ignore
+	{
+		false
+	};
+
+	if(ignore)
+		*status = rocksdb::Status::OK();
 }
 
 void

modules/console.cc

@@ -1323,6 +1323,35 @@ catch(const std::out_of_range &e)
 	return true;
 }
 
+bool
+console_cmd__db__resume(opt &out, const string_view &line)
+try
+{
+	const params param{line, " ",
+	{
+		"dbname",
+	}};
+
+	const auto dbname
+	{
+		param.at("dbname")
+	};
+
+	auto &database
+	{
+		db::database::get(dbname)
+	};
+
+	resume(database);
+	out << "resumed database " << dbname << std::endl;
+	return true;
+}
+catch(const std::out_of_range &e)
+{
+	out << "No open database by that name" << std::endl;
+	return true;
+}
+
 bool
 console_cmd__db__ticker(opt &out, const string_view &line)
 try