From 5dcc7cd93b13beeeae74d5cf9a3987a82e9a0fb6 Mon Sep 17 00:00:00 2001
From: Jason Volk <jason@zemos.net>
Date: Wed, 31 Oct 2018 15:29:00 -0700
Subject: [PATCH] ircd::db: Add error report state w/ interface w/ console cmd.

---
 include/ircd/db/database/database.h |  2 ++
 ircd/db.cc                          | 48 ++++++++++++++++++++++++-----
 modules/console.cc                  | 45 +++++++++++++++++++++++++++
 3 files changed, 88 insertions(+), 7 deletions(-)

diff --git a/include/ircd/db/database/database.h b/include/ircd/db/database/database.h
index b78a94af9..04fe994a4 100644
--- a/include/ircd/db/database/database.h
+++ b/include/ircd/db/database/database.h
@@ -19,6 +19,7 @@ namespace ircd::db
 	const std::string &name(const database &);
 	const std::string &uuid(const database &);
 	uint64_t sequence(const database &); // Latest sequence number
+	const std::vector<std::string> &errors(const database &);
 	std::vector<std::string> files(const database &, uint64_t &msz);
 	std::vector<std::string> files(const database &);
 	size_t file_count(const database &);
@@ -100,6 +101,7 @@ struct ircd::db::database
 	std::list<std::shared_ptr<column>> columns; // active only
 	std::string uuid;
 	std::unique_ptr<rocksdb::Checkpoint> checkpointer;
+	std::vector<std::string> errors;
 
 	operator std::shared_ptr<database>()         { return shared_from_this();                      }
 	operator const rocksdb::DB &() const         { return *d;                                      }
diff --git a/ircd/db.cc b/ircd/db.cc
index 88913e388..01470f0f0 100644
--- a/ircd/db.cc
+++ b/ircd/db.cc
@@ -379,10 +379,16 @@ ircd::db::resume(database &d)
 	assert(d.d);
 	const ctx::uninterruptible::nothrow ui;
 	const std::lock_guard<decltype(write_mutex)> lock{write_mutex};
+	const auto errors
+	{
+		db::errors(d)
+	};
+
 	log::debug
 	{
-		log, "'%s': Attempting to resume @%lu",
+		log, "'%s': Attempting to resume from %zu errors @%lu",
 		name(d),
+		errors.size(),
 		sequence(d)
 	};
 
@@ -391,11 +397,14 @@ ircd::db::resume(database &d)
 		d.d->Resume()
 	};
 
+	d.errors.clear();
+
 	log::info
 	{
-		log, "'%s': Resumed normal operation at sequence number %lu.",
+		log, "'%s': Resumed normal operation at sequence number %lu; cleared %zu errors",
 		name(d),
-		sequence(d)
+		sequence(d),
+		errors.size()
 	};
 }
 
@@ -537,6 +546,12 @@ ircd::db::files(const database &cd,
 	return ret;
 }
 
+const std::vector<std::string> &
+ircd::db::errors(const database &d)
+{
+	return d.errors;
+}
+
 uint64_t
 ircd::db::sequence(const database &cd)
 {
@@ -2371,15 +2386,17 @@ ircd::db::database::events::OnBackgroundError(rocksdb::BackgroundErrorReason rea
                                               rocksdb::Status *const status)
 noexcept
 {
+	assert(d);
 	assert(status);
-	log::error
+
+	thread_local char buf[1024];
+	const string_view str{fmt::sprintf
 	{
-		rog, "'%s' background %s error in %s :%s",
-		d->name,
+		buf, "%s error in %s :%s",
 		reflect(status->severity()),
 		reflect(reason),
 		status->ToString()
-	};
+	}};
 
 	// This is a legitimate when we want to use it. If the error is not
 	// suppressed the DB will enter read-only mode and will require a
@@ -2389,8 +2406,25 @@ noexcept
 		false
 	};
 
+	const log::facility fac
+	{
+		ignore?
+			log::facility::DERROR:
+			log::facility::ERROR
+	};
+
+	log::logf
+	{
+		log, fac, "'%s': %s", d->name, str
+	};
+
 	if(ignore)
+	{
 		*status = rocksdb::Status::OK();
+		return;
+	}
+
+	d->errors.emplace_back(str);
 }
 
 void
diff --git a/modules/console.cc b/modules/console.cc
index 74c0a571b..247ec2753 100644
--- a/modules/console.cc
+++ b/modules/console.cc
@@ -1352,6 +1352,42 @@ catch(const std::out_of_range &e)
 	return true;
 }
 
+bool
+console_cmd__db__errors(opt &out, const string_view &line)
+try
+{
+	const params param{line, " ",
+	{
+		"dbname",
+	}};
+
+	const auto dbname
+	{
+		param.at("dbname")
+	};
+
+	auto &database
+	{
+		db::database::get(dbname)
+	};
+
+	const auto &errors
+	{
+		db::errors(database)
+	};
+
+	size_t i(0);
+	for(const auto &error : errors)
+		out << std::setw(2) << std::left << (i++) << ':' << error << std::endl;
+
+	return true;
+}
+catch(const std::out_of_range &e)
+{
+	out << "No open database by that name" << std::endl;
+	return true;
+}
+
 bool
 console_cmd__db__ticker(opt &out, const string_view &line)
 try
@@ -2930,6 +2966,15 @@ try
 		}
 	}
 
+	if(!c && !errors(d).empty())
+	{
+		size_t i(0);
+		out << std::endl;
+		out << "ERRORS (" << errors(d).size() << "): " << std::endl;
+		for(const auto &error : errors(d))
+			out << std::setw(2) << (i++) << ':' << error << std::endl;
+	}
+
 	return true;
 }
 catch(const std::out_of_range &e)