diff --git a/include/ircd/db/database/database.h b/include/ircd/db/database/database.h index b78a94af9..04fe994a4 100644 --- a/include/ircd/db/database/database.h +++ b/include/ircd/db/database/database.h @@ -19,6 +19,7 @@ namespace ircd::db const std::string &name(const database &); const std::string &uuid(const database &); uint64_t sequence(const database &); // Latest sequence number + const std::vector &errors(const database &); std::vector files(const database &, uint64_t &msz); std::vector files(const database &); size_t file_count(const database &); @@ -100,6 +101,7 @@ struct ircd::db::database std::list> columns; // active only std::string uuid; std::unique_ptr checkpointer; + std::vector errors; operator std::shared_ptr() { return shared_from_this(); } operator const rocksdb::DB &() const { return *d; } diff --git a/ircd/db.cc b/ircd/db.cc index 88913e388..01470f0f0 100644 --- a/ircd/db.cc +++ b/ircd/db.cc @@ -379,10 +379,16 @@ ircd::db::resume(database &d) assert(d.d); const ctx::uninterruptible::nothrow ui; const std::lock_guard lock{write_mutex}; + const auto errors + { + db::errors(d) + }; + log::debug { - log, "'%s': Attempting to resume @%lu", + log, "'%s': Attempting to resume from %zu errors @%lu", name(d), + errors.size(), sequence(d) }; @@ -391,11 +397,14 @@ ircd::db::resume(database &d) d.d->Resume() }; + d.errors.clear(); + log::info { - log, "'%s': Resumed normal operation at sequence number %lu.", + log, "'%s': Resumed normal operation at sequence number %lu; cleared %zu errors", name(d), - sequence(d) + sequence(d), + errors.size() }; } @@ -537,6 +546,12 @@ ircd::db::files(const database &cd, return ret; } +const std::vector & +ircd::db::errors(const database &d) +{ + return d.errors; +} + uint64_t ircd::db::sequence(const database &cd) { @@ -2371,15 +2386,17 @@ ircd::db::database::events::OnBackgroundError(rocksdb::BackgroundErrorReason rea rocksdb::Status *const status) noexcept { + assert(d); assert(status); - log::error + + thread_local char buf[1024]; + const string_view str{fmt::sprintf { - rog, "'%s' background %s error in %s :%s", - d->name, + buf, "%s error in %s :%s", reflect(status->severity()), reflect(reason), status->ToString() - }; + }}; // This is a legitimate when we want to use it. If the error is not // suppressed the DB will enter read-only mode and will require a @@ -2389,8 +2406,25 @@ noexcept false }; + const log::facility fac + { + ignore? + log::facility::DERROR: + log::facility::ERROR + }; + + log::logf + { + log, fac, "'%s': %s", d->name, str + }; + if(ignore) + { *status = rocksdb::Status::OK(); + return; + } + + d->errors.emplace_back(str); } void diff --git a/modules/console.cc b/modules/console.cc index 74c0a571b..247ec2753 100644 --- a/modules/console.cc +++ b/modules/console.cc @@ -1352,6 +1352,42 @@ catch(const std::out_of_range &e) return true; } +bool +console_cmd__db__errors(opt &out, const string_view &line) +try +{ + const params param{line, " ", + { + "dbname", + }}; + + const auto dbname + { + param.at("dbname") + }; + + auto &database + { + db::database::get(dbname) + }; + + const auto &errors + { + db::errors(database) + }; + + size_t i(0); + for(const auto &error : errors) + out << std::setw(2) << std::left << (i++) << ':' << error << std::endl; + + return true; +} +catch(const std::out_of_range &e) +{ + out << "No open database by that name" << std::endl; + return true; +} + bool console_cmd__db__ticker(opt &out, const string_view &line) try @@ -2930,6 +2966,15 @@ try } } + if(!c && !errors(d).empty()) + { + size_t i(0); + out << std::endl; + out << "ERRORS (" << errors(d).size() << "): " << std::endl; + for(const auto &error : errors(d)) + out << std::setw(2) << (i++) << ':' << error << std::endl; + } + return true; } catch(const std::out_of_range &e)