From e09f272592c0611dbfd37518ce880c604d3d39dd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 23 Aug 2017 15:37:06 -0600 Subject: [PATCH] ircd::db: Improve/comment DB subsystem; Add version identification. --- include/ircd/db.h | 2 + include/ircd/db/column.h | 25 ++++++++++--- include/ircd/db/database.h | 76 +++++++++++++++++++++++--------------- ircd/db.cc | 34 +++++++++++++++-- rocksdb | 2 +- 5 files changed, 100 insertions(+), 39 deletions(-) diff --git a/include/ircd/db.h b/include/ircd/db.h index bbf9879ff..6fcc860b1 100644 --- a/include/ircd/db.h +++ b/include/ircd/db.h @@ -85,6 +85,8 @@ namespace rocksdb namespace ircd { namespace db { +extern const char *const version; + rocksdb::Slice slice(const string_view &); string_view slice(const rocksdb::Slice &); diff --git a/include/ircd/db/column.h b/include/ircd/db/column.h index aade76706..60f22a81a 100644 --- a/include/ircd/db/column.h +++ b/include/ircd/db/column.h @@ -26,14 +26,27 @@ namespace ircd { namespace db { -// Columns add the ability to run multiple LevelDB's in synchrony under the same database -// (directory). Each column is a fully distinct key/value store; they are merely joined -// for consistency. +// Columns add the ability to run multiple LevelDB's in synchrony under the same +// database (directory). Each column is a fully distinct key/value store; they +// are merely joined for consistency and possible performance advantages for +// concurrent multi-column lookups of the same key. // -// [GET] may be posted to a separate thread which incurs the time of IO while the calling -// ircd::context yields. +// This class is a handle to the real column instance `database::column` because the +// real column instance has to have a lifetime congruent to the open database. But +// that makes this object easier to work with, pass around, and construct. It will +// find the real `database::column` at any time. // -// [SET] usually occur without yielding your context because the DB is write-log oriented. +// [GET] If the data is not cached, your ircd::context will yield. Note that the +// request may be posted to a separate thread which incurs the time of IO. This is +// because RocksDB has minimalist origins and is not yet asynchronous. +// +// + In the future, your ircd::context will still yield but the internals here will +// interleave pending contexts. If RocksDB is clever enough to expose actual file +// descriptors or something we can interleave on, similar to the pgsql API, we can +// remove the IO/offload thread as well. +// +// [SET] usually occur without yielding your context because the DB is oriented +// around write-log appending so it can deal with heavier tasks later in background. // struct column { diff --git a/include/ircd/db/database.h b/include/ircd/db/database.h index 3bcaaebdc..21023f8fb 100644 --- a/include/ircd/db/database.h +++ b/include/ircd/db/database.h @@ -26,43 +26,44 @@ namespace ircd { namespace db { +// Database instance +// +// There can be only one instance of this class for each database, so it is +// always shared and must be make_shared(). The database is open when an +// instance is constructed and closed when the instance destructs. +// +// The construction must have the same consistent descriptor set used every +// time otherwise bad things happen. +// +// The instance registers and deregisters itself in a global set of open +// databases and can be found that way if necessary. +// struct database :std::enable_shared_from_this { + struct descriptor; struct options; - struct events; - struct stats; - struct logs; - struct mergeop; + struct events; + struct stats; + struct logs; + struct mergeop; struct snapshot; struct comparator; struct column; + using description = std::initializer_list; - static std::map dbs; // open databases + // central collection of open databases for iteration (non-owning) + static std::map dbs; - std::string name; - std::string path; - std::shared_ptr logs; - std::shared_ptr stats; - std::shared_ptr events; - std::shared_ptr mergeop; - std::shared_ptr cache; + std::string name; + std::string path; + std::shared_ptr logs; + std::shared_ptr stats; + std::shared_ptr events; + std::shared_ptr mergeop; + std::shared_ptr cache; std::map> columns; - custom_ptr d; - - public: - struct descriptor - { - using typing = std::pair; - - std::string name; - std::string explain; - typing type { typeid(string_view), typeid(string_view) }; - std::string options {}; - db::comparator cmp {}; - }; - - using description = std::initializer_list; + custom_ptr d; operator std::shared_ptr() { return shared_from_this(); } operator const rocksdb::DB &() const { return *d; } @@ -72,18 +73,35 @@ struct database column &operator[](const string_view &); database(const std::string &name, - const std::string &options = {}, - description = {}); + const std::string &options, + description); + + database(const std::string &name, + const std::string &options = {}); database() = default; database(database &&) = delete; database(const database &) = delete; ~database() noexcept; + // Get this instance from any column. static const database &get(const column &); static database &get(column &); }; +// Descriptor of a column when opening database. Database must be opened with +// a consistent set of descriptors describing what will be found upon opening. +struct database::descriptor +{ + using typing = std::pair; + + std::string name; + std::string explain; + typing type { typeid(string_view), typeid(string_view) }; + std::string options {}; + db::comparator cmp {}; +}; + // options <-> string struct database::options :std::string diff --git a/ircd/db.cc b/ircd/db.cc index 7dce2257e..7f549979b 100644 --- a/ircd/db.cc +++ b/ircd/db.cc @@ -20,6 +20,7 @@ * */ +#include #include #include #include @@ -34,11 +35,13 @@ namespace ircd { namespace db { +// Dedicated logging facility for the database subsystem struct log::log log { "db", 'D' // Database subsystem takes SNOMASK +D }; +// Functor to wrap calls made to the rocksdb API to check for errors. struct throw_on_error { throw_on_error(const rocksdb::Status & = rocksdb::Status::OK()); @@ -226,6 +229,20 @@ database::dbs } // namespace db } // namespace ircd +static char ircd_db_version[64]; +const char *const ircd::db::version(ircd_db_version); + +// Renders a version string from the defines included here. +__attribute__((constructor)) +static void +version_init() +{ + snprintf(ircd_db_version, sizeof(ircd_db_version), "%d.%d.%d", + ROCKSDB_MAJOR, + ROCKSDB_MINOR, + ROCKSDB_PATCH); +} + /////////////////////////////////////////////////////////////////////////////// // // database @@ -274,6 +291,15 @@ ircd::db::shared_from(const database::column &column) // database // +ircd::db::database::database(const std::string &name, + const std::string &optstr) +:database +{ + name, optstr, {} +} +{ +} + ircd::db::database::database(const std::string &name, const std::string &optstr, description description) @@ -302,13 +328,15 @@ try { std::make_shared(this) } -,cache{[this]() -> std::shared_ptr +,cache{[this] +() -> std::shared_ptr { //TODO: XXX const auto lru_cache_size{64_MiB}; return rocksdb::NewLRUCache(lru_cache_size); }()} -,d{[this, &description, &optstr]() -> custom_ptr +,d{[this, &description, &optstr] +() -> custom_ptr { rocksdb::DBOptions opts { @@ -769,7 +797,7 @@ ircd::db::database::logs::Logv(const rocksdb::InfoLogLevel level, char buf[1024]; const auto len { - std::vsnprintf(buf, sizeof(buf), fmt, ap) + vsnprintf(buf, sizeof(buf), fmt, ap) }; const auto str diff --git a/rocksdb b/rocksdb index 0a1bd9c50..7f5f0e0c0 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 0a1bd9c509786b9ab6365e263b867c1bbdca6cc7 +Subproject commit 7f5f0e0c03d20f84653a991de324810d16a33822