mirror of
https://github.com/matrix-construct/construct
synced 2024-06-02 18:18:56 +02:00
ircd::db: Integrate checkpointing with database name and path schema.
This commit is contained in:
Jason Volk
parent
4cb6845ac1
commit
ff21175036
|
|
@ -38,7 +38,7 @@ namespace ircd::db
|
|||
// Control panel
|
||||
void setopt(database &, const string_view &key, const string_view &val);
|
||||
void fdeletions(database &, const bool &enable, const bool &force = false);
|
||||
void checkpoint(database &, const string_view &dir);
|
||||
uint64_t checkpoint(database &);
|
||||
void check(database &);
|
||||
void compact(database &);
|
||||
void sort(database &, const bool &blocking = true);
|
||||
|
|
@ -60,6 +60,7 @@ namespace ircd::db
|
|||
///
|
||||
struct ircd::db::database
|
||||
:std::enable_shared_from_this<database>
|
||||
,instance_list<database>
|
||||
{
|
||||
struct descriptor;
|
||||
struct options;
|
||||
|
|
@ -75,12 +76,10 @@ struct ircd::db::database
|
|||
|
||||
using description = std::vector<descriptor>;
|
||||
|
||||
// central collection of open databases for iteration (non-owning)
|
||||
static std::map<string_view, database *> dbs;
|
||||
|
||||
std::string name;
|
||||
std::string path;
|
||||
uint64_t checkpoint;
|
||||
std::string optstr;
|
||||
std::string path;
|
||||
std::shared_ptr<struct env> env;
|
||||
std::shared_ptr<struct logs> logs;
|
||||
std::shared_ptr<struct stats> stats;
|
||||
|
|
@ -94,8 +93,7 @@ struct ircd::db::database
|
|||
std::vector<std::shared_ptr<column>> columns;
|
||||
std::unique_ptr<rocksdb::DB> d;
|
||||
std::string uuid;
|
||||
std::unique_ptr<rocksdb::Checkpoint> checkpoint;
|
||||
unique_const_iterator<decltype(dbs)> dbs_it;
|
||||
std::unique_ptr<rocksdb::Checkpoint> checkpointer;
|
||||
|
||||
operator std::shared_ptr<database>() { return shared_from_this(); }
|
||||
operator const rocksdb::DB &() const { return *d; }
|
||||
|
|
@ -114,11 +112,16 @@ struct ircd::db::database
|
|||
void operator()(const std::initializer_list<delta> &);
|
||||
void operator()(const delta &);
|
||||
|
||||
database(std::string name,
|
||||
database(const string_view &name,
|
||||
const uint64_t &checkpoint,
|
||||
std::string options,
|
||||
description);
|
||||
|
||||
database(std::string name,
|
||||
database(const string_view &name,
|
||||
std::string options,
|
||||
description);
|
||||
|
||||
database(const string_view &name,
|
||||
std::string options = {});
|
||||
|
||||
database() = default;
|
||||
|
|
@ -126,6 +129,12 @@ struct ircd::db::database
|
|||
database(const database &) = delete;
|
||||
~database() noexcept;
|
||||
|
||||
// Find this instance by name (and checkpoint id) in the instance list
|
||||
static database *get(std::nothrow_t, const string_view &name, const uint64_t &checkpoint);
|
||||
static database *get(std::nothrow_t, const string_view &name); // optionally "name:checkpoint"
|
||||
static database &get(const string_view &name, const uint64_t &checkpoint);
|
||||
static database &get(const string_view &name); // optionally "name:checkpoint"
|
||||
|
||||
// Get this instance from any column.
|
||||
static const database &get(const column &);
|
||||
static database &get(column &);
|
||||
|
|
|
|||
|
|
@ -85,7 +85,14 @@ namespace ircd::db
|
|||
{
|
||||
extern const char *const version;
|
||||
|
||||
std::string path(const std::string &name);
|
||||
// Utils for "name:checkpoint" string amalgam
|
||||
std::string namepoint(const string_view &name, const uint64_t &checkpoint);
|
||||
std::pair<string_view, uint64_t> namepoint(const string_view &name);
|
||||
|
||||
// Generate local filesytem path based on name / name:checkpoint / etc.
|
||||
std::string path(const string_view &name, const uint64_t &checkpoint);
|
||||
std::string path(const string_view &name);
|
||||
|
||||
std::vector<std::string> available();
|
||||
|
||||
string_view reflect(const pos &);
|
||||
|
|
|
|||
263
ircd/db.cc
263
ircd/db.cc
|
|
@ -58,10 +58,6 @@ ircd::db::rog
|
|||
"rdb", 'R'
|
||||
};
|
||||
|
||||
std::map<ircd::string_view, ircd::db::database *>
|
||||
ircd::db::database::dbs
|
||||
{};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// init
|
||||
|
|
@ -191,24 +187,41 @@ ircd::db::check(database &d)
|
|||
/// then be opened as its own database either read-only or read-write.
|
||||
/// Incremental backups and rollbacks can begin from this interface. Note
|
||||
/// this may be an expensive blocking operation.
|
||||
void
|
||||
ircd::db::checkpoint(database &d,
|
||||
const string_view &dir)
|
||||
uint64_t
|
||||
ircd::db::checkpoint(database &d)
|
||||
{
|
||||
if(!d.checkpoint)
|
||||
if(!d.checkpointer)
|
||||
throw error
|
||||
{
|
||||
"Checkpointing is not available for db(%p) '%s", &d, name(d)
|
||||
"Checkpointing is not available for db(%p) '%s",
|
||||
&d,
|
||||
name(d)
|
||||
};
|
||||
|
||||
const auto seqnum
|
||||
{
|
||||
sequence(d)
|
||||
};
|
||||
|
||||
const std::string dir
|
||||
{
|
||||
db::path(name(d), seqnum)
|
||||
};
|
||||
|
||||
throw_on_error
|
||||
{
|
||||
d.checkpoint->CreateCheckpoint(std::string{dir}, 0)
|
||||
d.checkpointer->CreateCheckpoint(dir, 0)
|
||||
};
|
||||
|
||||
log.debug("'%s': Checkpoint to `%s' complete",
|
||||
name(d),
|
||||
dir);
|
||||
log::debug
|
||||
{
|
||||
log, "'%s': Checkpoint at sequence %lu in `%s' complete",
|
||||
name(d),
|
||||
seqnum,
|
||||
dir
|
||||
};
|
||||
|
||||
return seqnum;
|
||||
}
|
||||
|
||||
/// This wraps RocksDB's "File Deletions" which means after RocksDB
|
||||
|
|
@ -376,11 +389,18 @@ ircd::db::name(const database &d)
|
|||
//
|
||||
// database
|
||||
//
|
||||
|
||||
namespace ircd::db
|
||||
{
|
||||
extern const database::description default_description;
|
||||
}
|
||||
|
||||
// Instance list linkage
|
||||
template<>
|
||||
decltype(ircd::util::instance_list<ircd::db::database>::list)
|
||||
ircd::util::instance_list<ircd::db::database>::list
|
||||
{};
|
||||
|
||||
decltype(ircd::db::default_description)
|
||||
ircd::db::default_description
|
||||
{
|
||||
|
|
@ -389,31 +409,120 @@ ircd::db::default_description
|
|||
{ "default" }
|
||||
};
|
||||
|
||||
ircd::db::database::database(std::string name,
|
||||
ircd::db::database &
|
||||
ircd::db::database::get(column &column)
|
||||
{
|
||||
assert(column.d);
|
||||
return *column.d;
|
||||
}
|
||||
|
||||
const ircd::db::database &
|
||||
ircd::db::database::get(const column &column)
|
||||
{
|
||||
assert(column.d);
|
||||
return *column.d;
|
||||
}
|
||||
|
||||
ircd::db::database &
|
||||
ircd::db::database::get(const string_view &name)
|
||||
{
|
||||
const auto pair
|
||||
{
|
||||
namepoint(name)
|
||||
};
|
||||
|
||||
return get(pair.first, pair.second);
|
||||
}
|
||||
|
||||
ircd::db::database &
|
||||
ircd::db::database::get(const string_view &name,
|
||||
const uint64_t &checkpoint)
|
||||
{
|
||||
auto *const &d
|
||||
{
|
||||
get(std::nothrow, name, checkpoint)
|
||||
};
|
||||
|
||||
if(likely(d))
|
||||
return *d;
|
||||
|
||||
throw checkpoint == uint64_t(-1)?
|
||||
std::out_of_range{"No database with that name exists"}:
|
||||
std::out_of_range{"No database with that name at that checkpoint exists"};
|
||||
}
|
||||
|
||||
ircd::db::database *
|
||||
ircd::db::database::get(std::nothrow_t,
|
||||
const string_view &name)
|
||||
{
|
||||
const auto pair
|
||||
{
|
||||
namepoint(name)
|
||||
};
|
||||
|
||||
return get(std::nothrow, pair.first, pair.second);
|
||||
}
|
||||
|
||||
ircd::db::database *
|
||||
ircd::db::database::get(std::nothrow_t,
|
||||
const string_view &name,
|
||||
const uint64_t &checkpoint)
|
||||
{
|
||||
for(auto *const &d : list)
|
||||
if(name == d->name)
|
||||
if(checkpoint == uint64_t(-1) || checkpoint == d->checkpoint)
|
||||
return d;
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//
|
||||
// database::database
|
||||
//
|
||||
|
||||
ircd::db::database::database(const string_view &name,
|
||||
std::string optstr)
|
||||
:database
|
||||
{
|
||||
std::move(name), std::move(optstr), default_description
|
||||
name, std::move(optstr), default_description
|
||||
}
|
||||
{
|
||||
}
|
||||
|
||||
ircd::db::database::database(std::string name,
|
||||
ircd::db::database::database(const string_view &name,
|
||||
std::string optstr,
|
||||
description description)
|
||||
:database
|
||||
{
|
||||
namepoint(name).first, namepoint(name).second, std::move(optstr), std::move(description)
|
||||
}
|
||||
{
|
||||
}
|
||||
|
||||
ircd::db::database::database(const string_view &name,
|
||||
const uint64_t &checkpoint,
|
||||
std::string optstr,
|
||||
description description)
|
||||
try
|
||||
:name
|
||||
{
|
||||
std::move(name)
|
||||
namepoint(name).first
|
||||
}
|
||||
,path
|
||||
,checkpoint
|
||||
{
|
||||
db::path(this->name)
|
||||
// a -1 may have been generated by the db::namepoint() util when the user
|
||||
// supplied just a name without a checkpoint. In the context of database
|
||||
// opening/creation -1 just defaults to 0.
|
||||
checkpoint == uint64_t(-1)? 0 : checkpoint
|
||||
}
|
||||
,optstr
|
||||
{
|
||||
std::move(optstr)
|
||||
}
|
||||
,path
|
||||
{
|
||||
db::path(this->name, this->checkpoint)
|
||||
}
|
||||
,env
|
||||
{
|
||||
std::make_shared<struct env>(this)
|
||||
|
|
@ -647,7 +756,7 @@ try
|
|||
|
||||
return ret;
|
||||
}()}
|
||||
,checkpoint{[this]
|
||||
,checkpointer{[this]
|
||||
{
|
||||
rocksdb::Checkpoint *checkpointer{nullptr};
|
||||
throw_on_error
|
||||
|
|
@ -657,10 +766,6 @@ try
|
|||
|
||||
return checkpointer;
|
||||
}()}
|
||||
,dbs_it
|
||||
{
|
||||
dbs, dbs.emplace(string_view{this->name}, this).first
|
||||
}
|
||||
{
|
||||
#ifdef RB_DEBUG
|
||||
log::notice
|
||||
|
|
@ -700,7 +805,7 @@ noexcept try
|
|||
property<uint64_t>(*this, rocksdb::DB::Properties::kBackgroundErrors));
|
||||
|
||||
flush(*this);
|
||||
this->checkpoint.reset(nullptr);
|
||||
this->checkpointer.reset(nullptr);
|
||||
this->columns.clear();
|
||||
log.debug("'%s': closed columns; synchronizing...",
|
||||
name);
|
||||
|
|
@ -846,20 +951,6 @@ catch(const std::out_of_range &e)
|
|||
id);
|
||||
}
|
||||
|
||||
ircd::db::database &
|
||||
ircd::db::database::get(column &column)
|
||||
{
|
||||
assert(column.d);
|
||||
return *column.d;
|
||||
}
|
||||
|
||||
const ircd::db::database &
|
||||
ircd::db::database::get(const column &column)
|
||||
{
|
||||
assert(column.d);
|
||||
return *column.d;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// database::comparator
|
||||
|
|
@ -5674,16 +5765,98 @@ ircd::db::throw_on_error::throw_on_error(const rocksdb::Status &s)
|
|||
std::vector<std::string>
|
||||
ircd::db::available()
|
||||
{
|
||||
const auto prefix(fs::get(fs::DB));
|
||||
const auto dirs(fs::ls(prefix));
|
||||
return dirs;
|
||||
const auto prefix
|
||||
{
|
||||
fs::get(fs::DB)
|
||||
};
|
||||
|
||||
const auto dirs
|
||||
{
|
||||
fs::ls(prefix)
|
||||
};
|
||||
|
||||
std::vector<std::string> ret;
|
||||
for(const auto &dir : dirs)
|
||||
{
|
||||
const auto name
|
||||
{
|
||||
lstrip(dir, prefix)
|
||||
};
|
||||
|
||||
const auto checkpoints
|
||||
{
|
||||
fs::ls(dir)
|
||||
};
|
||||
|
||||
for(const auto cpdir : checkpoints) try
|
||||
{
|
||||
const auto checkpoint
|
||||
{
|
||||
lstrip(lstrip(cpdir, dir), '/') //TODO: x-platform
|
||||
};
|
||||
|
||||
auto path
|
||||
{
|
||||
db::path(name, lex_cast<uint64_t>(checkpoint))
|
||||
};
|
||||
|
||||
ret.emplace_back(std::move(path));
|
||||
}
|
||||
catch(const bad_lex_cast &e)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string
|
||||
ircd::db::path(const std::string &name)
|
||||
ircd::db::path(const string_view &name)
|
||||
{
|
||||
const auto prefix(fs::get(fs::DB));
|
||||
return fs::make_path({prefix, name});
|
||||
const auto pair
|
||||
{
|
||||
namepoint(name)
|
||||
};
|
||||
|
||||
return path(pair.first, pair.second);
|
||||
}
|
||||
|
||||
std::string
|
||||
ircd::db::path(const string_view &name,
|
||||
const uint64_t &checkpoint)
|
||||
{
|
||||
const auto prefix
|
||||
{
|
||||
fs::get(fs::DB)
|
||||
};
|
||||
|
||||
return fs::make_path(
|
||||
{
|
||||
prefix, name, lex_cast(checkpoint)
|
||||
});
|
||||
}
|
||||
|
||||
std::pair<ircd::string_view, uint64_t>
|
||||
ircd::db::namepoint(const string_view &name_)
|
||||
{
|
||||
const auto s
|
||||
{
|
||||
split(name_, ':')
|
||||
};
|
||||
|
||||
return
|
||||
{
|
||||
s.first,
|
||||
s.second? lex_cast<uint64_t>(s.second) : uint64_t(-1)
|
||||
};
|
||||
}
|
||||
|
||||
std::string
|
||||
ircd::db::namepoint(const string_view &name,
|
||||
const uint64_t &checkpoint)
|
||||
{
|
||||
return std::string{name} + ':' + std::string{lex_cast(checkpoint)};
|
||||
}
|
||||
|
||||
std::pair<ircd::string_view, ircd::string_view>
|
||||
|
|
|
|||
|
|
@ -914,7 +914,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
sync(database);
|
||||
|
|
@ -948,7 +948,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
flush(database, sync);
|
||||
|
|
@ -982,7 +982,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
sort(database, blocking);
|
||||
|
|
@ -1031,7 +1031,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
if(!colname)
|
||||
|
|
@ -1089,7 +1089,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
// Special branch for integer properties that RocksDB aggregates.
|
||||
|
|
@ -1209,7 +1209,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
// Special branch for integer properties that RocksDB aggregates.
|
||||
|
|
@ -1284,7 +1284,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
if(!colname)
|
||||
|
|
@ -1443,7 +1443,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
const auto clear{[&out, &database]
|
||||
|
|
@ -1524,7 +1524,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
// Special branch for DBOptions
|
||||
|
|
@ -1577,7 +1577,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
uint64_t msz;
|
||||
|
|
@ -1611,7 +1611,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(param.at(0))
|
||||
db::database::get(param.at(0))
|
||||
};
|
||||
|
||||
if(!param[1] || param[1] == "*")
|
||||
|
|
@ -1687,7 +1687,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
for_each(database, seqnum, db::seq_closure_bool{[&out, &limit]
|
||||
|
|
@ -1740,7 +1740,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
get(database, seqnum, db::seq_closure{[&out]
|
||||
|
|
@ -1774,20 +1774,18 @@ try
|
|||
token(line, ' ', 0)
|
||||
};
|
||||
|
||||
const auto directory
|
||||
{
|
||||
token(line, ' ', 1)
|
||||
};
|
||||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(dbname)
|
||||
db::database::get(dbname)
|
||||
};
|
||||
|
||||
checkpoint(database, directory);
|
||||
const auto seqnum
|
||||
{
|
||||
checkpoint(database)
|
||||
};
|
||||
|
||||
out << "Checkpoint " << name(database)
|
||||
<< " to " << directory << " complete."
|
||||
<< " at sequence " << seqnum << " complete."
|
||||
<< std::endl;
|
||||
|
||||
return true;
|
||||
|
|
@ -1810,17 +1808,17 @@ console_cmd__db__list(opt &out, const string_view &line)
|
|||
{
|
||||
const auto name
|
||||
{
|
||||
lstrip(path, db::path("/"))
|
||||
replace(lstrip(lstrip(path, fs::DBPATH), '/'), "/", ":")
|
||||
};
|
||||
|
||||
const auto it
|
||||
const auto &d
|
||||
{
|
||||
db::database::dbs.find(name)
|
||||
db::database::get(std::nothrow, name)
|
||||
};
|
||||
|
||||
const auto &light
|
||||
{
|
||||
it != end(db::database::dbs)? "\033[1;42m \033[0m" : " "
|
||||
d? "\033[1;42m \033[0m" : " "
|
||||
};
|
||||
|
||||
out << "[" << light << "]"
|
||||
|
|
@ -1846,7 +1844,7 @@ try
|
|||
|
||||
auto &database
|
||||
{
|
||||
*db::database::dbs.at(param.at(0))
|
||||
db::database::get(param.at(0))
|
||||
};
|
||||
|
||||
out << std::left << std::setw(28) << std::setfill('_') << "UUID "
|
||||
|
|
|
|||
Loading…
Reference in a new issue