MirrorHub/construct
0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-11-25 16:22:35 +01:00
Code Issues Releases Wiki Activity

ircd::db: Develop object-store out of db system.

Browse source
This commit is contained in:
Jason Volk 2017-03-30 15:57:08 -07:00
parent 2ffab411df
commit 16c1326d40
10 changed files with 2141 additions and 849 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

333
include/ircd/db.h
View file

@ -23,305 +23,74 @@
#pragma once
#define HAVE_IRCD_DB_H
namespace rocksdb
{
struct DB;
struct ColumnFamilyHandle;
}
// IRCd Database
//
// Please see db/README.md for documentation.
//
namespace ircd {
namespace db {
// Errors for the database subsystem. The exceptions that use _HIDENAME
// are built from RocksDB errors which already have an info string with
// an included name.
//
IRCD_EXCEPTION(ircd::error, error)
IRCD_EXCEPTION(error, not_found)
IRCD_EXCEPTION(error, corruption)
IRCD_EXCEPTION(error, not_supported)
IRCD_EXCEPTION(error, invalid_argument)
IRCD_EXCEPTION(error, io_error)
IRCD_EXCEPTION(error, merge_in_progress)
IRCD_EXCEPTION(error, incomplete)
IRCD_EXCEPTION(error, shutdown_in_progress)
IRCD_EXCEPTION(error, timed_out)
IRCD_EXCEPTION(error, aborted)
IRCD_EXCEPTION(error, busy)
IRCD_EXCEPTION(error, expired)
IRCD_EXCEPTION(error, try_again)
IRCD_EXCEPTION_HIDENAME(error, corruption)
IRCD_EXCEPTION_HIDENAME(error, not_supported)
IRCD_EXCEPTION_HIDENAME(error, invalid_argument)
IRCD_EXCEPTION_HIDENAME(error, io_error)
IRCD_EXCEPTION_HIDENAME(error, merge_in_progress)
IRCD_EXCEPTION_HIDENAME(error, incomplete)
IRCD_EXCEPTION_HIDENAME(error, shutdown_in_progress)
IRCD_EXCEPTION_HIDENAME(error, timed_out)
IRCD_EXCEPTION_HIDENAME(error, aborted)
IRCD_EXCEPTION_HIDENAME(error, busy)
IRCD_EXCEPTION_HIDENAME(error, expired)
IRCD_EXCEPTION_HIDENAME(error, try_again)
std::string path(const std::string &name);
template<class T>
struct optval
:std::pair<T, ssize_t>
{
optval(const T &key, const ssize_t &val = std::numeric_limits<ssize_t>::min());
};
template<class T> using optlist = std::initializer_list<optval<T>>;
template<class T> bool has_opt(const optlist<T> &, const T &);
template<class T> ssize_t opt_val(const optlist<T> &, const T &);
// Reads may be posted to a separate thread which incurs the time of IO while the calling
// ircd::context yields.
enum class get
{
PIN, // Keep iter data in memory for iter lifetime (good for lots of ++/--)
CACHE, // Update the cache (CACHE is default for non-iterator operations)
NO_CACHE, // Do not update the cache (NO_CACHE is default for iterators)
NO_SNAPSHOT, // Snapshots provide consistent views for iteration.
NO_CHECKSUM, // Integrity of data will be checked unless this is specified
READAHEAD, // Pair with a size in bytes for prefetching additional data
};
struct gopts
:optlist<get>
{
template<class... list> gopts(list&&... l): optlist<get>{std::forward<list>(l)...} {}
};
// Writes usually occur without yielding your context because the DB is write-log oriented.
enum class set
{
FSYNC, // Uses kernel filesystem synchronization after write (slow)
NO_JOURNAL, // Write Ahead Log (WAL) for some crash recovery
MISSING_COLUMNS // No exception thrown when writing to a deleted column family
};
struct sopts
:optlist<set>
{
template<class... list> sopts(list&&... l): optlist<set>{std::forward<list>(l)...} {}
};
enum class opt
{
READ_ONLY, // Opens database without writing permitted.
OPEN_FAST, // Skips a lot of stuff to make opening a handle faster
OPEN_SMALL, // Optimizes the cache hierarchy for < 1GiB databases.
OPEN_BULKLOAD, // Optimizes the handle to accept a large amount of writes at once
NO_CREATE, // A new database may be created (if none found) unless this is specified
NO_EXISTING, // An error is given if database already exists
NO_CHECKSUM, // (paranoid_checks)
NO_MADV_DONTNEED, // Never issue MADV_DONTNEED (on windows turns off all pagecaching!)
NO_MADV_RANDOM, // Skip MADV_RANDOM on database file opening
FALLOCATE, // Allow use of fallocate()
NO_FALLOCATE, // Disallow use fallocate() calls
NO_FDATASYNC, // Flushing is only ever directed by the kernel pagecache
FSYNC, // Use fsync() instead of fdatasync()
MMAP_READS, // mmap() table files for reading
MMAP_WRITES, // mmap() table files for writing (hinders db journal)
STATS_THREAD, // Stats collection etc related to DB threading (thread_track)
STATS_MALLOC, // Stats collection for memory allocation when applicable
LRU_CACHE, // Pair with a size in bytes for the LRU cache size
};
struct opts
:optlist<opt>
{
template<class... list> opts(list&&... l): optlist<opt>{std::forward<list>(l)...} {}
};
enum op
{
GET,
SET,
MERGE,
DELETE,
DELETE_RANGE,
SINGLE_DELETE,
};
struct delta
:std::tuple<op, string_view, string_view>
{
delta(const enum op &op, const string_view &key, const string_view &val = {})
:std::tuple<enum op, string_view, string_view>{op, key, val}
{}
delta(const string_view &key, const string_view &val, const enum op &op = op::SET)
:std::tuple<enum op, string_view, string_view>{op, key, val}
{}
};
using merge_delta = std::pair<string_view, string_view>;
using merge_function = std::function<std::string (const string_view &key, const merge_delta &)>;
using update_function = std::function<std::string (const string_view &key, merge_delta &)>;
struct handle
{
struct const_iterator;
private:
std::shared_ptr<struct meta> meta;
rocksdb::ColumnFamilyHandle *h;
public:
using closure = std::function<void (const string_view &)>;
operator bool() const { return bool(h); }
bool operator!() const { return !h; }
// Iterations
const_iterator lower_bound(const string_view &key, const gopts & = {});
const_iterator upper_bound(const string_view &key, const gopts & = {});
const_iterator cbegin(const gopts & = {});
const_iterator cend(const gopts & = {});
// Perform a get into a closure. This offers a reference to the data with zero-copy.
void operator()(const string_view &key, const closure &func, const gopts & = {});
void operator()(const string_view &key, const gopts &, const closure &func);
// Perform operations in a sequence as a single transaction.
void operator()(const delta &, const sopts & = {});
void operator()(const std::initializer_list<delta> &, const sopts & = {});
void operator()(const op &, const string_view &key, const string_view &val = {}, const sopts & = {});
// Tests if key exists
bool has(const string_view &key, const gopts & = {});
// Get data into your buffer. The signed char buffer is null terminated; the unsigned is not.
size_t get(const string_view &key, char *const &buf, const size_t &max, const gopts & = {});
size_t get(const string_view &key, uint8_t *const &buf, const size_t &max, const gopts & = {});
std::string get(const string_view &key, const gopts & = {});
// Write data to the db
void set(const string_view &key, const string_view &value, const sopts & = {});
void set(const string_view &key, const uint8_t *const &buf, const size_t &size, const sopts & = {});
// Remove data from the db. not_found is never thrown.
void del(const string_view &key, const sopts & = {});
// Flush memory tables to disk (this column only).
void flush(const bool &blocking = false);
// Sync the write log (all columns)
void sync();
handle(const std::string &name,
const std::string &column = "default",
const opts & = {},
merge_function = {});
handle();
handle(handle &&) noexcept;
handle &operator=(handle &&) noexcept;
~handle() noexcept;
};
struct handle::const_iterator
{
using key_type = string_view;
using mapped_type = string_view;
using value_type = std::pair<key_type, mapped_type>;
using pointer = value_type *;
using reference = value_type &;
using difference_type = size_t;
using iterator_category = std::bidirectional_iterator_tag;
private:
struct state;
friend class handle;
std::unique_ptr<struct state> state;
mutable value_type val;
const_iterator(std::unique_ptr<struct state> &&);
public:
operator bool() const;
bool operator!() const;
bool operator<(const const_iterator &) const;
bool operator>(const const_iterator &) const;
bool operator==(const const_iterator &) const;
bool operator!=(const const_iterator &) const;
bool operator<=(const const_iterator &) const;
bool operator>=(const const_iterator &) const;
const value_type *operator->() const;
const value_type &operator*() const;
const_iterator &operator++();
const_iterator &operator--();
const_iterator() = default;
const_iterator(const_iterator &&) noexcept;
const_iterator(const const_iterator &) = delete;
~const_iterator() noexcept;
};
handle::const_iterator begin(handle &);
handle::const_iterator end(handle &);
struct init
{
init();
~init() noexcept;
};
// db subsystem has its own SNOMASK'ed logging facility.
// db subsystem has its own logging facility
extern struct log::log log;
} // namespace db
} // namespace ircd
// These are forward declarations to objects we may carry a pointer to.
// Users of ircd::db should not be dealing with these types.
namespace rocksdb
{
struct DB;
struct Options;
struct DBOptions;
struct ColumnFamilyOptions;
struct PlainTableOptions;
struct BlockBasedTableOptions;
struct Cache;
struct Iterator;
struct ColumnFamilyHandle;
struct Snapshot;
}
#include "db/opts.h"
#include "db/delta.h"
#include "db/database.h"
#include "db/column.h"
#include "db/const_iterator.h"
#include "db/row.h"
namespace ircd {
namespace db {
namespace json {
std::string merge_operator(const string_view &, const std::pair<string_view, string_view> &);
std::string path(const std::string &name);
std::vector<std::string> available();
struct obj
:handle
{
obj(const std::string &name,
const std::string &column = "default",
const opts &opts = {})
:handle{name, column, opts, merge_operator}
{}
};
} // namespace json
} // namespace db
} // namespace ircd
inline ircd::db::handle::const_iterator
ircd::db::end(handle &handle)
{
return handle.cend();
}
namespace ircd {
namespace db {
inline ircd::db::handle::const_iterator
ircd::db::begin(handle &handle)
{
return handle.cbegin();
}
std::string merge_operator(const string_view &, const std::pair<string_view, string_view> &);
template<class T>
ssize_t
ircd::db::opt_val(const optlist<T> &list,
const T &opt)
{
for(const auto &p : list)
if(p.first == opt)
return p.second;
return std::numeric_limits<ssize_t>::min();
}
template<class T>
bool
ircd::db::has_opt(const optlist<T> &list,
const T &opt)
{
for(const auto &p : list)
if(p.first == opt)
return true;
return false;
}
template<class T>
ircd::db::optval<T>::optval(const T &key,
const ssize_t &val)
:std::pair<T, ssize_t>{key, val}
{
}
} // namespace db
} // namespace ircd

53
include/ircd/db/README.md Normal file
View file

@ -0,0 +1,53 @@
IRCd Database
IRCd's database is presented here primarily as a persistent Object store.
In other words, the structure presented by the database can be represented
with JSON. This is built from the primitives of `column`s, `row`s and `cell`s.
Columns:
While a simple key-value store could naively store a JSON document as a textual
value, we provide additional structure schematized before opening a database:
Every member of a JSON object is a `column` in this database. To address members
within nested objects, we specify a `column` with a "foo.bar.baz" path syntax. This
puts all columns at the same level in our code, even though they may represent
deeply nested values.
Rows:
Since `columns` are technically independent key-value stores (they have their own
index), when an index key is the same between columns we call this a `row`. For basic
object storage the schema is such that we use the same keys between all columns. For
example, an index would be a username in a user database. The user database itself
takes the form of a single JSON object and any member lookup happens on a user's row.
Cells:
A `cell` is a single value in a `column` indexed by a key that should be able to form
a `row` between columns. Consider the following near-json expression:
users["root"] = {"password", "foobar"};
In the users database, we find the `column` "password" and the `row` for "root" and
set that `cell` to "foobar"
Consider these expressions for objects at some depth:
users["root"] = {"password.plaintext", "foobar"};
users["root"] = {"password", {"plaintext, "foobar"}};
The column is always found as "password.plaintext". We find it (and can iterate its members
if it were an object) by string-manipulating these full paths which all sit in a single map
and are always open, even if the cell is empty for some row.
Important notes:
!!!
The database system is plugged into the userspace context system to facilitate IO. This means
that an expensive database call (mostly on the read side) that has to do disk IO will suspend
your userspace context. Remember that when your userspace context resumes on the other side
of the call, the state of IRCd and even the database itself may have changed. We have a suite
of tools to mitigate this.
!!!
* While the database schema is modifiable at runtime (we can add and remove columns on
the fly) the database is very picky about opening the exact same way it last closed.
This means, for now, we have the full object schema explicitly specified when the DB
is first opened. All columns exist for the lifetime of the DB, whether or not you have
a handle to them.

155
include/ircd/db/column.h Normal file
View file

@ -0,0 +1,155 @@
/*
* Copyright (C) 2016 Charybdis Development Team
* Copyright (C) 2016 Jason Volk <jason@zemos.net>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice is present in all copies.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#pragma once
#define HAVE_IRCD_DB_COLUMN_H
//
// [GET] may be posted to a separate thread which incurs the time of IO while the calling
// ircd::context yields.
//
// [SET] usually occur without yielding your context because the DB is write-log oriented.
//
namespace ircd {
namespace db {
enum class set
{
FSYNC, // Uses kernel filesystem synchronization after write (slow)
NO_JOURNAL, // Write Ahead Log (WAL) for some crash recovery
MISSING_COLUMNS // No exception thrown when writing to a deleted column family
};
struct sopts
:optlist<set>
{
template<class... list> sopts(list&&... l): optlist<set>{std::forward<list>(l)...} {}
};
enum class get
{
PIN, // Keep iter data in memory for iter lifetime (good for lots of ++/--)
CACHE, // Update the cache (CACHE is default for non-iterator operations)
NO_CACHE, // Do not update the cache (NO_CACHE is default for iterators)
NO_SNAPSHOT, // This iterator will have the latest data (tailing)
NO_CHECKSUM, // Integrity of data will be checked unless this is specified
READAHEAD, // Pair with a size in bytes for prefetching additional data
};
struct gopts
:optlist<get>
{
database::snapshot snapshot;
template<class... list> gopts(list&&... l): optlist<get>{std::forward<list>(l)...} {}
};
// Columns add the ability to run multiple LevelDB's in synchrony under the same database
// (directory). Each column is a fully distinct key/value store; they are merely joined
// for consistency.
//
struct column
{
struct const_iterator;
using key_type = string_view;
using mapped_type = string_view;
using value_type = std::pair<key_type, mapped_type>;
using pointer = value_type *;
using reference = value_type &;
using difference_type = size_t;
using iterator = const_iterator;
protected:
using ColumnFamilyHandle = rocksdb::ColumnFamilyHandle;
std::shared_ptr<database> d;
database::column *c;
public:
operator const database &() const { return *d; }
operator const database::column &() const { return *c; }
operator std::shared_ptr<database>() const { return d; }
operator database &() { return *d; }
operator database::column &() { return *c; }
operator bool() const { return bool(d); }
bool operator!() const { return !d; }
// [GET] Iterations
const_iterator cbegin(const gopts & = {});
const_iterator cend(const gopts & = {});
const_iterator begin(const gopts & = {});
const_iterator end(const gopts & = {});
const_iterator find(const string_view &key, const gopts & = {});
const_iterator lower_bound(const string_view &key, const gopts & = {});
const_iterator upper_bound(const string_view &key, const gopts & = {});
// [GET] Tests if key exists
bool has(const string_view &key, const gopts & = {});
// [GET] Perform a get into a closure. This offers a reference to the data with zero-copy.
void operator()(const string_view &key, const view_closure &func, const gopts & = {});
void operator()(const string_view &key, const gopts &, const view_closure &func);
// [SET] Perform operations in a sequence as a single transaction.
void operator()(const delta &, const sopts & = {});
void operator()(const std::initializer_list<delta> &, const sopts & = {});
void operator()(const op &, const string_view &key, const string_view &val = {}, const sopts & = {});
// Flush memory tables to disk (this column only).
void flush(const bool &blocking = false);
column(std::shared_ptr<database>, database::column &);
column(database &, database::column &);
column(database &, const string_view &column);
column(database::column &);
column() = default;
};
// Get property data of a db column (column).
// R can optionally be uint64_t for some values.
template<class R = std::string> R property(column &, const string_view &name);
template<> std::string property(column &, const string_view &name);
template<> uint64_t property(column &, const string_view &name);
// Information about a column (column)
const std::string &name(const column &);
size_t file_count(column &);
size_t bytes(column &);
// [GET] Convenience functions to copy data into your buffer.
// The signed char buffer is null terminated; the unsigned is not.
size_t read(column &, const string_view &key, uint8_t *const &buf, const size_t &max, const gopts & = {});
string_view read(column &, const string_view &key, char *const &buf, const size_t &max, const gopts & = {});
std::string read(column &, const string_view &key, const gopts & = {});
// [SET] Write data to the db
void write(column &, const string_view &key, const string_view &value, const sopts & = {});
void write(column &, const string_view &key, const uint8_t *const &buf, const size_t &size, const sopts & = {});
// [SET] Remove data from the db. not_found is never thrown.
void del(column &, const string_view &key, const sopts & = {});
} // namespace db
} // namespace ircd

83
include/ircd/db/const_iterator.h Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright (C) 2016 Charybdis Development Team
* Copyright (C) 2016 Jason Volk <jason@zemos.net>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice is present in all copies.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#pragma once
#define HAVE_IRCD_DB_CONST_ITERATOR_H
namespace ircd {
namespace db {
struct column::const_iterator
{
struct state;
using key_type = string_view;
using mapped_type = string_view;
using value_type = std::pair<key_type, mapped_type>;
using pointer = value_type *;
using reference = value_type &;
using difference_type = size_t;
using iterator_category = std::bidirectional_iterator_tag;
private:
gopts opts;
database::column *c;
std::unique_ptr<rocksdb::Iterator> it;
mutable value_type val;
friend class column;
const_iterator(database::column &, std::unique_ptr<rocksdb::Iterator> &&, gopts = {});
public:
operator const database::column &() const { return *c; }
operator const database::snapshot &() const { return opts.snapshot; }
explicit operator const gopts &() const { return opts; }
operator database::column &() { return *c; }
explicit operator database::snapshot &() { return opts.snapshot; }
operator bool() const;
bool operator!() const;
bool operator<(const const_iterator &) const;
bool operator>(const const_iterator &) const;
bool operator==(const const_iterator &) const;
bool operator!=(const const_iterator &) const;
bool operator<=(const const_iterator &) const;
bool operator>=(const const_iterator &) const;
const value_type *operator->() const;
const value_type &operator*() const;
const_iterator &operator++();
const_iterator &operator--();
const_iterator();
const_iterator(const_iterator &&) noexcept;
const_iterator &operator=(const_iterator &&) noexcept;
~const_iterator() noexcept;
template<class pos> friend void seek(column::const_iterator &, const pos &);
friend void seek(column::const_iterator &, const string_view &key);
};
} // namespace db
} // namespace ircd

160
include/ircd/db/database.h Normal file
View file

@ -0,0 +1,160 @@
/*
* Copyright (C) 2016 Charybdis Development Team
* Copyright (C) 2016 Jason Volk <jason@zemos.net>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice is present in all copies.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#pragma once
#define HAVE_IRCD_DB_DATABASE_H
namespace ircd {
namespace db {
struct database
:std::enable_shared_from_this<struct database>
{
struct options;
struct events;
struct stats;
struct logs;
struct mergeop;
struct snapshot;
struct comparator;
struct column;
struct descriptor
{
using typing = std::pair<std::type_index, std::type_index>;
std::string name;
typing type { typeid(string_view), typeid(string_view) };
std::string options {};
db::comparator cmp {};
};
static std::map<string_view, database *> dbs; // open databases
std::string name;
std::string path;
std::shared_ptr<struct logs> logs;
std::shared_ptr<struct stats> stats;
std::shared_ptr<struct events> events;
std::shared_ptr<struct mergeop> mergeop;
std::shared_ptr<rocksdb::Cache> cache;
std::map<string_view, std::shared_ptr<column>> columns;
custom_ptr<rocksdb::DB> d;
public:
operator const rocksdb::DB &() const { return *d; }
operator rocksdb::DB &() { return *d; }
const column &operator[](const string_view &) const;
column &operator[](const string_view &);
database(const std::string &name,
const std::string &options = {},
std::initializer_list<descriptor> = {});
database() = default;
database(database &&) = delete;
database(const database &) = delete;
~database() noexcept;
};
// options <-> string
struct database::options
:std::string
{
struct map;
// Output of options structures from this string
explicit operator rocksdb::Options() const;
operator rocksdb::DBOptions() const;
operator rocksdb::ColumnFamilyOptions() const;
operator rocksdb::PlainTableOptions() const;
operator rocksdb::BlockBasedTableOptions() const;
// Input of options structures output to this string
explicit options(const rocksdb::ColumnFamilyOptions &);
explicit options(const rocksdb::DBOptions &);
explicit options(const database::column &);
explicit options(const database &);
// Input of options string from user
options(std::string string)
:std::string{std::move(string)}
{}
};
// options <-> map
struct database::options::map
:std::unordered_map<std::string, std::string>
{
// Output of options structures from map
operator rocksdb::DBOptions() const;
operator rocksdb::ColumnFamilyOptions() const;
operator rocksdb::PlainTableOptions() const;
operator rocksdb::BlockBasedTableOptions() const;
// Convert option string to map
map(const options &);
// Input of options map from user
map(std::unordered_map<std::string, std::string> m)
:std::unordered_map<std::string, std::string>{std::move(m)}
{}
};
struct database::snapshot
{
std::weak_ptr<database> d;
std::shared_ptr<const rocksdb::Snapshot> s;
public:
operator const database &() const { return *d.lock(); }
operator const rocksdb::Snapshot *() const { return s.get(); }
operator database &() { return *d.lock(); }
operator bool() const { return bool(s); }
bool operator !() const { return !s; }
snapshot(database &);
snapshot() = default;
~snapshot() noexcept;
};
// Get property data from all columns in DB. Only integer properties supported.
template<class R = uint64_t> R property(database &, const string_view &name);
template<> uint64_t property(database &, const string_view &name);
const std::string &name(const database::column &);
uint32_t id(const database::column &);
void drop(database::column &); // Request to erase column from db
uint64_t sequence(const database::snapshot &); // Sequence of a snapshot
uint64_t sequence(const database &); // Latest sequence number
void sync(database &); // Sync the write log (all columns)
} // namespace db
using database = db::database;
} // namespace ircd

64
include/ircd/db/delta.h Normal file
View file

@ -0,0 +1,64 @@
/*
* Copyright (C) 2016 Charybdis Development Team
* Copyright (C) 2016 Jason Volk <jason@zemos.net>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice is present in all copies.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#pragma once
#define HAVE_IRCD_DB_DELTA_H
namespace ircd {
namespace db {
enum op
{
GET,
SET,
MERGE,
DELETE,
DELETE_RANGE,
SINGLE_DELETE,
};
struct delta
:std::tuple<op, string_view, string_view>
{
delta(const enum op &op, const string_view &key, const string_view &val = {})
:std::tuple<enum op, string_view, string_view>{op, key, val}
{}
delta(const string_view &key, const string_view &val, const enum op &op = op::SET)
:std::tuple<enum op, string_view, string_view>{op, key, val}
{}
};
using merge_delta = std::pair<string_view, string_view>;
using merge_closure = std::function<std::string (const string_view &key, const merge_delta &)>;
using update_closure = std::function<std::string (const string_view &key, merge_delta &)>;
using view_closure = std::function<void (const string_view &)>;
struct comparator
{
std::string name;
std::function<bool (const string_view &, const string_view &)> less;
std::function<bool (const string_view &, const string_view &)> equal;
};
} // namespace db
} // namespace ircd

271
include/ircd/db/object.h Normal file
View file

@ -0,0 +1,271 @@
/*
* Copyright (C) 2016 Charybdis Development Team
* Copyright (C) 2016 Jason Volk <jason@zemos.net>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice is present in all copies.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#pragma once
#define HAVE_IRCD_DB_OBJECT_H
namespace ircd {
namespace db {
template<database *const &d>
struct transaction
{
string_view index;
database::snapshot snapshot;
//something transaction;
transaction(const string_view &index = {})
:index{index}
,snapshot{*d}
{}
};
template<class T,
database *const &d>
struct value
{
};
template<database *const &d>
struct value<void, d>
{
mutable column h;
transaction<d> *t;
value(const string_view &name,
transaction<d> &t)
:h{!name.empty()? column{*d, name} : column{}}
,t{&t}
{}
value()
:t{nullptr}
{}
};
template<database *const &d,
const char *const &prefix>
struct object
{
struct iterator;
using key_type = string_view;
using mapped_type = value<void, d>;
using value_type = std::pair<key_type, mapped_type>;
using pointer = value_type *;
using reference = value_type &;
using size_type = size_t;
using difference_type = ptrdiff_t;
transaction<d> *t;
iterator begin();
iterator end();
object(transaction<d> &t)
:t{&t}
{}
object()
:t{nullptr}
{}
};
template<database *const &d,
const char *const &prefix>
struct object<d, prefix>::iterator
{
using key_type = string_view;
using mapped_type = value<void, d>;
using value_type = std::pair<key_type, mapped_type>;
using pointer = value_type *;
using reference = value_type &;
using size_type = size_t;
using difference_type = ptrdiff_t;
friend class object<d, prefix>;
protected:
transaction<d> *t;
decltype(database::columns)::iterator it;
value_type last;
value_type val;
void seek_next();
public:
const value_type *operator->() const { return &val; }
const value_type &operator*() const { return *operator->(); }
bool operator==(const iterator &o) const { return it == o.it; }
bool operator!=(const iterator &o) const { return it != o.it; }
bool operator<(const iterator &o) const { return it < o.it; }
iterator &operator++()
{
++it;
seek_next();
return *this;
}
iterator(transaction<d> &t)
:t{&t}
{}
iterator()
:t{nullptr}
{}
};
template<database *const &d,
const char *const &prefix>
typename object<d, prefix>::iterator
object<d, prefix>::end()
{
iterator ret{};
ret.it = std::end(d->columns);
return ret;
}
template<database *const &d,
const char *const &prefix>
typename object<d, prefix>::iterator
object<d, prefix>::begin()
{
iterator ret{*t};
ret.it = std::begin(d->columns);
ret.seek_next();
return ret;
}
template<database *const &d,
const char *const &prefix>
void
object<d, prefix>::iterator::seek_next()
{
const auto ptc(tokens_count(prefix, "."));
while(it != std::end(d->columns))
{
const auto &pair(*it);
if(!startswith(pair.first, prefix))
{
++it;
continue;
}
const auto ktc(tokens_count(pair.first, "."));
if(ktc != ptc + 1)
{
const auto com(std::min(tokens_count(last.first, "."), ptc + 1));
if(!com || token(last.first, ".", com - 1) == token(pair.first, ".", com - 1))
{
++it;
continue;
}
}
bool bad(false);
const auto com(std::min(ktc, ptc));
if(com)
for(size_t i(0); i < com - 1 && !bad; i++)
if(token(prefix, ".", i) != token(pair.first, ".", i))
bad = true;
if(bad)
{
++it;
continue;
}
val.first = pair.first;
last.first = pair.first;
val.first = lstrip(val.first, prefix);
val.first = lstrip(val.first, '.');
val.first = split(val.first, '.').first;
val.second = value<void, d>{pair.first, *t};
break;
}
}
/*
template<database *const &d>
struct value<string_view ,d>
:value<void, d>
{
// hold iterator
operator string_view() const
{
std::cout << "read [" << this->name << "] " << std::endl;
return {};
}
value &operator=(const string_view &val)
{
std::cout << "write [" << this->name << "] " << val << std::endl;
return *this;
}
value(const string_view &name)
:value<void, d>{name}
{}
friend std::ostream &operator<<(std::ostream &s, const value<string_view, d> &v)
{
s << string_view{v};
return s;
}
};
*/
template<database *const &d>
struct value<int64_t, d>
:value<void, d>
{
int64_t def;
operator int64_t() const try
{
const auto val(read(this->h, this->t->index));
return lex_cast<int64_t>(val);
}
catch(const not_found &e)
{
return def;
}
value &operator=(const int64_t &val)
{
write(this->h, this->t->index, lex_cast(val));
return *this;
}
value(const string_view &name,
transaction<d> &t,
const int64_t &def = 0)
:value<void, d>{name, t}
,def{def}
{}
};
} // namespace db
} // namespace ircd

72
include/ircd/db/opts.h Normal file
View file

@ -0,0 +1,72 @@
/*
* Copyright (C) 2016 Charybdis Development Team
* Copyright (C) 2016 Jason Volk <jason@zemos.net>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice is present in all copies.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#pragma once
#define HAVE_IRCD_DB_OPTS_H
namespace ircd {
namespace db {
template<class T>
struct optval
:std::pair<T, ssize_t>
{
optval(const T &key, const ssize_t &val = std::numeric_limits<ssize_t>::min());
};
template<class T> using optlist = std::initializer_list<optval<T>>;
template<class T> bool has_opt(const optlist<T> &, const T &);
template<class T> ssize_t opt_val(const optlist<T> &, const T &);
} // namespace db
} // namespace ircd
template<class T>
ssize_t
ircd::db::opt_val(const optlist<T> &list,
const T &opt)
{
for(const auto &p : list)
if(p.first == opt)
return p.second;
return std::numeric_limits<ssize_t>::min();
}
template<class T>
bool
ircd::db::has_opt(const optlist<T> &list,
const T &opt)
{
for(const auto &p : list)
if(p.first == opt)
return true;
return false;
}
template<class T>
ircd::db::optval<T>::optval(const T &key,
const ssize_t &val)
:std::pair<T, ssize_t>{key, val}
{
}

66
include/ircd/db/row.h Normal file
View file

@ -0,0 +1,66 @@
/*
* Copyright (C) 2016 Charybdis Development Team
* Copyright (C) 2016 Jason Volk <jason@zemos.net>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice is present in all copies.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#pragma once
#define HAVE_IRCD_DB_ROW_H
namespace ircd {
namespace db {
struct cell
{
explicit
cell(database &,
column,
const string_view &key,
gopts opts = {});
};
struct row
{
using key_type = column;
using mapped_type = std::unique_ptr<rocksdb::Iterator>;
using value_type = std::pair<key_type, mapped_type>;
gopts opts;
std::vector<value_type> its;
template<class pos> friend void seek(row &, const pos &);
friend void seek(row &, const string_view &key);
public:
auto begin() const { return std::begin(its); }
auto end() const { return std::end(its); }
auto begin() { return std::begin(its); }
auto end() { return std::end(its); }
string_view operator[](const string_view &column);
row(database &, const string_view &key = {}, gopts = {});
row() = default;
row(row &&) noexcept;
row &operator=(row &&) noexcept;
~row() noexcept;
};
} // namespace db
} // namespace ircd

1733
ircd/db.cc
View file

File diff suppressed because it is too large Load diff