0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-11-29 10:12:39 +01:00

ircd:Ⓜ️ Checkpoint single-level/splitless state query.

This commit is contained in:
Jason Volk 2018-01-28 08:45:09 -08:00
parent 6bc6d9f8a1
commit 4cfd41327a
2 changed files with 394 additions and 124 deletions

View file

@ -23,15 +23,35 @@ namespace ircd::m::dbs
size_t keys(const node &); size_t keys(const node &);
size_t vals(const node &); size_t vals(const node &);
size_t children(const node &);
json::array key(const node &, const size_t &); json::array key(const node &, const size_t &);
string_view val(const node &, const size_t &); string_view val(const node &, const size_t &);
int keycmp(const json::array &a, const json::array &b);
json::array make_key(const mutable_buffer &out, const string_view &type, const string_view &state_key); json::array make_key(const mutable_buffer &out, const string_view &type, const string_view &state_key);
size_t find(const node &, const json::array &key); size_t find(const node &, const json::array &key);
size_t find(const node &, const string_view &type, const string_view &state_key); size_t find(const node &, const string_view &type, const string_view &state_key);
json::object make_node(const mutable_buffer &out, const std::initializer_list<json::array> &keys, const std::initializer_list<string_view> &vals); json::object make_node(const mutable_buffer &out, const json::array *const &keys, const size_t &kn, const string_view *const &vals, const size_t &vn);
string_view make_node(db::iov &txn, const mutable_buffer &hash, const std::initializer_list<json::array> &keys, const std::initializer_list<string_view> &vals); json::object make_into(const mutable_buffer &out, const node &old, const size_t &pos, const json::array &key, const string_view &val);
json::object get_node(db::column &, const mutable_buffer &buf, const string_view &id);
json::object get_node(const mutable_buffer &buf, const string_view &id); using id_closure = std::function<void (const string_view &)>;
using node_closure = std::function<void (const json::object &)>;
void get_node(db::column &, const string_view &id, const node_closure &);
void get_node(const string_view &id, const node_closure &);
string_view set_node(db::iov &txn, const mutable_buffer &hash, const json::array *const &keys, const size_t &kn, const string_view *const &vals, const size_t &vn);
string_view set_into(db::iov &txn, const mutable_buffer &hash, const node &old, const size_t &pos, const json::array &key, const string_view &val);
void get_head(db::column &, const id::room &, const id_closure &);
void get_head(const id::room &, const id_closure &);
string_view get_head(const id::room &, const mutable_buffer &buf);
void set_head(db::iov &txn, const id::room &, const string_view &head);
void get_value(const string_view &head, const json::array &key, const id_closure &);
void get_value(const string_view &head, const string_view &type, const string_view &state_key, const id_closure &);
void get_value__room(const id::room &, const string_view &type, const string_view &state_key, const id_closure &);
void insert(db::iov &txn, const id::room &, const json::array &key, const id::event &);
void insert(db::iov &txn, const id::room &, const string_view &type, const string_view &state_key, const id::event &);
void append_indexes(const event &, db::iov &iov); void append_indexes(const event &, db::iov &iov);
void append_nodes(const event &, db::iov &iov); void append_nodes(const event &, db::iov &iov);

View file

@ -10,6 +10,34 @@
#include <ircd/m/m.h> #include <ircd/m/m.h>
namespace ircd::m
{
struct indexer;
extern std::set<std::shared_ptr<indexer>> indexers;
extern const std::unique_ptr<indexer> indexer_origin_joined;
}
struct ircd::m::indexer
{
//TODO: collapse
struct concat;
struct concat_s; //TODO: special
struct concat_v;
struct concat_2v;
struct concat_3vs; //TODO: special
std::string name;
virtual void operator()(const event &, db::iov &iov, const db::op &op) const {}
indexer(std::string name)
:name{std::move(name)}
{}
virtual ~indexer() noexcept = default;
};
decltype(ircd::m::dbs::databases) decltype(ircd::m::dbs::databases)
ircd::m::dbs::databases ircd::m::dbs::databases
{}; {};
@ -76,8 +104,8 @@ ircd::m::dbs::write(const event &event,
txn, at<"event_id"_>(event), event txn, at<"event_id"_>(event), event
}; };
//if(defined(json::get<"state_key"_>(event))) if(defined(json::get<"state_key"_>(event)))
// append_nodes(event, txn); append_nodes(event, txn);
append_indexes(event, txn); append_indexes(event, txn);
} }
@ -86,56 +114,221 @@ void
ircd::m::dbs::append_nodes(const event &event, ircd::m::dbs::append_nodes(const event &event,
db::iov &txn) db::iov &txn)
{ {
thread_local char key_buf[1_KiB], hash_buf[64]; const auto &type{at<"type"_>(event)};
const auto &state_key{at<"state_key"_>(event)};
const auto &event_id{at<"event_id"_>(event)};
const auto &room_id{at<"room_id"_>(event)};
const json::array key if(type == "m.room.create")
{ {
make_key(key_buf, at<"type"_>(event), at<"state_key"_>(event)) thread_local char key[512], head[64];
}; const json::array keys[]
{
{ make_key(key, type, state_key) }
};
const string_view hash const string_view vals[]
{
{ event_id }
};
set_head(txn, room_id, set_node(txn, head, keys, 1, vals, 1));
return;
}
insert(txn, room_id, type, state_key, event_id);
}
void
ircd::m::dbs::append_indexes(const event &event,
db::iov &iov)
{
for(const auto &ptr : indexers)
{ {
make_node(txn, hash_buf, { key }, { at<"event_id"_>(event) }) const m::indexer &indexer{*ptr};
}; indexer(event, iov, db::op::SET);
}
if(json::get<"type"_>(event) == "m.room.member")
{
const m::indexer &indexer{*indexer_origin_joined};
if(json::get<"membership"_>(event) == "join")
indexer(event, iov, db::op::SET);
//else
// indexer(event, iov, db::op::DELETE);
}
} }
// //
// node // node
// //
ircd::json::object void
ircd::m::dbs::get_node(const mutable_buffer &buf, ircd::m::dbs::insert(db::iov &txn,
const string_view &id) const id::room &room_id,
const string_view &type,
const string_view &state_key,
const id::event &event_id)
{
char key[512];
return insert(txn, room_id, make_key(key, type, state_key), event_id);
}
void
ircd::m::dbs::insert(db::iov &txn,
const id::room &room_id,
const json::array &key,
const id::event &event_id)
{ {
db::column column db::column column
{ {
*event::events, "state_node" *event::events, "state_node"
}; };
return get_node(column, buf, id); // Start with the root node ID for room
char nextbuf[512];
string_view nextid
{
get_head(room_id, nextbuf)
};
while(nextid)
{
get_node(column, nextid, [&](const auto &node)
{
std::cout << "@" << nextid << " " << node << std::endl;
const auto pos(find(node, key));
char headbuf[512];
const auto head(set_into(txn, headbuf, node, pos, key, event_id));
set_head(txn, room_id, head);
});
nextid = {};
}
} }
ircd::json::object void
ircd::m::dbs::get_node(db::column &column, ircd::m::dbs::get_value__room(const id::room &room_id,
const mutable_buffer &buf, const string_view &type,
const string_view &id) const string_view &state_key,
const id_closure &closure)
{ {
return db::read(column, id, buf); char head[64];
get_head(room_id, [&head](const string_view &id)
{
strlcpy(head, unquote(id));
});
return get_value(head, type, state_key, closure);
}
void
ircd::m::dbs::get_value(const string_view &head,
const string_view &type,
const string_view &state_key,
const id_closure &closure)
{
char key[512];
return get_value(head, make_key(key, type, state_key), closure);
}
void
ircd::m::dbs::get_value(const string_view &head,
const json::array &key,
const id_closure &closure)
{
db::column column
{
*event::events, "state_node"
};
char nextbuf[512];
string_view nextid{head}; do
{
get_node(column, nextid, [&key, &closure, &nextid, &nextbuf]
(const auto &node)
{
const auto pos{find(node, key)};
const auto &v{unquote(val(node, pos))};
if(valid(id::EVENT, v))
{
if(dbs::key(node, pos) != key)
throw m::NOT_FOUND{};
nextid = {};
closure(v);
}
else nextid = { nextbuf, strlcpy(nextbuf, v) };
});
}
while(nextid);
}
void
ircd::m::dbs::set_head(db::iov &iov,
const id::room &room_id,
const string_view &head_id)
{
db::iov::append
{
iov, db::delta
{
"state_head", // col
room_id, // key
head_id, // val
}
};
} }
ircd::string_view ircd::string_view
ircd::m::dbs::make_node(db::iov &iov, ircd::m::dbs::get_head(const id::room &room_id,
const mutable_buffer &hashbuf, const mutable_buffer &buf)
const std::initializer_list<json::array> &keys, {
const std::initializer_list<string_view> &vals) string_view ret;
get_head(room_id, [&ret, &buf]
(const string_view &head)
{
ret = { data(buf), strlcpy(buf, head) };
});
return ret;
}
void
ircd::m::dbs::get_head(const id::room &room_id,
const id_closure &closure)
{
db::column column
{
*event::events, "state_head"
};
get_head(column, room_id, closure);
}
void
ircd::m::dbs::get_head(db::column &column,
const id::room &room_id,
const id_closure &closure)
{
column(room_id, closure);
}
ircd::string_view
ircd::m::dbs::set_into(db::iov &iov,
const mutable_buffer &hashbuf,
const node &old,
const size_t &pos,
const json::array &key,
const string_view &val)
{ {
thread_local char buf[2_KiB]; thread_local char buf[2_KiB];
const ctx::critical_assertion ca; const ctx::critical_assertion ca;
const string_view node const string_view node
{ {
make_node(buf, keys, vals) make_into(buf, old, pos, key, val)
}; };
const sha256::buf hash const sha256::buf hash
@ -162,29 +355,127 @@ ircd::m::dbs::make_node(db::iov &iov,
return hashb64; return hashb64;
} }
ircd::string_view
ircd::m::dbs::set_node(db::iov &iov,
const mutable_buffer &hashbuf,
const json::array *const &keys,
const size_t &kn,
const string_view *const &vals,
const size_t &vn)
{
thread_local char buf[2_KiB];
const ctx::critical_assertion ca;
const string_view node
{
make_node(buf, keys, kn, vals, vn)
};
const sha256::buf hash
{
sha256{const_buffer{node}}
};
const auto hashb64
{
b64encode_unpadded(hashbuf, hash)
};
db::iov::append
{
iov, db::delta
{
db::op::SET,
"state_node", // col
hashb64, // key
node, // val
}
};
return hashb64;
}
void
ircd::m::dbs::get_node(const string_view &node_id,
const node_closure &closure)
{
db::column column
{
*event::events, "state_node"
};
get_node(column, node_id, closure);
}
void
ircd::m::dbs::get_node(db::column &column,
const string_view &node_id,
const node_closure &closure)
{
column(node_id, closure);
}
ircd::json::object
ircd::m::dbs::make_into(const mutable_buffer &out,
const node &old,
const size_t &pos,
const json::array &_key,
const string_view &_val)
{
const size_t kn{keys(old) + 1};
json::array _keys[kn];
{
size_t i(0), j(0);
while(i < pos)
_keys[i++] = key(old, j++);
_keys[i++] = _key;
while(i < keys(old) + 1)
_keys[i++] = key(old, j++);
}
const size_t vn{vals(old) + 1};
string_view _vals[vn + 1];
{
size_t i(0), j(0);
while(i < pos)
_vals[i++] = val(old, j++);
_vals[i++] = _val;
while(i < vals(old) + 1)
_vals[i++] = val(old, j++);
}
return make_node(out, _keys, kn, _vals, vn);
}
ircd::json::object ircd::json::object
ircd::m::dbs::make_node(const mutable_buffer &out, ircd::m::dbs::make_node(const mutable_buffer &out,
const std::initializer_list<json::array> &_keys, const json::array *const &keys_,
const std::initializer_list<string_view> &_vals) const size_t &kn,
const string_view *const &vals_,
const size_t &vn)
{ {
assert(_keys.size() <= 2); assert(kn > 0 && vn > 0);
assert(_vals.size() <= 3); assert(kn == vn || kn + 1 == vn);
json::value keys[2], vals[3];
{
size_t i(0);
for(const auto &key : _keys)
keys[i++] = key;
i = 0; json::value keys[kn];
for(const auto &val : _vals) {
vals[i++] = val; for(size_t i(0); i < kn; ++i)
keys[i] = keys_[i];
}
json::value vals[vn];
{
for(size_t i(0); i < vn; ++i)
vals[i] = vals_[i];
}; };
json::iov iov; json::iov iov;
const json::iov::push push[] const json::iov::push push[]
{ {
{ iov, { "k"_sv, { keys, _keys.size() } } }, { iov, { "k"_sv, { keys, kn } } },
{ iov, { "v"_sv, { vals, _vals.size() } } }, { iov, { "v"_sv, { vals, vn } } },
}; };
return { data(out), json::print(out, iov) }; return { data(out), json::print(out, iov) };
@ -204,39 +495,40 @@ size_t
ircd::m::dbs::find(const node &node, ircd::m::dbs::find(const node &node,
const json::array &parts) const json::array &parts)
{ {
const auto &keys
{
json::get<"k"_>(node)
};
size_t ret{0}; size_t ret{0};
for(const json::array key : keys) for(const json::array key : json::get<"k"_>(node))
{ if(keycmp(parts, key) <= 0)
auto kit(begin(key));
auto pit(begin(parts));
for(; pit != end(parts) && kit != end(key); ++pit, ++kit)
{
const auto &keyp{*kit};
const auto &part{*pit};
assert(surrounds(part, '"'));
assert(surrounds(keyp, '"'));
if(part < keyp)
return ret;
if(part > keyp)
break;
}
if(kit == end(key))
return ret; return ret;
else
++ret; ++ret;
}
return ret; return ret;
} }
int
ircd::m::dbs::keycmp(const json::array &a,
const json::array &b)
{
auto ait(begin(a));
auto bit(begin(b));
for(; ait != end(a) && bit != end(b); ++ait, ++bit)
{
assert(surrounds(*ait, '"'));
assert(surrounds(*bit, '"'));
if(*ait < *bit)
return -1;
if(*bit < *ait)
return 1;
}
assert(ait == end(a) || bit == end(b));
return ait == end(a) && bit != end(b)? -1:
ait == end(a) && bit == end(b)? 0:
1;
}
ircd::json::array ircd::json::array
ircd::m::dbs::make_key(const mutable_buffer &out, ircd::m::dbs::make_key(const mutable_buffer &out,
const string_view &type, const string_view &type,
@ -270,15 +562,26 @@ ircd::m::dbs::key(const node &node,
} }
size_t size_t
ircd::m::dbs::keys(const node &node) ircd::m::dbs::children(const node &node)
{ {
return json::get<"k"_>(node).count(); size_t ret(0);
for(const auto &v : json::get<"v"_>(node))
if(!valid(id::EVENT, v))
++ret;
return ret;
} }
size_t size_t
ircd::m::dbs::vals(const node &node) ircd::m::dbs::vals(const node &node)
{ {
return json::get<"v"_>(node).count(); return json::get<"v"_>(node).count();
}
size_t
ircd::m::dbs::keys(const node &node)
{
return json::get<"k"_>(node).count();
} }
// //
@ -555,59 +858,6 @@ ircd::m::dbs::exists(const event::id &event_id)
// //
// //
namespace ircd::m
{
struct indexer;
extern std::set<std::shared_ptr<indexer>> indexers;
extern const std::unique_ptr<indexer> indexer_origin_joined;
}
struct ircd::m::indexer
{
//TODO: collapse
struct concat;
struct concat_s; //TODO: special
struct concat_v;
struct concat_2v;
struct concat_3vs; //TODO: special
std::string name;
virtual void operator()(const event &, db::iov &iov, const db::op &op) const {}
indexer(std::string name)
:name{std::move(name)}
{}
virtual ~indexer() noexcept;
};
ircd::m::indexer::~indexer()
noexcept
{
}
void
ircd::m::dbs::append_indexes(const event &event,
db::iov &iov)
{
for(const auto &ptr : indexers)
{
const m::indexer &indexer{*ptr};
indexer(event, iov, db::op::SET);
}
if(json::get<"type"_>(event) == "m.room.member")
{
const m::indexer &indexer{*indexer_origin_joined};
if(json::get<"membership"_>(event) == "join")
indexer(event, iov, db::op::SET);
//else
// indexer(event, iov, db::op::DELETE);
}
}
struct ircd::m::indexer::concat struct ircd::m::indexer::concat
:indexer :indexer
{ {