0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-09-29 20:28:52 +02:00

ircd:Ⓜ️:state: Make it work.

This commit is contained in:
Jason Volk 2018-02-01 15:18:45 -08:00
parent e3f671a523
commit 50a0983fa0
2 changed files with 402 additions and 164 deletions

View file

@ -23,15 +23,15 @@ namespace ircd::m::state
constexpr size_t KEY_MAX_SZ { 256 + 256 + 16 };
constexpr size_t VAL_MAX_SZ { 256 + 16 };
constexpr size_t NODE_MAX_SZ { 4_KiB };
constexpr size_t NODE_MAX_KEY { 2 }; // tmp for now
constexpr size_t NODE_MAX_VAL { NODE_MAX_KEY };
constexpr size_t NODE_MAX_DEG { NODE_MAX_KEY + 1 }; // tmp for now
constexpr int8_t MAX_HEIGHT { 16 }; // good for few mil at any degree :)
int keycmp(const json::array &a, const json::array &b);
json::array make_key(const mutable_buffer &out, const string_view &type, const string_view &state_key);
void make_key(const string_view &type, const string_view &state_key, const key_closure &);
json::object make_node(const mutable_buffer &out, const json::array *const &keys, const size_t &kn, const string_view *const &vals, const size_t &vn, const string_view *const &child, const size_t &cn);
template<class... args> string_view set_node(db::txn &txn, const mutable_buffer &id, args&&...);
string_view set_node(db::txn &txn, const mutable_buffer &id, const json::object &node);
void get_node(db::column &, const string_view &id, const node_closure &);
void get_node(const string_view &id, const node_closure &);
@ -68,6 +68,8 @@ struct ircd::m::state::node
json::property<name::c, json::array>
>
{
struct rep;
size_t keys() const;
size_t vals() const;
size_t childs() const;
@ -76,10 +78,20 @@ struct ircd::m::state::node
string_view val(const size_t &) const;
string_view child(const size_t &) const;
bool has_child(const size_t &) const;
size_t keys(json::array *const &out, const size_t &max) const;
size_t vals(string_view *const &out, const size_t &max) const;
size_t childs(string_view *const &out, const size_t &max) const;
size_t find(const json::array &key) const;
bool has_key(const json::array &key) const;
bool has_child(const size_t &) const;
using super_type::tuple;
using super_type::operator=;
};
#pragma GCC diagnostic pop
static_assert
(
ircd::m::state::NODE_MAX_KEY == ircd::m::state::NODE_MAX_VAL
);

View file

@ -10,6 +10,122 @@
#include <ircd/m/m.h>
struct ircd::m::state::node::rep
{
std::array<json::array, NODE_MAX_KEY + 1> keys;
std::array<string_view, NODE_MAX_VAL + 1> vals;
std::array<string_view, NODE_MAX_DEG + 1> chld;
size_t kn {0};
size_t vn {0};
size_t cn {0};
bool full() const;
bool overfull() const;
size_t find(const json::array &key) const;
void shr(const size_t &pos);
json::object write(const mutable_buffer &out);
string_view write(db::txn &, const mutable_buffer &id);
rep(const node &node);
rep() = default;
};
ircd::m::state::node::rep::rep(const node &node)
:kn{node.keys(keys.data(), keys.size())}
,vn{node.vals(vals.data(), vals.size())}
,cn{node.childs(chld.data(), chld.size())}
{
}
ircd::string_view
ircd::m::state::node::rep::write(db::txn &txn,
const mutable_buffer &idbuf)
{
thread_local char buf[NODE_MAX_SZ];
return set_node(txn, idbuf, write(buf));
}
ircd::json::object
ircd::m::state::node::rep::write(const mutable_buffer &out)
{
assert(kn == vn);
assert(cn <= kn + 1);
assert(kn > 0 && vn > 0);
assert(kn <= NODE_MAX_KEY);
assert(vn <= NODE_MAX_VAL);
assert(cn <= NODE_MAX_DEG);
json::value keys[kn];
{
for(size_t i(0); i < kn; ++i)
keys[i] = this->keys[i];
}
json::value vals[vn];
{
for(size_t i(0); i < vn; ++i)
vals[i] = this->vals[i];
};
json::value chld[cn];
{
for(size_t i(0); i < cn; ++i)
chld[i] = this->chld[i];
};
json::iov iov;
const json::iov::push push[]
{
{ iov, { "k"_sv, { keys, kn } } },
{ iov, { "v"_sv, { vals, vn } } },
{ iov, { "c"_sv, { chld, cn } } },
};
return { data(out), json::print(out, iov) };
}
/// Shift right.
void
ircd::m::state::node::rep::shr(const size_t &pos)
{
std::copy_backward(begin(keys) + pos, begin(keys) + kn, begin(keys) + kn + 1);
std::copy_backward(begin(vals) + pos, begin(vals) + vn, begin(vals) + vn + 1);
std::copy_backward(begin(chld) + pos, begin(chld) + cn, begin(chld) + cn + 1);
}
size_t
ircd::m::state::node::rep::find(const json::array &parts)
const
{
size_t i{0};
for(; i < kn; ++i)
if(keycmp(parts, keys[i]) <= 0)
return i;
else
++i;
return i;
}
bool
ircd::m::state::node::rep::overfull()
const
{
assert(kn == vn);
return kn > NODE_MAX_KEY;
}
bool
ircd::m::state::node::rep::full()
const
{
assert(kn == vn);
return kn >= NODE_MAX_KEY;
}
void
ircd::m::state::append_nodes(db::txn &txn,
const event &event)
@ -27,18 +143,17 @@ ircd::m::state::append_nodes(db::txn &txn,
const critical_assertion ca;
thread_local char key[KEY_MAX_SZ];
thread_local char head[ID_MAX_SZ];
thread_local char node[NODE_MAX_SZ];
const json::array keys[]
{
{ state::make_key(key, type, state_key) }
};
node::rep rep;
rep.keys[0] = make_key(key, type, state_key);
rep.kn = 1;
rep.vals[0] = event_id;
rep.vn = 1;
rep.chld[0] = string_view{};
rep.cn = 1;
const string_view vals[]
{
{ event_id }
};
set_head(txn, room_id, set_node(txn, head, keys, 1, vals, 1));
set_head(txn, room_id, set_node(txn, head, rep.write(node)));
return;
}
@ -58,6 +173,162 @@ ircd::m::state::insert(db::txn &txn,
return insert(txn, room_id, make_key(key, type, state_key), event_id);
}
namespace ircd::m::state
{
string_view
inserter(int8_t &height,
db::txn &txn,
const json::array &key,
const string_view &val,
const node &node,
const mutable_buffer &idbuf,
node::rep &push,
const mutable_buffer &pushbuf)
{
// Recursion metrics
const unwind down{[&height]{ --height; }};
if(unlikely(++height >= MAX_HEIGHT))
throw assertive{"recursion limit exceeded"};
// This function assumes that any node argument is a previously "existing"
// node which means it contains at least one key/value.
assert(node.keys() > 0);
assert(node.keys() == node.vals());
node::rep rep{node};
const auto pos{node.find(key)};
std::cout << int(height) << " " << pos << " " << node << " <---- " << key << std::endl;
if(keycmp(node.key(pos), key) == 0)
{
rep.keys[pos] = key;
rep.vals[pos] = val;
return rep.write(txn, idbuf);
}
if(node.childs() == 0 && !rep.full())
{
rep.shr(pos);
rep.keys[pos] = key;
++rep.kn;
rep.vals[pos] = val;
++rep.vn;
rep.chld[pos] = string_view{};
++rep.cn;
return rep.write(txn, idbuf);
}
if(node.childs() == 0 && rep.full())
{
rep.shr(pos);
rep.keys[pos] = key;
++rep.kn;
rep.vals[pos] = val;
++rep.vn;
size_t i(0);
node::rep left, right;
for(; i < rep.kn / 2; ++i)
{
left.keys[left.kn++] = rep.keys[i];
left.vals[left.vn++] = rep.vals[i];
left.chld[left.cn++] = string_view{};
}
push.keys[push.kn++] = rep.keys[i];
push.vals[push.vn++] = rep.vals[i];
for(++i; i < rep.kn; ++i)
{
right.keys[right.kn++] = rep.keys[i];
right.vals[right.vn++] = rep.vals[i];
right.chld[right.cn++] = string_view{};
}
thread_local char lc[ID_MAX_SZ], rc[ID_MAX_SZ];
push.chld[push.cn++] = left.write(txn, lc);
push.chld[push.cn++] = right.write(txn, rc);
const json::object ret{push.write(pushbuf)};
return ret;
}
if(!empty(node.child(pos)))
{
node::rep cpush;
string_view child_id;
get_node(node.child(pos), [&](const auto &node)
{
child_id = inserter(height, txn, key, val, node, idbuf, cpush, pushbuf);
});
if(!cpush.kn)
{
rep.chld[pos] = child_id;
return rep.write(txn, idbuf);
}
if(!rep.full())
{
rep.shr(pos);
rep.keys[pos] = cpush.keys[0];
++rep.kn;
rep.vals[pos] = cpush.vals[0];
++rep.vn;
rep.chld[pos] = cpush.chld[0];
rep.chld[pos+1] = cpush.chld[1];
++rep.cn;
return rep.write(txn, idbuf);
}
std::cout << "INTEGRATE PUSH T: " << child_id << std::endl;
std::cout << "INTEGRATE PUSH U: " << node << std::endl;
rep.shr(pos);
rep.keys[pos] = key;
++rep.kn;
rep.vals[pos] = val;
++rep.vn;
size_t i(0);
node::rep left, right;
for(; i < rep.kn / 2; ++i)
{
left.keys[left.kn++] = rep.keys[i];
left.vals[left.vn++] = rep.vals[i];
left.chld[left.cn++] = rep.chld[i];
}
push.keys[push.kn++] = rep.keys[i];
push.vals[push.vn++] = rep.vals[i];
for(++i; i < rep.kn; ++i)
{
right.keys[right.kn++] = rep.keys[i];
right.vals[right.vn++] = rep.vals[i];
right.chld[right.cn++] = rep.chld[i];
}
thread_local char lc[ID_MAX_SZ], rc[ID_MAX_SZ];
push.chld[push.cn++] = left.write(txn, lc);
push.chld[push.cn++] = right.write(txn, rc);
const json::object ret{push.write(pushbuf)};
return ret;
}
rep.shr(pos);
rep.keys[pos] = key;
++rep.kn;
rep.vals[pos] = val;
++rep.vn;
rep.chld[pos] = string_view{};
++rep.cn;
return rep.write(txn, idbuf);
}}
void
ircd::m::state::insert(db::txn &txn,
const id::room &room_id,
@ -67,20 +338,25 @@ ircd::m::state::insert(db::txn &txn,
db::column heads{*event::events, "state_head"};
db::column nodes{*event::events, "state_node"};
// Start with the root node ID for room.
char nextbuf[ID_MAX_SZ];
string_view nextid{get_head(heads, nextbuf, room_id)};
char idbuf[ID_MAX_SZ];
thread_local char pushbuf[NODE_MAX_SZ];
char prevbuf[ID_MAX_SZ];
string_view previd;
while(nextid) get_node(nodes, nextid, [&](const node &node)
string_view head
{
const auto pos(node.find(key));
const auto head(set_node(txn, nextbuf, node, pos, key, event_id));
set_head(txn, room_id, head);
nextid = {};
get_head(heads, idbuf, room_id)
};
node::rep push;
int8_t height{0};
get_node(head, [&](const node &node)
{
head = inserter(height, txn, key, event_id, node, idbuf, push, pushbuf);
});
if(push.kn)
head = push.write(txn, idbuf);
set_head(txn, room_id, head);
}
/// Convenience to get value from the current room head.
@ -133,22 +409,22 @@ ircd::m::state::get_value(db::column &column,
string_view nextid{head};
while(nextid) get_node(column, nextid, [&](const node &node)
{
const auto pos(node.find(key));
if(pos >= node.vals())
auto pos(node.find(key));
if(pos < node.keys() && node.key(pos) == key)
{
nextid = {};
closure(node.val(pos));
return;
}
const auto c(node.childs());
if(c && pos >= c)
pos = c - 1;
if(!node.has_child(pos))
throw m::NOT_FOUND{};
const auto &v(node.val(pos));
if(valid(id::EVENT, v))
{
if(node.key(pos) != key)
throw m::NOT_FOUND{};
nextid = {};
closure(v);
} else {
assert(size(v) < sizeof(nextbuf));
nextid = { nextbuf, strlcpy(nextbuf, v) };
}
nextid = { nextbuf, strlcpy(nextbuf, node.child(pos)) };
});
}
@ -222,23 +498,12 @@ ircd::m::state::get_node(db::column &column,
}
/// Writes a node to the db::txn and returns the id of this node (a hash) into
/// the buffer. The template allows for arguments to be forwarded to your
/// choice of the non-template make_node() overloads (exclude their leading
/// `out` buffer parameter).
template<class... args>
/// the buffer. The template allows for arguments to be forwarded to make_node()
ircd::string_view
ircd::m::state::set_node(db::txn &iov,
const mutable_buffer &hashbuf,
args&&... a)
const json::object &node)
{
thread_local char buf[NODE_MAX_SZ];
const ctx::critical_assertion ca;
const json::object node
{
make_node(buf, std::forward<args>(a)...)
};
const sha256::buf hash
{
sha256{const_buffer{node}}
@ -263,107 +528,6 @@ ircd::m::state::set_node(db::txn &iov,
return hashb64;
}
/// Add key/val pair to an existing node, which creates a new node printed
/// into the buffer `out`. If the key matches an existing key, it will be
/// replaced and the new node will have the same size as old.
ircd::json::object
ircd::m::state::make_node(const mutable_buffer &out,
const node &old,
const size_t &pos,
const json::array &key,
const string_view &val)
{
json::array keys[old.keys() + 1];
size_t kn{0};
{
size_t n(0);
while(kn < pos)
keys[kn++] = old.key(n++);
keys[kn++] = key;
if(pos < old.keys() && keycmp(key, old.key(n)) == 0)
n++;
while(kn < old.keys() + 1 && n < old.keys())
keys[kn++] = old.key(n++);
}
string_view vals[old.vals() + 1];
size_t vn{0};
{
size_t n(0);
while(vn < pos)
vals[vn++] = old.val(n++);
vals[vn++] = val;
if(kn == old.keys())
n++;
while(vn < old.vals() + 1 && n < old.vals())
vals[vn++] = old.val(n++);
}
assert(kn == old.keys() || kn == old.keys() + 1);
return make_node(out, keys, kn, vals, vn);
}
/// Prints a node into the buffer `out` using the keys and vals arguments
/// which must be pointers to arrays. Size of each array is specified in
/// the following argument. Each array must have at least one element each.
/// the chld array can have one more element than the keys array if desired.
ircd::json::object
ircd::m::state::make_node(const mutable_buffer &out,
const json::array *const &keys_,
const size_t &kn,
const string_view *const &vals_,
const size_t &vn,
const string_view *const &chld_,
const size_t &cn)
{
assert(kn > 0 && vn > 0);
assert(kn == vn);
assert(cn <= kn + 1);
json::value keys[kn];
{
for(size_t i(0); i < kn; ++i)
keys[i] = keys_[i];
}
json::value vals[vn];
{
for(size_t i(0); i < vn; ++i)
vals[i] = vals_[i];
};
json::value chld[cn];
{
for(size_t i(0); i < cn; ++i)
chld[i] = chld_[i];
};
json::iov iov;
const json::iov::push push[]
{
{ iov, { "k"_sv, { keys, kn } } },
{ iov, { "v"_sv, { vals, vn } } },
{ iov, { "c"_sv, { chld, cn } } },
};
return { data(out), json::print(out, iov) };
}
/// Convenience to close over the key creation using a stack buffer (hence
/// safe for reentrance / multiple closing)
void
ircd::m::state::make_key(const string_view &type,
const string_view &state_key,
const key_closure &closure)
{
char buf[KEY_MAX_SZ];
closure(make_key(buf, type, state_key));
}
/// Creates a key array from the most common key pattern of a matrix
/// room (type,state_key).
ircd::json::array
@ -415,6 +579,26 @@ ircd::m::state::keycmp(const json::array &a,
// node
//
// Count values that actually lead to other nodes
bool
ircd::m::state::node::has_child(const size_t &pos)
const
{
return !empty(child(pos));
}
// Count values that actually lead to other nodes
bool
ircd::m::state::node::has_key(const json::array &key)
const
{
const auto pos(find(key));
if(pos >= keys())
return false;
return keycmp(this->key(pos), key) == 0;
}
/// Find position for a val in node. Uses the keycmp(). If there is one
/// key in node, and the argument compares less than or equal to the key,
/// 0 is returned, otherwise 1 is returned. If there are two keys in node
@ -437,19 +621,54 @@ const
return ret;
}
// Count values that actually lead to other nodes
bool
ircd::m::state::node::has_child(const size_t &pos)
size_t
ircd::m::state::node::childs(string_view *const &out,
const size_t &max)
const
{
return !empty(child(pos));
size_t i(0);
for(const string_view &c : json::get<"c"_>(*this))
if(likely(i < max))
out[i++] = c;
return i;
}
size_t
ircd::m::state::node::vals(string_view *const &out,
const size_t &max)
const
{
size_t i(0);
for(const string_view &v : json::get<"v"_>(*this))
if(likely(i < max))
out[i++] = v;
return i;
}
size_t
ircd::m::state::node::keys(json::array *const &out,
const size_t &max)
const
{
size_t i(0);
for(const json::array &k : json::get<"k"_>(*this))
if(likely(i < max))
out[i++] = k;
return i;
}
ircd::string_view
ircd::m::state::node::child(const size_t &pos)
const
{
const json::array children{json::get<"c"_>(*this, json::empty_array)};
const json::array &children
{
json::get<"c"_>(*this, json::empty_array)
};
return unquote(children[pos]);
}
@ -458,7 +677,11 @@ ircd::string_view
ircd::m::state::node::val(const size_t &pos)
const
{
const json::array values{json::get<"v"_>(*this, json::empty_array)};
const json::array &values
{
json::get<"v"_>(*this, json::empty_array)
};
return unquote(values[pos]);
}
@ -467,9 +690,12 @@ ircd::json::array
ircd::m::state::node::key(const size_t &pos)
const
{
const json::array keys{json::get<"k"_>(*this, json::empty_array)};
const json::array ret{keys[pos]};
return ret;
const json::array &keys
{
json::get<"k"_>(*this, json::empty_array)
};
return keys[pos];
}
// Count children in node
@ -479,7 +705,7 @@ const
{
size_t ret(0);
for(const auto &c : json::get<"c"_>(*this))
ret += !empty(c);
ret += !empty(c) && c != json::empty_string;
return ret;
}