diff --git a/include/ircd/m/dbs.h b/include/ircd/m/dbs.h index bdf515c64..7dcfb3c91 100644 --- a/include/ircd/m/dbs.h +++ b/include/ircd/m/dbs.h @@ -21,7 +21,6 @@ namespace ircd::m::dbs bool exists(const event::id &); void append_indexes(const event &, db::txn &); - void append_nodes(const event &, db::txn &); void write(const event &, db::txn &); } diff --git a/include/ircd/m/state.h b/include/ircd/m/state.h index 8f8610fd3..8c4711e00 100644 --- a/include/ircd/m/state.h +++ b/include/ircd/m/state.h @@ -15,37 +15,39 @@ namespace ircd::m::state { struct node; - size_t keys(const node &); - size_t vals(const node &); - size_t children(const node &); - json::array key(const node &, const size_t &); - string_view val(const node &, const size_t &); - int keycmp(const json::array &a, const json::array &b); - json::array make_key(const mutable_buffer &out, const string_view &type, const string_view &state_key); - size_t find(const node &, const json::array &key); - size_t find(const node &, const string_view &type, const string_view &state_key); - json::object make_node(const mutable_buffer &out, const json::array *const &keys, const size_t &kn, const string_view *const &vals, const size_t &vn); - json::object make_into(const mutable_buffer &out, const node &old, const size_t &pos, const json::array &key, const string_view &val); - using id_closure = std::function; using node_closure = std::function; + using key_closure = std::function; + + constexpr size_t ID_MAX_SZ { 64 }; + constexpr size_t KEY_MAX_SZ { 256 + 256 + 16 }; + constexpr size_t NODE_MAX_SZ { 4_KiB }; + + int keycmp(const json::array &a, const json::array &b); + + json::array make_key(const mutable_buffer &out, const string_view &type, const string_view &state_key); + void make_key(const string_view &type, const string_view &state_key, const key_closure &); + + json::object make_node(const mutable_buffer &out, const json::array *const &keys, const size_t &kn, const string_view *const &vals, const size_t &vn); + json::object make_node(const mutable_buffer &out, const node &old, const size_t &pos, const json::array &key, const string_view &val); + template string_view set_node(db::txn &txn, const mutable_buffer &id, args&&...); void get_node(db::column &, const string_view &id, const node_closure &); void get_node(const string_view &id, const node_closure &); - string_view set_node(db::txn &txn, const mutable_buffer &hash, const json::array *const &keys, const size_t &kn, const string_view *const &vals, const size_t &vn); - string_view set_into(db::txn &txn, const mutable_buffer &hash, const node &old, const size_t &pos, const json::array &key, const string_view &val); - void get_head(db::column &, const id::room &, const id_closure &); - void get_head(const id::room &, const id_closure &); - string_view get_head(const id::room &, const mutable_buffer &buf); + string_view get_head(db::column &, const mutable_buffer &out, const id::room &); + string_view get_head(const mutable_buffer &out, const id::room &); void set_head(db::txn &txn, const id::room &, const string_view &head); + void get_value(db::column &, const string_view &head, const json::array &key, const id_closure &); void get_value(const string_view &head, const json::array &key, const id_closure &); void get_value(const string_view &head, const string_view &type, const string_view &state_key, const id_closure &); void get_value__room(const id::room &, const string_view &type, const string_view &state_key, const id_closure &); void insert(db::txn &txn, const id::room &, const json::array &key, const id::event &); void insert(db::txn &txn, const id::room &, const string_view &type, const string_view &state_key, const id::event &); + + void append_nodes(db::txn &, const event &); } namespace ircd::m::state::name @@ -63,6 +65,15 @@ struct ircd::m::state::node json::property > { + size_t keys() const; + size_t vals() const; + size_t children() const; + + json::array key(const size_t &) const; + string_view val(const size_t &) const; + + size_t find(const json::array &key) const; + using super_type::tuple; using super_type::operator=; }; diff --git a/ircd/m/dbs.cc b/ircd/m/dbs.cc index 9dd3dd989..eb1fead60 100644 --- a/ircd/m/dbs.cc +++ b/ircd/m/dbs.cc @@ -105,40 +105,11 @@ ircd::m::dbs::write(const event &event, }; if(defined(json::get<"state_key"_>(event))) - append_nodes(event, txn); + state::append_nodes(txn, event); append_indexes(event, txn); } -void -ircd::m::dbs::append_nodes(const event &event, - db::txn &txn) -{ - const auto &type{at<"type"_>(event)}; - const auto &state_key{at<"state_key"_>(event)}; - const auto &event_id{at<"event_id"_>(event)}; - const auto &room_id{at<"room_id"_>(event)}; - - if(type == "m.room.create") - { - thread_local char key[512], head[64]; - const json::array keys[] - { - { state::make_key(key, type, state_key) } - }; - - const string_view vals[] - { - { event_id } - }; - - state::set_head(txn, room_id, state::set_node(txn, head, keys, 1, vals, 1)); - return; - } - - state::insert(txn, room_id, type, state_key, event_id); -} - void ircd::m::dbs::append_indexes(const event &event, db::txn &txn) diff --git a/ircd/m/state.cc b/ircd/m/state.cc index 36fee40ce..4263962c9 100644 --- a/ircd/m/state.cc +++ b/ircd/m/state.cc @@ -10,6 +10,41 @@ #include +void +ircd::m::state::append_nodes(db::txn &txn, + const event &event) +{ + const auto &type{at<"type"_>(event)}; + const auto &state_key{at<"state_key"_>(event)}; + const auto &event_id{at<"event_id"_>(event)}; + const auto &room_id{at<"room_id"_>(event)}; + + if(type == "m.room.create") + { + // Because this is a new tree and nothing is read from the DB, all + // writes here are just copies into the txn and these buffers can + // remain off-stack. + const critical_assertion ca; + thread_local char key[KEY_MAX_SZ]; + thread_local char head[ID_MAX_SZ]; + + const json::array keys[] + { + { state::make_key(key, type, state_key) } + }; + + const string_view vals[] + { + { event_id } + }; + + set_head(txn, room_id, set_node(txn, head, keys, 1, vals, 1)); + return; + } + + state::insert(txn, room_id, type, state_key, event_id); +} + void ircd::m::state::insert(db::txn &txn, const id::room &room_id, @@ -17,7 +52,9 @@ ircd::m::state::insert(db::txn &txn, const string_view &state_key, const id::event &event_id) { - char key[512]; + // The insertion process reads from the DB and will yield this ircd::ctx + // so the key buffer must stay on this stack. + char key[KEY_MAX_SZ]; return insert(txn, room_id, make_key(key, type, state_key), event_id); } @@ -27,59 +64,48 @@ ircd::m::state::insert(db::txn &txn, const json::array &key, const id::event &event_id) { - db::column column + db::column heads{*event::events, "state_head"}; + db::column nodes{*event::events, "state_node"}; + + // Start with the root node ID for room. + char nextbuf[ID_MAX_SZ]; + string_view nextid{get_head(heads, nextbuf, room_id)}; + + char prevbuf[ID_MAX_SZ]; + string_view previd; + + while(nextid) get_node(nodes, nextid, [&](const node &node) { - *event::events, "state_node" - }; - - // Start with the root node ID for room - char nextbuf[512]; - string_view nextid - { - get_head(room_id, nextbuf) - }; - - while(nextid) - { - get_node(column, nextid, [&](const auto &node) - { - std::cout << "@" << nextid << " " << node << std::endl; - const auto pos(find(node, key)); - - char headbuf[512]; - const auto head(set_into(txn, headbuf, node, pos, key, event_id)); - set_head(txn, room_id, head); - }); - + const auto pos(node.find(key)); + const auto head(set_node(txn, nextbuf, node, pos, key, event_id)); + set_head(txn, room_id, head); nextid = {}; - } + }); } +/// Convenience to get value from the current room head. void ircd::m::state::get_value__room(const id::room &room_id, const string_view &type, const string_view &state_key, const id_closure &closure) { - char head[64]; - get_head(room_id, [&head](const string_view &id) - { - strlcpy(head, unquote(id)); - }); - - return get_value(head, type, state_key, closure); + char head[ID_MAX_SZ]; + return get_value(get_head(head, room_id), type, state_key, closure); } +/// Convenience to get value making a key void ircd::m::state::get_value(const string_view &head, const string_view &type, const string_view &state_key, const id_closure &closure) { - char key[512]; + char key[KEY_MAX_SZ]; return get_value(head, make_key(key, type, state_key), closure); } +/// see: get_value(); user does not have to supply column reference here void ircd::m::state::get_value(const string_view &head, const json::array &key, @@ -90,31 +116,43 @@ ircd::m::state::get_value(const string_view &head, *event::events, "state_node" }; - char nextbuf[512]; - string_view nextid{head}; do - { - get_node(column, nextid, [&key, &closure, &nextid, &nextbuf] - (const auto &node) - { - const auto pos{find(node, key)}; - if(pos >= vals(node)) - throw m::NOT_FOUND{}; - - const auto &v{unquote(val(node, pos))}; - if(valid(id::EVENT, v)) - { - if(state::key(node, pos) != key) - throw m::NOT_FOUND{}; - - nextid = {}; - closure(v); - } - else nextid = { nextbuf, strlcpy(nextbuf, v) }; - }); - } - while(nextid); + get_value(column, head, key, closure); } +/// Recursive query to find the leaf value for the given key, starting from +/// the given head node ID. Value can be viewed in the closure. This throws +/// m::NOT_FOUND if the exact key and its value does not exist in the tree; +/// no node ID's are ever returned here. +void +ircd::m::state::get_value(db::column &column, + const string_view &head, + const json::array &key, + const id_closure &closure) +{ + char nextbuf[ID_MAX_SZ]; + string_view nextid{head}; + while(nextid) get_node(column, nextid, [&](const node &node) + { + const auto pos(node.find(key)); + if(pos >= node.vals()) + throw m::NOT_FOUND{}; + + const auto &v(node.val(pos)); + if(valid(id::EVENT, v)) + { + if(node.key(pos) != key) + throw m::NOT_FOUND{}; + + nextid = {}; + closure(v); + } else { + assert(size(v) < sizeof(nextbuf)); + nextid = { nextbuf, strlcpy(nextbuf, v) }; + } + }); +} + +/// Set the root node ID for a room in this db transaction. void ircd::m::state::set_head(db::txn &iov, const id::room &room_id, @@ -131,12 +169,27 @@ ircd::m::state::set_head(db::txn &iov, }; } +/// Copy a room's root node ID into buffer. ircd::string_view -ircd::m::state::get_head(const id::room &room_id, - const mutable_buffer &buf) +ircd::m::state::get_head(const mutable_buffer &buf, + const id::room &room_id) +{ + db::column column + { + *event::events, "state_head" + }; + + return get_head(column, buf, room_id); +} + +/// Copy a room's root node ID into buffer; already have column reference. +ircd::string_view +ircd::m::state::get_head(db::column &column, + const mutable_buffer &buf, + const id::room &room_id) { string_view ret; - get_head(room_id, [&ret, &buf] + column(room_id, [&ret, &buf] (const string_view &head) { ret = { data(buf), strlcpy(buf, head) }; @@ -145,109 +198,10 @@ ircd::m::state::get_head(const id::room &room_id, return ret; } -void -ircd::m::state::get_head(const id::room &room_id, - const id_closure &closure) -{ - db::column column - { - *event::events, "state_head" - }; - - get_head(column, room_id, closure); -} - -void -ircd::m::state::get_head(db::column &column, - const id::room &room_id, - const id_closure &closure) -{ - column(room_id, closure); -} - -ircd::string_view -ircd::m::state::set_into(db::txn &iov, - const mutable_buffer &hashbuf, - const node &old, - const size_t &pos, - const json::array &key, - const string_view &val) -{ - thread_local char buf[2_KiB]; - const ctx::critical_assertion ca; - - const string_view node - { - make_into(buf, old, pos, key, val) - }; - - const sha256::buf hash - { - sha256{const_buffer{node}} - }; - - const auto hashb64 - { - b64encode_unpadded(hashbuf, hash) - }; - - db::txn::append - { - iov, db::delta - { - db::op::SET, - "state_node", // col - hashb64, // key - node, // val - } - }; - - return hashb64; -} - -ircd::string_view -ircd::m::state::set_node(db::txn &iov, - const mutable_buffer &hashbuf, - const json::array *const &keys, - const size_t &kn, - const string_view *const &vals, - const size_t &vn) -{ - thread_local char buf[2_KiB]; - const ctx::critical_assertion ca; - - const string_view node - { - make_node(buf, keys, kn, vals, vn) - }; - - const sha256::buf hash - { - sha256{const_buffer{node}} - }; - - const auto hashb64 - { - b64encode_unpadded(hashbuf, hash) - }; - - db::txn::append - { - iov, db::delta - { - db::op::SET, - "state_node", // col - hashb64, // key - node, // val - } - }; - - return hashb64; -} - +/// View a node by ID. void ircd::m::state::get_node(const string_view &node_id, - const node_closure &closure) + const node_closure &closure) { db::column column { @@ -257,6 +211,8 @@ ircd::m::state::get_node(const string_view &node_id, get_node(column, node_id, closure); } +/// View a node by ID. This can be used when user already has a reference +/// to the db column. void ircd::m::state::get_node(db::column &column, const string_view &node_id, @@ -265,40 +221,96 @@ ircd::m::state::get_node(db::column &column, column(node_id, closure); } -ircd::json::object -ircd::m::state::make_into(const mutable_buffer &out, - const node &old, - const size_t &pos, - const json::array &_key, - const string_view &_val) +/// Writes a node to the db::txn and returns the id of this node (a hash) into +/// the buffer. The template allows for arguments to be forwarded to your +/// choice of the non-template make_node() overloads (exclude their leading +/// `out` buffer parameter). +template +ircd::string_view +ircd::m::state::set_node(db::txn &iov, + const mutable_buffer &hashbuf, + args&&... a) { - const size_t kn{keys(old) + 1}; - json::array _keys[kn]; + thread_local char buf[NODE_MAX_SZ]; + const ctx::critical_assertion ca; + + const json::object node { - size_t i(0), j(0); - while(i < pos) - _keys[i++] = key(old, j++); + make_node(buf, std::forward(a)...) + }; - _keys[i++] = _key; - while(i < keys(old) + 1) - _keys[i++] = key(old, j++); - } - - const size_t vn{vals(old) + 1}; - string_view _vals[vn + 1]; + const sha256::buf hash { - size_t i(0), j(0); - while(i < pos) - _vals[i++] = val(old, j++); + sha256{const_buffer{node}} + }; - _vals[i++] = _val; - while(i < vals(old) + 1) - _vals[i++] = val(old, j++); - } + const auto hashb64 + { + b64encode_unpadded(hashbuf, hash) + }; - return make_node(out, _keys, kn, _vals, vn); + db::txn::append + { + iov, db::delta + { + db::op::SET, + "state_node", // col + hashb64, // key + node, // val + } + }; + + return hashb64; } +/// Add key/val pair to an existing node, which creates a new node printed +/// into the buffer `out`. If the key matches an existing key, it will be +/// replaced and the new node will have the same size as old. +ircd::json::object +ircd::m::state::make_node(const mutable_buffer &out, + const node &old, + const size_t &pos, + const json::array &key, + const string_view &val) +{ + json::array keys[old.keys() + 1]; + size_t kn{0}; + { + size_t n(0); + while(kn < pos) + keys[kn++] = old.key(n++); + + keys[kn++] = key; + if(pos < old.keys() && keycmp(key, old.key(n)) == 0) + n++; + + while(kn < old.keys() + 1 && n < old.keys()) + keys[kn++] = old.key(n++); + } + + string_view vals[old.vals() + 1]; + size_t vn{0}; + { + size_t n(0); + while(vn < pos) + vals[vn++] = old.val(n++); + + vals[vn++] = val; + if(kn == old.keys()) + n++; + + while(vn < old.vals() + 1 && n < old.vals()) + vals[vn++] = old.val(n++); + } + + assert(kn == old.keys() || kn == old.keys() + 1); + return make_node(out, keys, kn, vals, vn); +} + +/// Prints a node into the buffer `out` using the keys and vals arguments +/// which must be pointers to arrays. Size of each array is specified in +/// the following argument. Each array must have at least one element each. +/// the vals array can have one more element than the keys array if desired. ircd::json::object ircd::m::state::make_node(const mutable_buffer &out, const json::array *const &keys_, @@ -331,30 +343,40 @@ ircd::m::state::make_node(const mutable_buffer &out, return { data(out), json::print(out, iov) }; } -size_t -ircd::m::state::find(const node &node, - const string_view &type, - const string_view &state_key) +/// Convenience to close over the key creation using a stack buffer (hence +/// safe for reentrance / multiple closing) +void +ircd::m::state::make_key(const string_view &type, + const string_view &state_key, + const key_closure &closure) { - thread_local char buf[1_KiB]; - const ctx::critical_assertion ca; - return find(node, make_key(buf, type, state_key)); + char buf[KEY_MAX_SZ]; + closure(make_key(buf, type, state_key)); } -size_t -ircd::m::state::find(const node &node, - const json::array &parts) +/// Creates a key array from the most common key pattern of a matrix +/// room (type,state_key). +ircd::json::array +ircd::m::state::make_key(const mutable_buffer &out, + const string_view &type, + const string_view &state_key) { - size_t ret{0}; - for(const json::array key : json::get<"k"_>(node)) - if(keycmp(parts, key) <= 0) - return ret; - else - ++ret; + const json::value key_parts[] + { + type, state_key + }; - return ret; + const json::value key + { + key_parts, 2 + }; + + return { data(out), json::print(out, key) }; } +/// Compares two keys. Keys are arrays of strings which become safely +/// concatenated for a linear lexical comparison. Returns -1 if a less +/// than b; 0 if equal; 1 if a greater than b. int ircd::m::state::keycmp(const json::array &a, const json::array &b) @@ -379,57 +401,73 @@ ircd::m::state::keycmp(const json::array &a, 1; } -ircd::json::array -ircd::m::state::make_key(const mutable_buffer &out, - const string_view &type, - const string_view &state_key) -{ - const json::value key_parts[] - { - type, state_key - }; - - const json::value key - { - key_parts, 2 - }; - - return { data(out), json::print(out, key) }; -} - -ircd::string_view -ircd::m::state::val(const node &node, - const size_t &pos) -{ - return json::at<"v"_>(node).at(pos); -} - -ircd::json::array -ircd::m::state::key(const node &node, - const size_t &pos) -{ - return json::at<"k"_>(node).at(pos); -} +// +// node +// +/// Find position for a val in node. Uses the keycmp(). If there is one +/// key in node, and the argument compares less than or equal to the key, +/// 0 is returned, otherwise 1 is returned. If there are two keys in node +/// and argument compares less than both, 0 is returned; equal to key[0], +/// 0 is returned; greater than key[0] and less than or equal to key[1], +/// 1 is returned; greater than both: 2 is returned. Note that there can +/// be one more vals() than keys() in a node (this is usually a "full node") +/// but there might not be, and the returned pos might be out of range. size_t -ircd::m::state::children(const node &node) +ircd::m::state::node::find(const json::array &parts) +const +{ + size_t ret{0}; + for(const json::array key : json::get<"k"_>(*this)) + if(keycmp(parts, key) <= 0) + return ret; + else + ++ret; + + return ret; +} + +// Get value at position pos (throws out_of_range) +ircd::string_view +ircd::m::state::node::val(const size_t &pos) +const +{ + return unquote(json::at<"v"_>(*this).at(pos)); +} + +// Get key at position pos (throws out_of_range) +ircd::json::array +ircd::m::state::node::key(const size_t &pos) +const +{ + return json::at<"k"_>(*this).at(pos); +} + +// Count values that actually lead to other nodes +size_t +ircd::m::state::node::children() +const { size_t ret(0); - for(const auto &v : json::get<"v"_>(node)) + for(const auto &v : json::get<"v"_>(*this)) if(!valid(id::EVENT, v)) ++ret; return ret; } +// Count values in node size_t -ircd::m::state::vals(const node &node) +ircd::m::state::node::vals() +const { - return json::get<"v"_>(node).count(); + return json::get<"v"_>(*this).count(); } +/// Count keys in node size_t -ircd::m::state::keys(const node &node) +ircd::m::state::node::keys() +const { - return json::get<"k"_>(node).count(); + return json::get<"k"_>(*this).count(); }