From 843f78548b042d70081ef02978270bc9750430e8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 5 Feb 2019 17:18:53 -0800 Subject: [PATCH] ircd::m::dbs: Add event_refs metadata column. --- include/ircd/m/dbs.h | 14 ++++ ircd/m_dbs.cc | 175 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/include/ircd/m/dbs.h b/include/ircd/m/dbs.h index 082c948cd..e6ecf3aed 100644 --- a/include/ircd/m/dbs.h +++ b/include/ircd/m/dbs.h @@ -34,6 +34,7 @@ namespace ircd::m::dbs // Event metadata columns extern db::column event_idx; // event_id => event_idx extern db::column event_json; // event_idx => full json + extern db::index event_refs; // event_idx | event_idx extern db::index room_head; // room_id | event_id => event_idx extern db::index room_events; // room_id | depth, event_idx => node_id extern db::index room_joined; // room_id | origin, member => event_idx @@ -41,6 +42,10 @@ namespace ircd::m::dbs extern db::column state_node; // node_id => state::node // Lowlevel util + constexpr size_t EVENT_REFS_KEY_MAX_SIZE {sizeof(event::idx) + sizeof(event::idx)}; + string_view event_refs_key(const mutable_buffer &out, const event::idx &tgt, const event::idx &referer); + std::pair event_refs_key(const string_view &amalgam); + constexpr size_t ROOM_HEAD_KEY_MAX_SIZE {id::MAX_SIZE + 1 + id::MAX_SIZE}; string_view room_head_key(const mutable_buffer &out, const id::room &, const id::event &); string_view room_head_key(const string_view &amalgam); @@ -175,6 +180,15 @@ namespace ircd::m::dbs::desc extern conf::item events__event_json__bloom__bits; extern const db::descriptor events__event_json; + // events graph + extern conf::item events__event_refs__block__size; + extern conf::item events__event_refs__meta_block__size; + extern conf::item events__event_refs__cache__size; + extern conf::item events__event_refs__cache_comp__size; + extern conf::item events__event_refs__bloom__bits; + extern const db::prefix_transform events__event_refs__pfx; + extern const db::descriptor events__event_refs; + // room head mapping sequence extern conf::item events__room_head__block__size; extern conf::item events__room_head__meta_block__size; diff --git a/ircd/m_dbs.cc b/ircd/m_dbs.cc index 01808b4a6..a22e818b9 100644 --- a/ircd/m_dbs.cc +++ b/ircd/m_dbs.cc @@ -31,6 +31,11 @@ decltype(ircd::m::dbs::event_json) ircd::m::dbs::event_json {}; +/// Linkage for a reference to the event_refs column. +decltype(ircd::m::dbs::event_refs) +ircd::m::dbs::event_refs +{}; + /// Linkage for a reference to the room_head column decltype(ircd::m::dbs::room_head) ircd::m::dbs::room_head @@ -134,6 +139,7 @@ ircd::m::dbs::init::init(std::string dbopts) // Construct global convenience references for the metadata columns event_idx = db::column{*events, desc::events__event_idx.name}; event_json = db::column{*events, desc::events__event_json.name}; + event_refs = db::index{*events, desc::events__event_refs.name}; room_head = db::index{*events, desc::events__room_head.name}; room_events = db::index{*events, desc::events__room_events.name}; room_joined = db::index{*events, desc::events__room_joined.name}; @@ -923,6 +929,171 @@ ircd::m::dbs::desc::events__event_json size_t(events__event_json__meta_block__size), }; +// +// event_refs +// + +decltype(ircd::m::dbs::desc::events__event_refs__block__size) +ircd::m::dbs::desc::events__event_refs__block__size +{ + { "name", "ircd.m.dbs.events._event_refs.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::events__event_refs__meta_block__size) +ircd::m::dbs::desc::events__event_refs__meta_block__size +{ + { "name", "ircd.m.dbs.events._event_refs.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::events__event_refs__cache__size) +ircd::m::dbs::desc::events__event_refs__cache__size +{ + { + { "name", "ircd.m.dbs.events._event_refs.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{events__event_refs__cache__size}; + db::capacity(db::cache(event_refs), value); + } +}; + +decltype(ircd::m::dbs::desc::events__event_refs__cache_comp__size) +ircd::m::dbs::desc::events__event_refs__cache_comp__size +{ + { + { "name", "ircd.m.dbs.events._event_refs.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{events__event_refs__cache_comp__size}; + db::capacity(db::cache_compressed(event_refs), value); + } +}; + +decltype(ircd::m::dbs::desc::events__event_refs__bloom__bits) +ircd::m::dbs::desc::events__event_refs__bloom__bits +{ + { "name", "ircd.m.dbs.events._event_refs.bloom.bits" }, + { "default", 0L }, +}; + +const ircd::db::prefix_transform +ircd::m::dbs::desc::events__event_refs__pfx +{ + "_event_refs", + [](const string_view &key) + { + return size(key) >= sizeof(event::idx) * 2; + }, + + [](const string_view &key) + { + assert(size(key) >= sizeof(event::idx)); + return string_view + { + data(key), data(key) + sizeof(event::idx) + }; + } +}; + +ircd::string_view +ircd::m::dbs::event_refs_key(const mutable_buffer &out, + const event::idx &tgt, + const event::idx &src) +{ + assert(size(out) >= sizeof(event::idx) * 2); + event::idx *const &key + { + reinterpret_cast(data(out)) + }; + + key[0] = tgt; + key[1] = src; + return string_view + { + data(out), data(out) + sizeof(event::idx) * 2 + }; +} + +std::pair +ircd::m::dbs::event_refs_key(const string_view &amalgam) +{ + assert(size(amalgam) >= sizeof(event::idx) * 2); + const event::idx *const &key + { + reinterpret_cast(data(amalgam)) + }; + + return + { + key[0], key[1] + }; +} + +const ircd::db::descriptor +ircd::m::dbs::desc::events__event_refs +{ + // name + "_event_refs", + + // explanation + R"(Inverse reference graph of events. + + event_idx | event_idx => -- + + The first part of the key is the event being referenced. The second part + of the key is the event which refers to the first event somewhere in its + prev_events references. + + The prefix transform is in effect; an event may be referenced multiple + times. We can find all the events we have which reference a target. The + database must contain both events (hence they have event::idx numbers). + + The value is currently unused/empty; we may eventually store metadata with + information about this reference (i.e. is depth adjacent? is the ref + redundant with another in the same event and should not be made? etc). + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(events_cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(events_cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(events__event_refs__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(events__event_refs__block__size), + + // meta_block size + size_t(events__event_refs__meta_block__size), +}; + // // room_head // @@ -3065,6 +3236,10 @@ ircd::m::dbs::desc::events // Mapping of event_idx to full json events__event_json, + // event_idx | event_idx + // Reverse mapping of the event reference graph. + events__event_refs, + // (room_id, (depth, event_idx)) => (state_root) // Sequence of all events for a room, ever. events__room_events,