From ecbdf00f228e1f0147c1ccf90443781bf4169479 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 16 Apr 2019 13:42:18 -0700 Subject: [PATCH] ircd::m::dbs: Add event_type metadata column. --- include/ircd/m/dbs.h | 17 ++++ ircd/m_dbs.cc | 183 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/include/ircd/m/dbs.h b/include/ircd/m/dbs.h index 7691fbc63..109cb428c 100644 --- a/include/ircd/m/dbs.h +++ b/include/ircd/m/dbs.h @@ -35,6 +35,7 @@ namespace ircd::m::dbs extern db::column event_idx; // event_id => event_idx extern db::column event_json; // event_idx => full json extern db::index event_refs; // event_idx | ref_type, event_idx + extern db::index event_type; // type | event_idx extern db::index event_sender; // host | local, event_idx extern db::index room_head; // room_id | event_id => event_idx extern db::index room_events; // room_id | depth, event_idx => node_id @@ -56,6 +57,10 @@ namespace ircd::m::dbs string_view event_sender_key(const mutable_buffer &out, const id::user &, const event::idx &); std::tuple event_sender_key(const string_view &amalgam); + constexpr size_t EVENT_TYPE_KEY_MAX_SIZE {event::TYPE_MAX_SIZE + 1 + 8}; + string_view event_type_key(const mutable_buffer &out, const string_view &, const event::idx & = 0); + std::tuple event_type_key(const string_view &amalgam); + constexpr size_t ROOM_HEAD_KEY_MAX_SIZE {id::MAX_SIZE + 1 + id::MAX_SIZE}; string_view room_head_key(const mutable_buffer &out, const id::room &, const id::event &); string_view room_head_key(const string_view &amalgam); @@ -144,6 +149,9 @@ struct ircd::m::dbs::write_opts /// Whether to update the event_sender index. bool event_sender {true}; + + /// Whether to update the event_type index. + bool event_type {true}; }; /// Types of references indexed by event_refs. This is a single byte integer, @@ -267,6 +275,14 @@ namespace ircd::m::dbs::desc extern const db::prefix_transform events__event_sender__pfx; extern const db::descriptor events__event_sender; + // events type + extern conf::item events__event_type__block__size; + extern conf::item events__event_type__meta_block__size; + extern conf::item events__event_type__cache__size; + extern conf::item events__event_type__cache_comp__size; + extern const db::prefix_transform events__event_type__pfx; + extern const db::descriptor events__event_type; + // room head mapping sequence extern conf::item events__room_head__block__size; extern conf::item events__room_head__meta_block__size; @@ -321,6 +337,7 @@ namespace ircd::m::dbs string_view _index_redact(db::txn &, const event &, const write_opts &); string_view _index_other(db::txn &, const event &, const write_opts &); string_view _index_room(db::txn &, const event &, const write_opts &); + void _index_event_type(db::txn &, const event &, const write_opts &); void _index_event_sender(db::txn &, const event &, const write_opts &); void _index_event_refs_m_room_redaction(db::txn &, const event &, const write_opts &); void _index_event_refs_m_receipt_m_read(db::txn &, const event &, const write_opts &); diff --git a/ircd/m_dbs.cc b/ircd/m_dbs.cc index 515d16bd2..ebac1408f 100644 --- a/ircd/m_dbs.cc +++ b/ircd/m_dbs.cc @@ -41,6 +41,11 @@ decltype(ircd::m::dbs::event_sender) ircd::m::dbs::event_sender {}; +/// Linkage for a reference to the event_type column. +decltype(ircd::m::dbs::event_type) +ircd::m::dbs::event_type +{}; + /// Linkage for a reference to the room_head column decltype(ircd::m::dbs::room_head) ircd::m::dbs::room_head @@ -146,6 +151,7 @@ ircd::m::dbs::init::init(std::string dbopts) event_json = db::column{*events, desc::events__event_json.name}; event_refs = db::index{*events, desc::events__event_refs.name}; event_sender = db::index{*events, desc::events__event_sender.name}; + event_type = db::index{*events, desc::events__event_type.name}; room_head = db::index{*events, desc::events__room_head.name}; room_events = db::index{*events, desc::events__room_events.name}; room_joined = db::index{*events, desc::events__room_joined.name}; @@ -335,6 +341,9 @@ ircd::m::dbs::_index_event(db::txn &txn, if(opts.event_sender) _index_event_sender(txn, event, opts); + + if(opts.event_type) + _index_event_type(txn, event, opts); } void @@ -754,6 +763,30 @@ ircd::m::dbs::_index_event_sender(db::txn &txn, }; } +void +ircd::m::dbs::_index_event_type(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.event_type); + assert(opts.event_idx); + assert(json::get<"type"_>(event)); + + thread_local char buf[EVENT_TYPE_KEY_MAX_SIZE]; + const string_view &key + { + event_type_key(buf, at<"type"_>(event), opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_type, + { + opts.op, key, string_view{} + } + }; +} + ircd::string_view ircd::m::dbs::_index_room(db::txn &txn, const event &event, @@ -1771,6 +1804,152 @@ ircd::m::dbs::desc::events__event_sender size_t(events__event_sender__meta_block__size), }; +// +// event_type +// + +decltype(ircd::m::dbs::desc::events__event_type__block__size) +ircd::m::dbs::desc::events__event_type__block__size +{ + { "name", "ircd.m.dbs.events._event_type.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::events__event_type__meta_block__size) +ircd::m::dbs::desc::events__event_type__meta_block__size +{ + { "name", "ircd.m.dbs.events._event_type.meta_block.size" }, + { "default", 4096L }, +}; + +decltype(ircd::m::dbs::desc::events__event_type__cache__size) +ircd::m::dbs::desc::events__event_type__cache__size +{ + { + { "name", "ircd.m.dbs.events._event_type.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{events__event_type__cache__size}; + db::capacity(db::cache(event_type), value); + } +}; + +decltype(ircd::m::dbs::desc::events__event_type__cache_comp__size) +ircd::m::dbs::desc::events__event_type__cache_comp__size +{ + { + { "name", "ircd.m.dbs.events._event_type.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{events__event_type__cache_comp__size}; + db::capacity(db::cache_compressed(event_type), value); + } +}; + +ircd::string_view +ircd::m::dbs::event_type_key(const mutable_buffer &out_, + const string_view &type, + const event::idx &event_idx) +{ + assert(size(out_) >= EVENT_TYPE_KEY_MAX_SIZE); + + mutable_buffer out{out_}; + consume(out, copy(out, type)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, byte_view(event_idx))); + return { data(out_), data(out) }; +} + +std::tuple +ircd::m::dbs::event_type_key(const string_view &amalgam) +{ + assert(size(amalgam) == sizeof(event::idx) + 1); + const auto &key + { + amalgam.substr(1) + }; + + assert(size(key) == sizeof(event::idx)); + return + { + byte_view(key) + }; +} + +const ircd::db::prefix_transform +ircd::m::dbs::desc::events__event_type__pfx +{ + "_event_type", + [](const string_view &key) + { + return has(key, '\0'); + }, + + [](const string_view &key) + { + const auto &parts + { + split(key, '\0') + }; + + return parts.first; + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events__event_type +{ + // name + "_event_type", + + // explanation + R"(Index of types of events. + + type | event_idx => -- + + The types of events are indexed by this column. All events of a specific type can be + iterated efficiently. The type string forms the prefix domain. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + events__event_type__pfx, + + // drop column + false, + + // cache size + bool(events_cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(events_cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, + + // expect queries hit + false, + + // block size + size_t(events__event_type__block__size), + + // meta_block size + size_t(events__event_type__meta_block__size), +}; + // // room_head // @@ -3962,6 +4141,10 @@ ircd::m::dbs::desc::events // Mapping of senders to event_idx's they are the sender of. events__event_sender, + // type | event_idx + // Mapping of type strings to event_idx's of that type. + events__event_type, + // (room_id, (depth, event_idx)) => (state_root) // Sequence of all events for a room, ever. events__room_events,