From b8239d45cd33e079e9cb9848c2d350963ccd1241 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 25 Mar 2020 11:06:29 -0700 Subject: [PATCH] ircd::m::dbs: Split dbs unit per column; naming simplifications; major reorg. --- doc/TUNING.md | 4 +- include/ircd/m/dbs/dbs.h | 14 +- include/ircd/m/dbs/event_column.h | 84 +- include/ircd/m/dbs/event_horizon.h | 15 +- include/ircd/m/dbs/event_idx.h | 14 +- include/ircd/m/dbs/event_json.h | 14 +- include/ircd/m/dbs/event_refs.h | 22 +- include/ircd/m/dbs/event_sender.h | 14 +- include/ircd/m/dbs/event_state.h | 14 +- include/ircd/m/dbs/event_type.h | 14 +- include/ircd/m/dbs/room_events.h | 16 +- include/ircd/m/dbs/room_head.h | 13 +- include/ircd/m/dbs/room_joined.h | 16 +- include/ircd/m/dbs/room_state.h | 16 +- include/ircd/m/dbs/room_state_space.h | 18 +- include/ircd/m/dbs/room_type.h | 16 +- matrix/Makefile.am | 15 + matrix/dbs.cc | 5315 +------------------------ matrix/dbs_desc.cc | 547 +++ matrix/dbs_event_column.cc | 868 ++++ matrix/dbs_event_horizon.cc | 310 ++ matrix/dbs_event_idx.cc | 170 + matrix/dbs_event_json.cc | 188 + matrix/dbs_event_refs.cc | 782 ++++ matrix/dbs_event_sender.cc | 297 ++ matrix/dbs_event_state.cc | 278 ++ matrix/dbs_event_type.cc | 191 + matrix/dbs_room_events.cc | 312 ++ matrix/dbs_room_head.cc | 229 ++ matrix/dbs_room_joined.cc | 244 ++ matrix/dbs_room_state.cc | 234 ++ matrix/dbs_room_state_space.cc | 331 ++ matrix/dbs_room_type.cc | 314 ++ matrix/events.cc | 6 +- 34 files changed, 5547 insertions(+), 5388 deletions(-) create mode 100644 matrix/dbs_desc.cc create mode 100644 matrix/dbs_event_column.cc create mode 100644 matrix/dbs_event_horizon.cc create mode 100644 matrix/dbs_event_idx.cc create mode 100644 matrix/dbs_event_json.cc create mode 100644 matrix/dbs_event_refs.cc create mode 100644 matrix/dbs_event_sender.cc create mode 100644 matrix/dbs_event_state.cc create mode 100644 matrix/dbs_event_type.cc create mode 100644 matrix/dbs_room_events.cc create mode 100644 matrix/dbs_room_head.cc create mode 100644 matrix/dbs_room_joined.cc create mode 100644 matrix/dbs_room_state.cc create mode 100644 matrix/dbs_room_state_space.cc create mode 100644 matrix/dbs_room_type.cc diff --git a/doc/TUNING.md b/doc/TUNING.md index 3afcb9d23..ecafccf05 100644 --- a/doc/TUNING.md +++ b/doc/TUNING.md @@ -74,7 +74,7 @@ output from the above command, use the following command where `` is replaced by one of the names under `COLUMN` in the above output: ``` -conf ircd.m.dbs.events..cache.size +conf ircd.m.dbs..cache.size ``` To alter a cache size, set the configuration item with a byte value. In the @@ -82,7 +82,7 @@ example below we will set the `_event_json` cache size to 256 MiB. This change will take effect immediately and the cache will grow or shrink to that size. ``` -conf set ircd.m.dbs.events._event_json.cache.size 268435456 +conf set ircd.m.dbs._event_json.cache.size 268435456 ``` > Tip: The best metric to figure out which caches are inadequate is not diff --git a/include/ircd/m/dbs/dbs.h b/include/ircd/m/dbs/dbs.h index 5ce3b1358..1517ec0db 100644 --- a/include/ircd/m/dbs/dbs.h +++ b/include/ircd/m/dbs/dbs.h @@ -19,10 +19,10 @@ namespace ircd::m::dbs enum class ref :uint8_t; // General confs - extern conf::item events_cache_enable; - extern conf::item events_cache_comp_enable; - extern conf::item events_mem_write_buffer_size; - extern conf::item events_sst_write_buffer_size; + extern conf::item cache_enable; + extern conf::item cache_comp_enable; + extern conf::item mem_write_buffer_size; + extern conf::item sst_write_buffer_size; // Database instance extern std::shared_ptr events; @@ -197,3 +197,9 @@ struct ircd::m::dbs::init init(const string_view &servername, std::string dbopts = {}); ~init() noexcept; }; + +// Internal utils (here for now) +namespace ircd::m::dbs +{ + event::idx find_event_idx(const event::id &, const write_opts &); +} diff --git a/include/ircd/m/dbs/event_column.h b/include/ircd/m/dbs/event_column.h index 1bfc8e114..ec19fa073 100644 --- a/include/ircd/m/dbs/event_column.h +++ b/include/ircd/m/dbs/event_column.h @@ -25,6 +25,8 @@ namespace ircd::m::dbs event::size() }; + void _index_event_cols(db::txn &, const event &, const write_opts &); + // There is one position in this array corresponding to each property // in the m::event tuple, however, the db::column in this position may // be default-initialized if this column is not used. @@ -33,53 +35,53 @@ namespace ircd::m::dbs namespace ircd::m::dbs::desc { - extern conf::item events___event__bloom__bits; + extern conf::item _event__bloom__bits; - extern conf::item events__content__block__size; - extern conf::item events__content__meta_block__size; - extern conf::item events__content__cache__size; - extern conf::item events__content__cache_comp__size; - extern const db::descriptor events_content; + extern conf::item content__block__size; + extern conf::item content__meta_block__size; + extern conf::item content__cache__size; + extern conf::item content__cache_comp__size; + extern const db::descriptor content; - extern conf::item events__depth__block__size; - extern conf::item events__depth__meta_block__size; - extern conf::item events__depth__cache__size; - extern conf::item events__depth__cache_comp__size; - extern const db::descriptor events_depth; + extern conf::item depth__block__size; + extern conf::item depth__meta_block__size; + extern conf::item depth__cache__size; + extern conf::item depth__cache_comp__size; + extern const db::descriptor depth; - extern conf::item events__event_id__block__size; - extern conf::item events__event_id__meta_block__size; - extern conf::item events__event_id__cache__size; - extern conf::item events__event_id__cache_comp__size; - extern const db::descriptor events_event_id; + extern conf::item event_id__block__size; + extern conf::item event_id__meta_block__size; + extern conf::item event_id__cache__size; + extern conf::item event_id__cache_comp__size; + extern const db::descriptor event_id; - extern conf::item events__origin_server_ts__block__size; - extern conf::item events__origin_server_ts__meta_block__size; - extern conf::item events__origin_server_ts__cache__size; - extern conf::item events__origin_server_ts__cache_comp__size; - extern const db::descriptor events_origin_server_ts; + extern conf::item origin_server_ts__block__size; + extern conf::item origin_server_ts__meta_block__size; + extern conf::item origin_server_ts__cache__size; + extern conf::item origin_server_ts__cache_comp__size; + extern const db::descriptor origin_server_ts; - extern conf::item events__room_id__block__size; - extern conf::item events__room_id__meta_block__size; - extern conf::item events__room_id__cache__size; - extern conf::item events__room_id__cache_comp__size; - extern const db::descriptor events_room_id; + extern conf::item room_id__block__size; + extern conf::item room_id__meta_block__size; + extern conf::item room_id__cache__size; + extern conf::item room_id__cache_comp__size; + extern const db::descriptor room_id; - extern conf::item events__sender__block__size; - extern conf::item events__sender__meta_block__size; - extern conf::item events__sender__cache__size; - extern conf::item events__sender__cache_comp__size; - extern const db::descriptor events_sender; + extern conf::item sender__block__size; + extern conf::item sender__meta_block__size; + extern conf::item sender__cache__size; + extern conf::item sender__cache_comp__size; + extern const db::descriptor sender; - extern conf::item events__state_key__block__size; - extern conf::item events__state_key__meta_block__size; - extern conf::item events__state_key__cache__size; - extern conf::item events__state_key__cache_comp__size; - extern const db::descriptor events_state_key; + extern conf::item state_key__block__size; + extern conf::item state_key__meta_block__size; + extern conf::item state_key__cache__size; + extern conf::item state_key__cache_comp__size; + extern const db::descriptor state_key; - extern conf::item events__type__block__size; - extern conf::item events__type__meta_block__size; - extern conf::item events__type__cache__size; - extern conf::item events__type__cache_comp__size; - extern const db::descriptor events_type; + extern conf::item type__block__size; + extern conf::item type__meta_block__size; + extern conf::item type__cache__size; + extern conf::item type__cache_comp__size; + extern const db::descriptor type; } diff --git a/include/ircd/m/dbs/event_horizon.h b/include/ircd/m/dbs/event_horizon.h index f6e0af73d..ee6c733d2 100644 --- a/include/ircd/m/dbs/event_horizon.h +++ b/include/ircd/m/dbs/event_horizon.h @@ -22,16 +22,19 @@ namespace ircd::m::dbs string_view event_horizon_key(const mutable_buffer &out, const id::event &); std::tuple event_horizon_key(const string_view &amalgam); + void _index_event_horizon_resolve(db::txn &, const event &, const write_opts &); //query + void _index_event_horizon(db::txn &, const event &, const write_opts &, const id::event &); + // event_id | event_idx extern db::domain event_horizon; } namespace ircd::m::dbs::desc { - extern conf::item events__event_horizon__block__size; - extern conf::item events__event_horizon__meta_block__size; - extern conf::item events__event_horizon__cache__size; - extern conf::item events__event_horizon__cache_comp__size; - extern const db::prefix_transform events__event_horizon__pfx; - extern const db::descriptor events__event_horizon; + extern conf::item event_horizon__block__size; + extern conf::item event_horizon__meta_block__size; + extern conf::item event_horizon__cache__size; + extern conf::item event_horizon__cache_comp__size; + extern const db::prefix_transform event_horizon__pfx; + extern const db::descriptor event_horizon; } diff --git a/include/ircd/m/dbs/event_idx.h b/include/ircd/m/dbs/event_idx.h index 3f2e71879..2370d3f7f 100644 --- a/include/ircd/m/dbs/event_idx.h +++ b/include/ircd/m/dbs/event_idx.h @@ -13,15 +13,17 @@ namespace ircd::m::dbs { + void _index_event_id(db::txn &, const event &, const write_opts &); + extern db::column event_idx; // event_id => event_idx } namespace ircd::m::dbs::desc { - extern conf::item events__event_idx__block__size; - extern conf::item events__event_idx__meta_block__size; - extern conf::item events__event_idx__cache__size; - extern conf::item events__event_idx__cache_comp__size; - extern conf::item events__event_idx__bloom__bits; - extern const db::descriptor events__event_idx; + extern conf::item event_idx__block__size; + extern conf::item event_idx__meta_block__size; + extern conf::item event_idx__cache__size; + extern conf::item event_idx__cache_comp__size; + extern conf::item event_idx__bloom__bits; + extern const db::descriptor event_idx; } diff --git a/include/ircd/m/dbs/event_json.h b/include/ircd/m/dbs/event_json.h index 41aa9b5f6..5e335f353 100644 --- a/include/ircd/m/dbs/event_json.h +++ b/include/ircd/m/dbs/event_json.h @@ -13,16 +13,18 @@ namespace ircd::m::dbs { + void _index_event_json(db::txn &, const event &, const write_opts &); + // event_idx => full json extern db::column event_json; } namespace ircd::m::dbs::desc { - extern conf::item events__event_json__block__size; - extern conf::item events__event_json__meta_block__size; - extern conf::item events__event_json__cache__size; - extern conf::item events__event_json__cache_comp__size; - extern conf::item events__event_json__bloom__bits; - extern const db::descriptor events__event_json; + extern conf::item event_json__block__size; + extern conf::item event_json__meta_block__size; + extern conf::item event_json__cache__size; + extern conf::item event_json__cache_comp__size; + extern conf::item event_json__bloom__bits; + extern const db::descriptor event_json; } diff --git a/include/ircd/m/dbs/event_refs.h b/include/ircd/m/dbs/event_refs.h index 53554bc96..b4d64240d 100644 --- a/include/ircd/m/dbs/event_refs.h +++ b/include/ircd/m/dbs/event_refs.h @@ -28,9 +28,6 @@ namespace ircd::m::dbs 0xFFUL << ref_shift }; - // event_idx | ref_type, event_idx - extern db::domain event_refs; - string_view event_refs_key(const mutable_buffer &out, const event::idx &tgt, @@ -42,17 +39,22 @@ namespace ircd::m::dbs string_view reflect(const ref &); + + void _index_event_refs(db::txn &, const event &, const write_opts &); + + // event_idx | ref_type, event_idx + extern db::domain event_refs; } namespace ircd::m::dbs::desc { - extern conf::item events__event_refs__block__size; - extern conf::item events__event_refs__meta_block__size; - extern conf::item events__event_refs__cache__size; - extern conf::item events__event_refs__cache_comp__size; - extern const db::prefix_transform events__event_refs__pfx; - extern const db::comparator events__event_refs__cmp; - extern const db::descriptor events__event_refs; + extern conf::item event_refs__block__size; + extern conf::item event_refs__meta_block__size; + extern conf::item event_refs__cache__size; + extern conf::item event_refs__cache_comp__size; + extern const db::prefix_transform event_refs__pfx; + extern const db::comparator event_refs__cmp; + extern const db::descriptor event_refs; } /// Types of references indexed by event_refs. This is a single byte integer, diff --git a/include/ircd/m/dbs/event_sender.h b/include/ircd/m/dbs/event_sender.h index 1f149a364..96fab0213 100644 --- a/include/ircd/m/dbs/event_sender.h +++ b/include/ircd/m/dbs/event_sender.h @@ -29,6 +29,8 @@ namespace ircd::m::dbs string_view event_sender_origin_key(const mutable_buffer &out, const id::user &, const event::idx &); std::tuple event_sender_origin_key(const string_view &amalgam); + void _index_event_sender(db::txn &, const event &, const write_opts &); + // mxid | event_idx // host | local, event_idx (see event_sender_origin.h) extern db::domain event_sender; @@ -36,10 +38,10 @@ namespace ircd::m::dbs namespace ircd::m::dbs::desc { - extern conf::item events__event_sender__block__size; - extern conf::item events__event_sender__meta_block__size; - extern conf::item events__event_sender__cache__size; - extern conf::item events__event_sender__cache_comp__size; - extern const db::prefix_transform events__event_sender__pfx; - extern const db::descriptor events__event_sender; + extern conf::item event_sender__block__size; + extern conf::item event_sender__meta_block__size; + extern conf::item event_sender__cache__size; + extern conf::item event_sender__cache_comp__size; + extern const db::prefix_transform event_sender__pfx; + extern const db::descriptor event_sender; } diff --git a/include/ircd/m/dbs/event_state.h b/include/ircd/m/dbs/event_state.h index 3eef478f0..24820fe98 100644 --- a/include/ircd/m/dbs/event_state.h +++ b/include/ircd/m/dbs/event_state.h @@ -30,6 +30,8 @@ namespace ircd::m::dbs string_view event_state_key(const mutable_buffer &out, const event_state_tuple &); event_state_tuple event_state_key(const string_view &); + void _index_event_state(db::txn &, const event &, const write_opts &); + // state_key, type, room_id, depth, event_idx extern db::domain event_state; } @@ -37,10 +39,10 @@ namespace ircd::m::dbs namespace ircd::m::dbs::desc { // events _event_state - extern conf::item events__event_state__block__size; - extern conf::item events__event_state__meta_block__size; - extern conf::item events__event_state__cache__size; - extern conf::item events__event_state__cache_comp__size; - extern const db::comparator events__event_state__cmp; - extern const db::descriptor events__event_state; + extern conf::item event_state__block__size; + extern conf::item event_state__meta_block__size; + extern conf::item event_state__cache__size; + extern conf::item event_state__cache_comp__size; + extern const db::comparator event_state__cmp; + extern const db::descriptor event_state; } diff --git a/include/ircd/m/dbs/event_type.h b/include/ircd/m/dbs/event_type.h index 62ac3cb05..ffca4b517 100644 --- a/include/ircd/m/dbs/event_type.h +++ b/include/ircd/m/dbs/event_type.h @@ -21,6 +21,8 @@ namespace ircd::m::dbs string_view event_type_key(const mutable_buffer &out, const string_view &, const event::idx & = 0); std::tuple event_type_key(const string_view &amalgam); + void _index_event_type(db::txn &, const event &, const write_opts &); + // type | event_idx => - extern db::domain event_type; } @@ -28,10 +30,10 @@ namespace ircd::m::dbs namespace ircd::m::dbs::desc { // events type - extern conf::item events__event_type__block__size; - extern conf::item events__event_type__meta_block__size; - extern conf::item events__event_type__cache__size; - extern conf::item events__event_type__cache_comp__size; - extern const db::prefix_transform events__event_type__pfx; - extern const db::descriptor events__event_type; + extern conf::item event_type__block__size; + extern conf::item event_type__meta_block__size; + extern conf::item event_type__cache__size; + extern conf::item event_type__cache_comp__size; + extern const db::prefix_transform event_type__pfx; + extern const db::descriptor event_type; } diff --git a/include/ircd/m/dbs/room_events.h b/include/ircd/m/dbs/room_events.h index 1d67badb2..9e6627623 100644 --- a/include/ircd/m/dbs/room_events.h +++ b/include/ircd/m/dbs/room_events.h @@ -22,6 +22,8 @@ namespace ircd::m::dbs string_view room_events_key(const mutable_buffer &out, const id::room &, const uint64_t &depth); std::tuple room_events_key(const string_view &amalgam); + void _index_room_events(db::txn &, const event &, const write_opts &); + // room_id | depth, event_idx => node_id extern db::domain room_events; } @@ -29,11 +31,11 @@ namespace ircd::m::dbs namespace ircd::m::dbs::desc { // room events sequence - extern conf::item events__room_events__block__size; - extern conf::item events__room_events__meta_block__size; - extern conf::item events__room_events__cache__size; - extern conf::item events__room_events__cache_comp__size; - extern const db::prefix_transform events__room_events__pfx; - extern const db::comparator events__room_events__cmp; - extern const db::descriptor events__room_events; + extern conf::item room_events__block__size; + extern conf::item room_events__meta_block__size; + extern conf::item room_events__cache__size; + extern conf::item room_events__cache_comp__size; + extern const db::prefix_transform room_events__pfx; + extern const db::comparator room_events__cmp; + extern const db::descriptor room_events; } diff --git a/include/ircd/m/dbs/room_head.h b/include/ircd/m/dbs/room_head.h index b06dac576..60506090a 100644 --- a/include/ircd/m/dbs/room_head.h +++ b/include/ircd/m/dbs/room_head.h @@ -21,15 +21,18 @@ namespace ircd::m::dbs string_view room_head_key(const mutable_buffer &out, const id::room &, const id::event &); string_view room_head_key(const string_view &amalgam); + void _index_room_head_resolve(db::txn &, const event &, const write_opts &); + void _index_room_head(db::txn &, const event &, const write_opts &); + // room_id | event_id => event_idx extern db::domain room_head; } namespace ircd::m::dbs::desc { - extern conf::item events__room_head__block__size; - extern conf::item events__room_head__meta_block__size; - extern conf::item events__room_head__cache__size; - extern const db::prefix_transform events__room_head__pfx; - extern const db::descriptor events__room_head; + extern conf::item room_head__block__size; + extern conf::item room_head__meta_block__size; + extern conf::item room_head__cache__size; + extern const db::prefix_transform room_head__pfx; + extern const db::descriptor room_head; } diff --git a/include/ircd/m/dbs/room_joined.h b/include/ircd/m/dbs/room_joined.h index 88692c455..42eeacd94 100644 --- a/include/ircd/m/dbs/room_joined.h +++ b/include/ircd/m/dbs/room_joined.h @@ -22,17 +22,19 @@ namespace ircd::m::dbs string_view room_joined_key(const mutable_buffer &out, const id::room &, const string_view &origin); std::tuple room_joined_key(const string_view &amalgam); + void _index_room_joined(db::txn &, const event &, const write_opts &); + // room_id | origin, member => event_idx extern db::domain room_joined; } namespace ircd::m::dbs::desc { - extern conf::item events__room_joined__block__size; - extern conf::item events__room_joined__meta_block__size; - extern conf::item events__room_joined__cache__size; - extern conf::item events__room_joined__cache_comp__size; - extern conf::item events__room_joined__bloom__bits; - extern const db::prefix_transform events__room_joined__pfx; - extern const db::descriptor events__room_joined; + extern conf::item room_joined__block__size; + extern conf::item room_joined__meta_block__size; + extern conf::item room_joined__cache__size; + extern conf::item room_joined__cache_comp__size; + extern conf::item room_joined__bloom__bits; + extern const db::prefix_transform room_joined__pfx; + extern const db::descriptor room_joined; } diff --git a/include/ircd/m/dbs/room_state.h b/include/ircd/m/dbs/room_state.h index 046930b03..377fb4834 100644 --- a/include/ircd/m/dbs/room_state.h +++ b/include/ircd/m/dbs/room_state.h @@ -22,17 +22,19 @@ namespace ircd::m::dbs string_view room_state_key(const mutable_buffer &out, const id::room &, const string_view &type); std::tuple room_state_key(const string_view &amalgam); + void _index_room_state(db::txn &, const event &, const write_opts &); + // room_id | type, state_key => event_idx extern db::domain room_state; } namespace ircd::m::dbs::desc { - extern conf::item events__room_state__block__size; - extern conf::item events__room_state__meta_block__size; - extern conf::item events__room_state__cache__size; - extern conf::item events__room_state__cache_comp__size; - extern conf::item events__room_state__bloom__bits; - extern const db::prefix_transform events__room_state__pfx; - extern const db::descriptor events__room_state; + extern conf::item room_state__block__size; + extern conf::item room_state__meta_block__size; + extern conf::item room_state__cache__size; + extern conf::item room_state__cache_comp__size; + extern conf::item room_state__bloom__bits; + extern const db::prefix_transform room_state__pfx; + extern const db::descriptor room_state; } diff --git a/include/ircd/m/dbs/room_state_space.h b/include/ircd/m/dbs/room_state_space.h index ebe09ed70..62cbbb93b 100644 --- a/include/ircd/m/dbs/room_state_space.h +++ b/include/ircd/m/dbs/room_state_space.h @@ -30,18 +30,20 @@ namespace ircd::m::dbs string_view room_state_space_key(const mutable_buffer &out, const id::room &); room_state_space_key_parts room_state_space_key(const string_view &amalgam); + void _index_room_state_space(db::txn &, const event &, const write_opts &); + // room_id | type, state_key, depth, event_idx => -- extern db::domain room_state_space; } namespace ircd::m::dbs::desc { - extern conf::item events__room_state_space__block__size; - extern conf::item events__room_state_space__meta_block__size; - extern conf::item events__room_state_space__cache__size; - extern conf::item events__room_state_space__cache_comp__size; - extern conf::item events__room_state_space__bloom__bits; - extern const db::prefix_transform events__room_state_space__pfx; - extern const db::comparator events__room_state_space__cmp; - extern const db::descriptor events__room_state_space; + extern conf::item room_state_space__block__size; + extern conf::item room_state_space__meta_block__size; + extern conf::item room_state_space__cache__size; + extern conf::item room_state_space__cache_comp__size; + extern conf::item room_state_space__bloom__bits; + extern const db::prefix_transform room_state_space__pfx; + extern const db::comparator room_state_space__cmp; + extern const db::descriptor room_state_space; } diff --git a/include/ircd/m/dbs/room_type.h b/include/ircd/m/dbs/room_type.h index 50b10cf9e..1e1e5a3c9 100644 --- a/include/ircd/m/dbs/room_type.h +++ b/include/ircd/m/dbs/room_type.h @@ -35,6 +35,8 @@ namespace ircd::m::dbs const uint64_t &depth = -1, const event::idx & = -1); + void _index_room_type(db::txn &, const event &, const write_opts &); + // room_id | type, depth, event_idx extern db::domain room_type; } @@ -42,11 +44,11 @@ namespace ircd::m::dbs namespace ircd::m::dbs::desc { // room events sequence - extern conf::item events__room_type__block__size; - extern conf::item events__room_type__meta_block__size; - extern conf::item events__room_type__cache__size; - extern conf::item events__room_type__cache_comp__size; - extern const db::prefix_transform events__room_type__pfx; - extern const db::comparator events__room_type__cmp; - extern const db::descriptor events__room_type; + extern conf::item room_type__block__size; + extern conf::item room_type__meta_block__size; + extern conf::item room_type__cache__size; + extern conf::item room_type__cache_comp__size; + extern const db::prefix_transform room_type__pfx; + extern const db::comparator room_type__cmp; + extern const db::descriptor room_type; } diff --git a/matrix/Makefile.am b/matrix/Makefile.am index 338aa8a5c..3066af00c 100644 --- a/matrix/Makefile.am +++ b/matrix/Makefile.am @@ -60,6 +60,21 @@ libircd_matrix_la_SOURCES =# libircd_matrix_la_SOURCES += name.cc libircd_matrix_la_SOURCES += id.cc libircd_matrix_la_SOURCES += dbs.cc +libircd_matrix_la_SOURCES += dbs_event_idx.cc +libircd_matrix_la_SOURCES += dbs_event_json.cc +libircd_matrix_la_SOURCES += dbs_event_column.cc +libircd_matrix_la_SOURCES += dbs_event_refs.cc +libircd_matrix_la_SOURCES += dbs_event_horizon.cc +libircd_matrix_la_SOURCES += dbs_event_sender.cc +libircd_matrix_la_SOURCES += dbs_event_type.cc +libircd_matrix_la_SOURCES += dbs_event_state.cc +libircd_matrix_la_SOURCES += dbs_room_events.cc +libircd_matrix_la_SOURCES += dbs_room_type.cc +libircd_matrix_la_SOURCES += dbs_room_state.cc +libircd_matrix_la_SOURCES += dbs_room_state_space.cc +libircd_matrix_la_SOURCES += dbs_room_joined.cc +libircd_matrix_la_SOURCES += dbs_room_head.cc +libircd_matrix_la_SOURCES += dbs_desc.cc libircd_matrix_la_SOURCES += hook.cc libircd_matrix_la_SOURCES += event.cc libircd_matrix_la_SOURCES += event_cached.cc diff --git a/matrix/dbs.cc b/matrix/dbs.cc index 16b2ac39f..ef374755e 100644 --- a/matrix/dbs.cc +++ b/matrix/dbs.cc @@ -1,7 +1,7 @@ -// Matrix Construct +// The Construct // -// Copyright (C) Matrix Construct Developers, Authors & Contributors -// Copyright (C) 2016-2018 Jason Volk +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk // // Permission to use, copy, modify, and/or distribute this software for any // purpose with or without fee is hereby granted, provided that the above @@ -10,130 +10,56 @@ /// Residence of the events database instance pointer. decltype(ircd::m::dbs::events) -ircd::m::dbs::events -{}; - -/// Linkage for a cache of the columns of the events database which directly -/// correspond to a property in the matrix event object. This array allows -/// for constant time access to a column the same way one can make constant -/// time access to a property in m::event. -decltype(ircd::m::dbs::event_column) -ircd::m::dbs::event_column -{}; - -/// Linkage for a reference to the event_seq column. -decltype(ircd::m::dbs::event_idx) -ircd::m::dbs::event_idx -{}; - -/// Linkage for a reference to the event_json column. -decltype(ircd::m::dbs::event_json) -ircd::m::dbs::event_json -{}; - -/// Linkage for a reference to the event_refs column. -decltype(ircd::m::dbs::event_refs) -ircd::m::dbs::event_refs -{}; - -/// Linkage for a reference to the event_horizon column. -decltype(ircd::m::dbs::event_horizon) -ircd::m::dbs::event_horizon -{}; - -/// Linkage for a reference to the event_sender column. -decltype(ircd::m::dbs::event_sender) -ircd::m::dbs::event_sender -{}; - -/// Linkage for a reference to the event_type column. -decltype(ircd::m::dbs::event_type) -ircd::m::dbs::event_type -{}; - -/// Linkage for a reference to the event_state column. -decltype(ircd::m::dbs::event_state) -ircd::m::dbs::event_state -{}; - -/// Linkage for a reference to the room_head column -decltype(ircd::m::dbs::room_head) -ircd::m::dbs::room_head -{}; - -/// Linkage for a reference to the room_events column -decltype(ircd::m::dbs::room_events) -ircd::m::dbs::room_events -{}; - -/// Linkage for a reference to the room_type column -decltype(ircd::m::dbs::room_type) -ircd::m::dbs::room_type -{}; - -/// Linkage for a reference to the room_joined column -decltype(ircd::m::dbs::room_joined) -ircd::m::dbs::room_joined -{}; - -/// Linkage for a reference to the room_state column -decltype(ircd::m::dbs::room_state) -ircd::m::dbs::room_state -{}; - -/// Linkage for a reference to the room_state_space column -decltype(ircd::m::dbs::room_state_space) -ircd::m::dbs::room_state_space -{}; +ircd::m::dbs::events; /// Coarse variable for enabling the uncompressed cache on the events database; /// note this conf item is only effective by setting an environmental variable /// before daemon startup. It has no effect in any other regard. -decltype(ircd::m::dbs::events_cache_enable) -ircd::m::dbs::events_cache_enable +decltype(ircd::m::dbs::cache_enable) +ircd::m::dbs::cache_enable { - { "name", "ircd.m.dbs.events.__cache_enable" }, - { "default", true }, + { "name", "ircd.m.dbs.cache.enable" }, + { "default", true }, }; /// Coarse variable for enabling the compressed cache on the events database; /// note this conf item is only effective by setting an environmental variable /// before daemon startup. It has no effect in any other regard. -decltype(ircd::m::dbs::events_cache_comp_enable) -ircd::m::dbs::events_cache_comp_enable +decltype(ircd::m::dbs::cache_comp_enable) +ircd::m::dbs::cache_comp_enable { - { "name", "ircd.m.dbs.events.__cache_comp_enable" }, - { "default", false }, -}; - -/// Value determines the size of writes when creating SST files (i.e during -/// compaction). Consider that write calls are yield-points for IRCd and the -/// time spent filling the write buffer between calls may hog the CPU doing -/// compression during that time etc. (writable_file_max_buffer_size) -decltype(ircd::m::dbs::events_sst_write_buffer_size) -ircd::m::dbs::events_sst_write_buffer_size -{ - { - { "name", "ircd.m.dbs.events.sst.write_buffer_size" }, - { "default", long(1_MiB) }, - }, [] - { - static const auto key{"writable_file_max_buffer_size"_sv}; - const size_t &value{events_sst_write_buffer_size}; - if(events) - db::setopt(*events, key, lex_cast(value)); - } + { "name", "ircd.m.dbs.cache.comp.enable" }, + { "default", false }, }; /// The size of the memory buffer for new writes to the DB (backed by the WAL /// on disk). When this buffer is full it is flushed to sorted SST files on /// disk. If this is 0, a per-column value can be used; otherwise this value /// takes precedence as a total value for all columns. (db_write_buffer_size) -decltype(ircd::m::dbs::events_mem_write_buffer_size) -ircd::m::dbs::events_mem_write_buffer_size +decltype(ircd::m::dbs::mem_write_buffer_size) +ircd::m::dbs::mem_write_buffer_size { - { "name", "ircd.m.dbs.events.mem.write_buffer_size" }, - { "default", 0L }, + { "name", "ircd.m.dbs.mem.write_buffer_size" }, + { "default", 0L }, +}; + +/// Value determines the size of writes when creating SST files (i.e during +/// compaction). Consider that write calls are yield-points for IRCd and the +/// time spent filling the write buffer between calls may hog the CPU doing +/// compression during that time etc. (writable_file_max_buffer_size) +decltype(ircd::m::dbs::sst_write_buffer_size) +ircd::m::dbs::sst_write_buffer_size +{ + { + { "name", "ircd.m.dbs.sst.write_buffer_size" }, + { "default", long(1_MiB) }, + }, [] + { + static const string_view key{"writable_file_max_buffer_size"}; + const size_t &value{sst_write_buffer_size}; + if(events) + db::setopt(*events, key, lex_cast(value)); + } }; // @@ -188,19 +114,19 @@ ircd::m::dbs::init::init(const string_view &servername, }; // Construct global convenience references for the metadata columns - event_idx = db::column{*events, desc::events__event_idx.name}; - event_json = db::column{*events, desc::events__event_json.name}; - event_refs = db::domain{*events, desc::events__event_refs.name}; - event_horizon = db::domain{*events, desc::events__event_horizon.name}; - event_sender = db::domain{*events, desc::events__event_sender.name}; - event_type = db::domain{*events, desc::events__event_type.name}; - event_state = db::domain{*events, desc::events__event_state.name}; - room_head = db::domain{*events, desc::events__room_head.name}; - room_events = db::domain{*events, desc::events__room_events.name}; - room_type = db::domain{*events, desc::events__room_type.name}; - room_joined = db::domain{*events, desc::events__room_joined.name}; - room_state = db::domain{*events, desc::events__room_state.name}; - room_state_space = db::domain{*events, desc::events__room_state_space.name}; + event_idx = db::column{*events, desc::event_idx.name}; + event_json = db::column{*events, desc::event_json.name}; + event_refs = db::domain{*events, desc::event_refs.name}; + event_horizon = db::domain{*events, desc::event_horizon.name}; + event_sender = db::domain{*events, desc::event_sender.name}; + event_type = db::domain{*events, desc::event_type.name}; + event_state = db::domain{*events, desc::event_state.name}; + room_head = db::domain{*events, desc::room_head.name}; + room_events = db::domain{*events, desc::room_events.name}; + room_type = db::domain{*events, desc::room_type.name}; + room_joined = db::domain{*events, desc::room_joined.name}; + room_state = db::domain{*events, desc::room_state.name}; + room_state_space = db::domain{*events, desc::room_state_space.name}; } /// Shuts down the m::dbs subsystem; closes the events database. The extern @@ -247,33 +173,7 @@ ircd::m::dbs::write_opts::appendix_all{[] namespace ircd::m::dbs { - static void _index_room_joined(db::txn &, const event &, const write_opts &); - static void _index_room_redact(db::txn &, const event &, const write_opts &); //query - static void _index_room_state_space(db::txn &, const event &, const write_opts &); - static void _index_room_state(db::txn &, const event &, const write_opts &); - static void _index_room_head_resolve(db::txn &, const event &, const write_opts &); - static void _index_room_head(db::txn &, const event &, const write_opts &); - static void _index_room_type(db::txn &, const event &, const write_opts &); - static void _index_room_events(db::txn &, const event &, const write_opts &); - static void _index_room(db::txn &, const event &, const write_opts &); - static void _index_event_state(db::txn &, const event &, const write_opts &); - static void _index_event_type(db::txn &, const event &, const write_opts &); - static void _index_event_sender(db::txn &, const event &, const write_opts &); - static void _index_event_horizon_resolve(db::txn &, const event &, const write_opts &); //query - void _index_event_horizon(db::txn &, const event &, const write_opts &, const id::event &); - static void _index_event_refs_m_room_redaction(db::txn &, const event &, const write_opts &); //query - static void _index_event_refs_m_receipt_m_read(db::txn &, const event &, const write_opts &); //query - static void _index_event_refs_m_relates_m_reply(db::txn &, const event &, const write_opts &); //query - static void _index_event_refs_m_relates(db::txn &, const event &, const write_opts &); //query - static void _index_event_refs_state(db::txn &, const event &, const write_opts &); // query - static void _index_event_refs_auth(db::txn &, const event &, const write_opts &); //query - static void _index_event_refs_prev(db::txn &, const event &, const write_opts &); //query - static void _index_event_refs(db::txn &, const event &, const write_opts &); - static void _index_event_json(db::txn &, const event &, const write_opts &); - static void _index_event_cols(db::txn &, const event &, const write_opts &); - static void _index_event_id(db::txn &, const event &, const write_opts &); - static void _index_event(db::txn &, const event &, const write_opts &); - + static void _index(db::txn &, const event &, const write_opts &); static void blacklist(db::txn &txn, const event::id &, const write_opts &); } @@ -292,9 +192,7 @@ try "Cannot write to database: no index specified for event." }; - _index_event(txn, event, opts); - if(json::get<"room_id"_>(event)) - _index_room(txn, event, opts); + _index(txn, event, opts); } catch(const std::exception &e) { @@ -342,7 +240,20 @@ ircd::m::dbs::blacklist(db::txn &txn, namespace ircd::m::dbs { - static event::idx find_event_idx(const event::id &, const write_opts &); + static void _index_room_redact(db::txn &, const event &, const write_opts &); + static void _index_room(db::txn &, const event &, const write_opts &); + static void _index_event(db::txn &, const event &, const write_opts &); +} + +void +ircd::m::dbs::_index(db::txn &txn, + const event &event, + const write_opts &opts) +{ + _index_event(txn, event, opts); + + if(json::get<"room_id"_>(event)) + _index_room(txn, event, opts); } void @@ -375,863 +286,6 @@ ircd::m::dbs::_index_event(db::txn &txn, _index_event_horizon_resolve(txn, event, opts); } -void -ircd::m::dbs::_index_event_id(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_ID)); - assert(opts.event_idx); - assert(event.event_id); - - db::txn::append - { - txn, dbs::event_idx, - { - opts.op, - string_view{event.event_id}, - byte_view(opts.event_idx) - } - }; - - // For a v1 event, the "event_id" property will be saved into the `event_id` - // column by the direct property->column indexer. - if(json::get<"event_id"_>(event)) - return; - - // For v3+ events, the direct column indexer won't see any "event_id" - // property. In this case we insert the `event.event_id` manually into - // that column here. - db::txn::append - { - txn, event_column.at(json::indexof()), - { - opts.op, - byte_view(opts.event_idx), - string_view{event.event_id}, - } - }; -} - -void -ircd::m::dbs::_index_event_cols(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_COLS)); - assert(opts.event_idx); - const byte_view key - { - opts.event_idx - }; - - size_t i{0}; - for_each(event, [&txn, &opts, &key, &i] - (const auto &, auto&& val) - { - auto &column - { - event_column.at(i++) - }; - - if(!column) - return; - - if(value_required(opts.op) && !defined(json::value(val))) - return; - - db::txn::append - { - txn, column, db::column::delta - { - opts.op, - string_view{key}, - value_required(opts.op)? - byte_view{val}: - byte_view{} - } - }; - }); -} - -void -ircd::m::dbs::_index_event_json(db::txn &txn, - const event &event, - const write_opts &opts) -{ - const ctx::critical_assertion ca; - thread_local char buf[m::event::MAX_SIZE]; - assert(opts.appendix.test(appendix::EVENT_JSON)); - assert(opts.event_idx); - - const string_view &key - { - byte_view(opts.event_idx) - }; - - const string_view &val - { - // If an already-strung json::object is carried by the event and - // the opts allow us, we use it directly. This is not the default - // path unless the developer knows the source JSON is good enough - // to store directly. - opts.op == db::op::SET && event.source && opts.json_source? - string_view{event.source}: - - // If an already-strung json::object is carried by the event we - // re-stringify it into a temporary buffer. This is the common case - // because the original source might be crap JSON w/ spaces etc. - opts.op == db::op::SET && event.source? - json::stringify(mutable_buffer{buf}, event.source): - - // If no source was given with the event we can generate it. - opts.op == db::op::SET? - json::stringify(mutable_buffer{buf}, event): - - // Empty value; generally for a non-SET db::op - string_view{} - }; - - db::txn::append - { - txn, event_json, - { - opts.op, // db::op - key, // key - val, // val - } - }; -} - -void -ircd::m::dbs::_index_event_refs(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - - if(opts.event_refs.test(uint(ref::NEXT))) - _index_event_refs_prev(txn, event, opts); - - if(opts.event_refs.test(uint(ref::NEXT_AUTH))) - _index_event_refs_auth(txn, event, opts); - - if(opts.event_refs.test(uint(ref::NEXT_STATE)) || - opts.event_refs.test(uint(ref::PREV_STATE))) - _index_event_refs_state(txn, event, opts); - - if(opts.event_refs.test(uint(ref::M_RECEIPT__M_READ))) - _index_event_refs_m_receipt_m_read(txn, event, opts); - - if(opts.event_refs.test(uint(ref::M_RELATES))) - _index_event_refs_m_relates(txn, event, opts); - - if(opts.event_refs.test(uint(ref::M_RELATES))) - _index_event_refs_m_relates_m_reply(txn, event, opts); - - if(opts.event_refs.test(uint(ref::M_ROOM_REDACTION))) - _index_event_refs_m_room_redaction(txn, event, opts); -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_refs_prev(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - assert(opts.event_refs.test(uint(ref::NEXT))); - - const event::prev prev{event}; - for(size_t i(0); i < prev.prev_events_count(); ++i) - { - const event::id &prev_id - { - prev.prev_event(i) - }; - - const event::idx &prev_idx - { - find_event_idx(prev_id, opts) - }; - - if(opts.appendix.test(appendix::EVENT_HORIZON) && !prev_idx) - { - _index_event_horizon(txn, event, opts, prev_id); - continue; - } - else if(!prev_idx) - { - log::dwarning - { - log, "No index found to ref %s PREV of %s", - string_view{prev_id}, - string_view{event.event_id}, - }; - - continue; - } - - thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; - assert(opts.event_idx != 0 && prev_idx != 0); - assert(opts.event_idx != prev_idx); - const string_view &key - { - event_refs_key(buf, prev_idx, ref::NEXT, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; - } -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_refs_auth(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - assert(opts.event_refs.test(uint(ref::NEXT_AUTH))); - - if(!m::room::auth::is_power_event(event)) - return; - - const event::prev prev{event}; - for(size_t i(0); i < prev.auth_events_count(); ++i) - { - const event::id &auth_id - { - prev.auth_event(i) - }; - - const event::idx &auth_idx - { - find_event_idx(auth_id, opts) - }; - - if(unlikely(!auth_idx)) - { - if(opts.appendix.test(appendix::EVENT_HORIZON)) - _index_event_horizon(txn, event, opts, auth_id); - - log::error - { - log, "No index found to ref %s AUTH of %s", - string_view{auth_id}, - string_view{event.event_id} - }; - - continue; - } - - thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; - assert(opts.event_idx != 0 && auth_idx != 0); - assert(opts.event_idx != auth_idx); - const string_view &key - { - event_refs_key(buf, auth_idx, ref::NEXT_AUTH, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; - } -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_refs_state(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - assert(opts.event_refs.test(uint(ref::NEXT_STATE)) || - opts.event_refs.test(uint(ref::PREV_STATE))); - - if(!json::get<"room_id"_>(event)) - return; - - if(!json::get<"state_key"_>(event)) - return; - - const m::room room - { - at<"room_id"_>(event) //TODO: ABA ABA ABA ABA - }; - - const m::room::state state - { - room - }; - - const event::idx &prev_state_idx - { - opts.allow_queries? - state.get(std::nothrow, at<"type"_>(event), at<"state_key"_>(event)): // query - 0UL - }; - - // No previous state; nothing to do. - if(!prev_state_idx) - return; - - // If the previous state's event_idx is greater than the event_idx of the - // event we're transacting this is almost surely a replay/rewrite. Bail - // out for now rather than corrupting the graph. - if(unlikely(prev_state_idx >= opts.event_idx)) - return; - - thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; - assert(opts.event_idx != 0 && prev_state_idx != 0); - assert(opts.event_idx != prev_state_idx); - - if(opts.event_refs.test(uint(ref::NEXT_STATE))) - { - const string_view &key - { - event_refs_key(buf, prev_state_idx, ref::NEXT_STATE, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; - } - - if(opts.event_refs.test(uint(ref::PREV_STATE))) - { - const string_view &key - { - event_refs_key(buf, opts.event_idx, ref::PREV_STATE, prev_state_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; - } -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_refs_m_receipt_m_read(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - assert(opts.event_refs.test(uint(ref::M_RECEIPT__M_READ))); - - if(json::get<"type"_>(event) != "ircd.read") - return; - - if(!my_host(json::get<"origin"_>(event))) - return; - - //TODO: disallow local forge? - - const json::string &event_id - { - json::get<"content"_>(event).get("event_id") - }; - - const event::idx &event_idx - { - find_event_idx(event_id, opts) - }; - - if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) - { - _index_event_horizon(txn, event, opts, event_id); - return; - } - else if(!event_idx) - { - log::dwarning - { - log, "No index found to ref %s M_RECEIPT__M_READ of %s", - string_view{event_id}, - string_view{event.event_id} - }; - - return; - } - - thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; - assert(opts.event_idx != 0 && event_idx != 0); - assert(opts.event_idx != event_idx); - const string_view &key - { - event_refs_key(buf, event_idx, ref::M_RECEIPT__M_READ, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_refs_m_relates(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - assert(opts.event_refs.test(uint(ref::M_RELATES))); - - if(!json::get<"content"_>(event).has("m.relates_to")) - return; - - if(json::type(json::get<"content"_>(event).get("m.relates_to")) != json::OBJECT) - return; - - const json::object &m_relates_to - { - json::get<"content"_>(event).get("m.relates_to") - }; - - const json::string &event_id - { - m_relates_to.get("event_id") - }; - - if(!event_id) - return; - - if(!valid(m::id::EVENT, event_id)) - { - log::derror - { - log, "Cannot index m.relates_to in %s; '%s' is not an event_id.", - string_view{event.event_id}, - string_view{event_id} - }; - - return; - } - - const event::idx &event_idx - { - find_event_idx(event_id, opts) - }; - - if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) - { - // If we don't have the event being related to yet, place a marker in - // the event_horizon indicating need for re-evaluation later. - _index_event_horizon(txn, event, opts, event_id); - return; - } - else if(!event_idx) - { - log::derror - { - log, "Cannot index m.relates_to in %s; referenced %s not found.", - string_view{event.event_id}, - string_view{event_id} - }; - - return; - } - - thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; - assert(opts.event_idx != 0 && event_idx != 0); - assert(opts.event_idx != event_idx); - const string_view &key - { - event_refs_key(buf, event_idx, ref::M_RELATES, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_refs_m_relates_m_reply(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - assert(opts.event_refs.test(uint(ref::M_RELATES))); - - if(json::get<"type"_>(event) != "m.room.message") - return; - - if(!json::get<"content"_>(event).has("m.relates_to")) - return; - - if(json::type(json::get<"content"_>(event).get("m.relates_to")) != json::OBJECT) - return; - - const json::object &m_relates_to - { - json::get<"content"_>(event).get("m.relates_to") - }; - - if(!m_relates_to.has("m.in_reply_to")) - return; - - if(json::type(m_relates_to.get("m.in_reply_to")) != json::OBJECT) - { - log::derror - { - log, "Cannot index m.in_reply_to in %s; not an OBJECT.", - string_view{event.event_id} - }; - - return; - } - - const json::object &m_in_reply_to - { - m_relates_to.get("m.in_reply_to") - }; - - const json::string &event_id - { - m_in_reply_to.get("event_id") - }; - - if(!valid(m::id::EVENT, event_id)) - { - log::derror - { - log, "Cannot index m.in_reply_to in %s; '%s' is not an event_id.", - string_view{event.event_id}, - string_view{event_id} - }; - - return; - } - - const event::idx &event_idx - { - find_event_idx(event_id, opts) - }; - - if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) - { - _index_event_horizon(txn, event, opts, event_id); - return; - } - else if(!event_idx) - { - log::dwarning - { - log, "Cannot index m.in_reply_to in %s; referenced %s not found.", - string_view{event.event_id}, - string_view{event_id} - }; - - return; - } - - thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; - assert(opts.event_idx != 0 && event_idx != 0); - assert(opts.event_idx != event_idx); - const string_view &key - { - event_refs_key(buf, event_idx, ref::M_RELATES, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_refs_m_room_redaction(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_REFS)); - assert(opts.event_refs.test(uint(ref::M_ROOM_REDACTION))); - - if(json::get<"type"_>(event) != "m.room.redaction") - return; - - if(!valid(m::id::EVENT, json::get<"redacts"_>(event))) - return; - - const auto &event_id - { - json::get<"redacts"_>(event) - }; - - const event::idx &event_idx - { - find_event_idx(event_id, opts) - }; - - if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) - { - _index_event_horizon(txn, event, opts, event_id); - return; - } - else if(!event_idx) - { - log::dwarning - { - log, "Cannot index m.room.redaction in %s; referenced %s not found.", - string_view{event.event_id}, - string_view{event_id} - }; - - return; - } - - thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; - assert(opts.event_idx != 0 && event_idx != 0); - assert(opts.event_idx != event_idx); - const string_view &key - { - event_refs_key(buf, event_idx, ref::M_ROOM_REDACTION, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_refs, - { - opts.op, key - } - }; -} - -void -ircd::m::dbs::_index_event_horizon(db::txn &txn, - const event &event, - const write_opts &opts, - const m::event::id &unresolved_id) -{ - thread_local char buf[EVENT_HORIZON_KEY_MAX_SIZE]; - assert(opts.appendix.test(appendix::EVENT_HORIZON)); - assert(opts.event_idx != 0 && unresolved_id); - const string_view &key - { - event_horizon_key(buf, unresolved_id, opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_horizon, - { - opts.op, key - } - }; -} - -// NOTE: QUERY -void -ircd::m::dbs::_index_event_horizon_resolve(db::txn &txn, - const event &event, - const write_opts &opts) -{ - char buf[EVENT_HORIZON_KEY_MAX_SIZE]; - assert(opts.appendix.test(appendix::EVENT_HORIZON_RESOLVE)); - assert(opts.event_idx != 0); - assert(event.event_id); - const string_view &key - { - event_horizon_key(buf, event.event_id) - }; - - auto it - { - dbs::event_horizon.begin(key) - }; - - for(; it; ++it) - { - const auto parts - { - event_horizon_key(it->first) - }; - - const auto &event_idx - { - std::get<0>(parts) - }; - - assert(event_idx != 0); - assert(event_idx != opts.event_idx); - const event::fetch _event - { - event_idx, std::nothrow - }; - - if(!_event.valid) - { - log::dwarning - { - log, "Horizon resolve for %s @%lu not possible @%lu", - string_view{event.event_id}, - opts.event_idx, - event_idx, - }; - - continue; - } - - log::debug - { - log, "Horizon resolve for %s @%lu; remisé %s @%lu", - string_view{event.event_id}, - opts.event_idx, - string_view{_event.event_id}, - event_idx, - }; - - // Make the references on behalf of the future event - write_opts _opts; - _opts.op = opts.op; - _opts.event_idx = event_idx; - _opts.appendix.reset(); - _opts.appendix.set(appendix::EVENT_REFS); - _opts.appendix.set(appendix::ROOM_REDACT); - _opts.event_refs = opts.horizon_resolve; - _opts.interpose = &txn; - write(txn, _event, _opts); - - // Delete the event_horizon entry after resolving. - thread_local char buf[EVENT_HORIZON_KEY_MAX_SIZE]; - const string_view &key - { - event_horizon_key(buf, event.event_id, event_idx) - }; - - db::txn::append - { - txn, dbs::event_horizon, - { - opts.op == db::op::SET? - db::op::DELETE: - db::op::SET, - key - } - }; - } -} - -void -ircd::m::dbs::_index_event_sender(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_SENDER)); - assert(json::get<"sender"_>(event)); - assert(opts.event_idx); - - thread_local char buf[2][EVENT_SENDER_KEY_MAX_SIZE]; - const string_view &sender_key - { - event_sender_key(buf[0], at<"sender"_>(event), opts.event_idx) - }; - - const string_view &sender_origin_key - { - event_sender_origin_key(buf[1], at<"sender"_>(event), opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_sender, - { - opts.op, sender_key - } - }; - - db::txn::append - { - txn, dbs::event_sender, - { - opts.op, sender_origin_key - } - }; -} - -void -ircd::m::dbs::_index_event_type(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_TYPE)); - assert(json::get<"type"_>(event)); - assert(opts.event_idx); - - thread_local char buf[EVENT_TYPE_KEY_MAX_SIZE]; - const string_view &key - { - event_type_key(buf, at<"type"_>(event), opts.event_idx) - }; - - db::txn::append - { - txn, dbs::event_type, - { - opts.op, key - } - }; -} - -void -ircd::m::dbs::_index_event_state(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::EVENT_STATE)); - assert(json::get<"type"_>(event)); - assert(opts.event_idx); - - if(!defined(json::get<"state_key"_>(event))) - return; - - thread_local char buf[EVENT_STATE_KEY_MAX_SIZE]; - db::txn::append - { - txn, dbs::event_state, - { - opts.op, event_state_key(buf, event_state_tuple - { - at<"state_key"_>(event), - at<"type"_>(event), - at<"room_id"_>(event), - at<"depth"_>(event), - opts.event_idx, - }) - } - }; -} - void ircd::m::dbs::_index_room(db::txn &txn, const event &event, @@ -1267,177 +321,6 @@ ircd::m::dbs::_index_room(db::txn &txn, _index_room_redact(txn, event, opts); } -/// Adds the entry for the room_events column into the txn. -void -ircd::m::dbs::_index_room_events(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::ROOM_EVENTS)); - - thread_local char buf[ROOM_EVENTS_KEY_MAX_SIZE]; - const ctx::critical_assertion ca; - const string_view &key - { - room_events_key(buf, at<"room_id"_>(event), at<"depth"_>(event), opts.event_idx) - }; - - db::txn::append - { - txn, room_events, - { - opts.op, // db::op - key, // key, - } - }; -} - -/// Adds the entry for the room_type column into the txn. -void -ircd::m::dbs::_index_room_type(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::ROOM_TYPE)); - - thread_local char buf[ROOM_TYPE_KEY_MAX_SIZE]; - const ctx::critical_assertion ca; - const string_view &key - { - room_type_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"depth"_>(event), opts.event_idx) - }; - - db::txn::append - { - txn, room_type, - { - opts.op, // db::op - key, // key, - } - }; -} - -void -ircd::m::dbs::_index_room_head(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::ROOM_HEAD)); - assert(opts.event_idx); - assert(event.event_id); - - const ctx::critical_assertion ca; - thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE]; - const string_view &key - { - room_head_key(buf, at<"room_id"_>(event), event.event_id) - }; - - db::txn::append - { - txn, room_head, - { - opts.op, - key, - byte_view{opts.event_idx} - } - }; -} - -void -ircd::m::dbs::_index_room_head_resolve(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::ROOM_HEAD_RESOLVE)); - - //TODO: If op is DELETE and we are deleting this event and thereby - //TODO: potentially creating a gap in the reference graph (just for us - //TODO: though) can we *re-add* the prev_events to the head? - - if(opts.op != db::op::SET) - return; - - const event::prev prev{event}; - for(size_t i(0); i < prev.prev_events_count(); ++i) - { - const auto &event_id - { - prev.prev_event(i) - }; - - thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE]; - const ctx::critical_assertion ca; - const string_view &key - { - room_head_key(buf, at<"room_id"_>(event), event_id) - }; - - db::txn::append - { - txn, room_head, - { - db::op::DELETE, - key, - } - }; - } -} - -void -ircd::m::dbs::_index_room_state(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::ROOM_STATE)); - - const ctx::critical_assertion ca; - thread_local char buf[ROOM_STATE_KEY_MAX_SIZE]; - const string_view &key - { - room_state_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"state_key"_>(event)) - }; - - const string_view val - { - byte_view(opts.event_idx) - }; - - db::txn::append - { - txn, room_state, - { - opts.op, - key, - value_required(opts.op)? val : string_view{}, - } - }; -} - -void -ircd::m::dbs::_index_room_state_space(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::ROOM_STATE_SPACE)); - - const ctx::critical_assertion ca; - thread_local char buf[ROOM_STATE_SPACE_KEY_MAX_SIZE]; - const string_view &key - { - room_state_space_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"state_key"_>(event), at<"depth"_>(event), opts.event_idx) - }; - - db::txn::append - { - txn, room_state_space, - { - opts.op, - key, - } - }; -} - // NOTE: QUERY void ircd::m::dbs::_index_room_redact(db::txn &txn, @@ -1505,59 +388,6 @@ ircd::m::dbs::_index_room_redact(db::txn &txn, }; } -/// Adds the entry for the room_joined column into the txn. -void -ircd::m::dbs::_index_room_joined(db::txn &txn, - const event &event, - const write_opts &opts) -{ - assert(opts.appendix.test(appendix::ROOM_JOINED)); - assert(at<"type"_>(event) == "m.room.member"); - - thread_local char buf[ROOM_JOINED_KEY_MAX_SIZE]; - const ctx::critical_assertion ca; - const string_view &key - { - room_joined_key(buf, at<"room_id"_>(event), at<"origin"_>(event), at<"state_key"_>(event)) - }; - - const string_view &membership - { - m::membership(event) - }; - - assert(!empty(membership)); - - db::op op; - if(opts.op == db::op::SET) switch(hash(membership)) - { - case hash("join"): - op = db::op::SET; - break; - - case hash("ban"): - case hash("leave"): - op = db::op::DELETE; - break; - - default: - return; - } - else if(opts.op == db::op::DELETE) - op = opts.op; - else - return; - - db::txn::append - { - txn, room_joined, - { - op, - key, - } - }; -} - // NOTE: QUERY ircd::m::event::idx ircd::m::dbs::find_event_idx(const event::id &event_id, @@ -1572,4020 +402,3 @@ ircd::m::dbs::find_event_idx(const event::id &event_id, return ret; } - -// -// Database descriptors -// - -// -// event_idx -// - -decltype(ircd::m::dbs::desc::events__event_idx__block__size) -ircd::m::dbs::desc::events__event_idx__block__size -{ - { "name", "ircd.m.dbs.events._event_idx.block.size" }, - { "default", 256L }, -}; - -decltype(ircd::m::dbs::desc::events__event_idx__meta_block__size) -ircd::m::dbs::desc::events__event_idx__meta_block__size -{ - { "name", "ircd.m.dbs.events._event_idx.meta_block.size" }, - { "default", 2048L }, -}; - -decltype(ircd::m::dbs::desc::events__event_idx__cache__size) -ircd::m::dbs::desc::events__event_idx__cache__size -{ - { - { "name", "ircd.m.dbs.events._event_idx.cache.size" }, - { "default", long(64_MiB) }, - }, [] - { - const size_t &value{events__event_idx__cache__size}; - db::capacity(db::cache(event_idx), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_idx__cache_comp__size) -ircd::m::dbs::desc::events__event_idx__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._event_idx.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__event_idx__cache_comp__size}; - db::capacity(db::cache_compressed(event_idx), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_idx__bloom__bits) -ircd::m::dbs::desc::events__event_idx__bloom__bits -{ - { "name", "ircd.m.dbs.events._event_idx.bloom.bits" }, - { "default", 10L }, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_idx -{ - // name - "_event_idx", - - // explanation - R"(Maps matrix event_id strings into internal index numbers. - - event_id => event_idx - - The key is an event_id and the value is the index number to be used as the - key to all the event data columns. The index number is referred to as the - event_idx and is a fixed 8 byte unsigned integer. All other columns which - may key on an event_id string instead use this event_idx index number. The - index number was generated sequentially based on the order the event was - written to the database. Index numbers start at 1 because 0 is used as a - sentinel value and is not valid. The index numbers throughout the database - generally do not have gaps and can be iterated, however gaps may exist when - an event is erased from the database (which is rare for the matrix - application). - - )", - - // typing (key, value) - { - typeid(string_view), typeid(uint64_t) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, //uses conf item - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events__event_idx__bloom__bits), - - // expect queries hit - false, - - // block size - size_t(events__event_idx__block__size), - - // meta_block size - size_t(events__event_idx__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// event_json -// - -decltype(ircd::m::dbs::desc::events__event_json__block__size) -ircd::m::dbs::desc::events__event_json__block__size -{ - { "name", "ircd.m.dbs.events._event_json.block.size" }, - { "default", long(1_KiB) }, -}; - -decltype(ircd::m::dbs::desc::events__event_json__meta_block__size) -ircd::m::dbs::desc::events__event_json__meta_block__size -{ - { "name", "ircd.m.dbs.events._event_json.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_json__cache__size) -ircd::m::dbs::desc::events__event_json__cache__size -{ - { - { "name", "ircd.m.dbs.events._event_json.cache.size" }, - { "default", long(64_MiB) }, - }, [] - { - const size_t &value{events__event_json__cache__size}; - db::capacity(db::cache(event_json), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_json__cache_comp__size) -ircd::m::dbs::desc::events__event_json__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._event_json.cache_comp.size" }, - { "default", long(0_MiB) }, - }, [] - { - const size_t &value{events__event_json__cache_comp__size}; - db::capacity(db::cache_compressed(event_json), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_json__bloom__bits) -ircd::m::dbs::desc::events__event_json__bloom__bits -{ - { "name", "ircd.m.dbs.events._event_json.bloom.bits" }, - { "default", 9L }, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_json -{ - // name - "_event_json", - - // explanation - R"(Full JSON object of an event. - - event_idx => event_json - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, //uses conf item - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events__event_json__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__event_json__block__size), - - // meta_block size - size_t(events__event_json__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestLargestSeqFirst"s, - - // target_file_size - { - 2_GiB, // base - 1L, // multiplier - }, - - // max_bytes_for_level[8] - { - { 128_MiB, 1L }, // max_bytes_for_level_base - { 0L, 0L }, // max_bytes_for_level[0] - { 0L, 1L }, // max_bytes_for_level[1] - { 0L, 1L }, // max_bytes_for_level[2] - { 0L, 3L }, // max_bytes_for_level[3] - { 0L, 7L }, // max_bytes_for_level[4] - { 0L, 15L }, // max_bytes_for_level[5] - { 0L, 31L }, // max_bytes_for_level[6] - }, -}; - -// -// event_refs -// - -decltype(ircd::m::dbs::desc::events__event_refs__block__size) -ircd::m::dbs::desc::events__event_refs__block__size -{ - { "name", "ircd.m.dbs.events._event_refs.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_refs__meta_block__size) -ircd::m::dbs::desc::events__event_refs__meta_block__size -{ - { "name", "ircd.m.dbs.events._event_refs.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_refs__cache__size) -ircd::m::dbs::desc::events__event_refs__cache__size -{ - { - { "name", "ircd.m.dbs.events._event_refs.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__event_refs__cache__size}; - db::capacity(db::cache(event_refs), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_refs__cache_comp__size) -ircd::m::dbs::desc::events__event_refs__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._event_refs.cache_comp.size" }, - { "default", long(0_MiB) }, - }, [] - { - const size_t &value{events__event_refs__cache_comp__size}; - db::capacity(db::cache_compressed(event_refs), value); - } -}; - -ircd::string_view -ircd::m::dbs::reflect(const ref &type) -{ - switch(type) - { - case ref::NEXT: return "NEXT"; - case ref::NEXT_AUTH: return "NEXT_AUTH"; - case ref::NEXT_STATE: return "NEXT_STATE"; - case ref::PREV_STATE: return "PREV_STATE"; - case ref::M_RECEIPT__M_READ: return "M_RECEIPT__M_READ"; - case ref::M_RELATES: return "M_RELATES"; - case ref::M_ROOM_REDACTION: return "M_ROOM_REDACTION"; - } - - return "????"; -} - -ircd::string_view -ircd::m::dbs::event_refs_key(const mutable_buffer &out, - const event::idx &tgt, - const ref &type, - const event::idx &src) -{ - assert((src & ref_mask) == 0); - assert(size(out) >= sizeof(event::idx) * 2); - event::idx *const &key - { - reinterpret_cast(data(out)) - }; - - key[0] = tgt; - key[1] = src; - key[1] |= uint64_t(type) << ref_shift; - return string_view - { - data(out), data(out) + sizeof(event::idx) * 2 - }; -} - -std::tuple -ircd::m::dbs::event_refs_key(const string_view &amalgam) -{ - const event::idx key - { - byte_view{amalgam} - }; - - return - { - ref(key >> ref_shift), key & ~ref_mask - }; -} - -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__event_refs__pfx -{ - "_event_refs", - [](const string_view &key) - { - return size(key) >= sizeof(event::idx) * 2; - }, - - [](const string_view &key) - { - assert(size(key) >= sizeof(event::idx)); - return string_view - { - data(key), data(key) + sizeof(event::idx) - }; - } -}; - -const ircd::db::comparator -ircd::m::dbs::desc::events__event_refs__cmp -{ - "_event_refs", - - // less - [](const string_view &a, const string_view &b) - { - static const size_t half(sizeof(event::idx)); - static const size_t full(half * 2); - - assert(size(a) >= half); - assert(size(b) >= half); - const event::idx *const key[2] - { - reinterpret_cast(data(a)), - reinterpret_cast(data(b)), - }; - - return - key[0][0] < key[1][0]? true: - key[0][0] > key[1][0]? false: - size(a) < size(b)? true: - size(a) > size(b)? false: - size(a) == half? false: - key[0][1] < key[1][1]? true: - false; - }, - - // equal - [](const string_view &a, const string_view &b) - { - return size(a) == size(b) && memcmp(data(a), data(b), size(a)) == 0; - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_refs -{ - // name - "_event_refs", - - // explanation - R"(Inverse reference graph of events. - - event_idx | ref, event_idx => -- - - The first part of the key is the event being referenced. The second part - of the key is the event which refers to the first event somewhere in its - prev_events references. The event_idx in the second part of the key also - contains a dbs::ref type in its highest order byte so we can store - different kinds of references. - - The prefix transform is in effect; an event may be referenced multiple - times. We can find all the events we have which reference a target, and - why. The database must already contain both events (hence they have - event::idx numbers). - - The value is currently unused/empty; we may eventually store metadata with - information about this reference (i.e. is depth adjacent? is the ref - redundant with another in the same event and should not be made? etc). - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - events__event_refs__cmp, - - // prefix transform - events__event_refs__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, //uses conf item - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - 0, - - // expect queries hit - true, - - // block size - size_t(events__event_refs__block__size), - - // meta_block size - size_t(events__event_refs__meta_block__size), - - // compression - {}, // no compression for this column - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// event_horizon -// - -decltype(ircd::m::dbs::desc::events__event_horizon__block__size) -ircd::m::dbs::desc::events__event_horizon__block__size -{ - { "name", "ircd.m.dbs.events._event_horizon.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_horizon__meta_block__size) -ircd::m::dbs::desc::events__event_horizon__meta_block__size -{ - { "name", "ircd.m.dbs.events._event_horizon.meta_block.size" }, - { "default", 1024L }, -}; - -decltype(ircd::m::dbs::desc::events__event_horizon__cache__size) -ircd::m::dbs::desc::events__event_horizon__cache__size -{ - { - { "name", "ircd.m.dbs.events._event_horizon.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__event_horizon__cache__size}; - db::capacity(db::cache(event_horizon), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_horizon__cache_comp__size) -ircd::m::dbs::desc::events__event_horizon__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._event_horizon.cache_comp.size" }, - { "default", long(0_MiB) }, - }, [] - { - const size_t &value{events__event_horizon__cache_comp__size}; - db::capacity(db::cache_compressed(event_horizon), value); - } -}; - -ircd::string_view -ircd::m::dbs::event_horizon_key(const mutable_buffer &out, - const event::id &event_id) -{ - return event_horizon_key(out, event_id, 0UL); -} - -ircd::string_view -ircd::m::dbs::event_horizon_key(const mutable_buffer &out, - const event::id &event_id, - const event::idx &event_idx) -{ - mutable_buffer buf(out); - consume(buf, copy(buf, event_id)); - - if(event_idx) - { - consume(buf, copy(buf, "\0"_sv)); - consume(buf, copy(buf, byte_view(event_idx))); - } - - const string_view ret - { - data(out), data(buf) - }; - - assert(size(ret) == size(event_id) || size(ret) == size(event_id) + sizeof(event::idx) + 1); - return ret; -} - -std::tuple -ircd::m::dbs::event_horizon_key(const string_view &amalgam) -{ - assert(size(amalgam) == 1 + sizeof(event::idx)); - assert(amalgam[0] == '\0'); - - const byte_view &event_idx - { - amalgam.substr(1) - }; - - return - { - static_cast(event_idx) - }; -} - -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__event_horizon__pfx -{ - "_event_horizon", - - [](const string_view &key) - { - return has(key, '\0'); - }, - - [](const string_view &key) - { - assert(size(key) >= sizeof(event::idx)); - return split(key, '\0').first; - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_horizon -{ - // name - "_event_horizon", - - // explanation - R"(Unresolved references in the reverse reference graph of events. - - event_id | event_idx => -- - - The first part of the key is an event_id which the server does not have. - The suffix of the key is the index number of an event which the server - does have and it contains a reference to event_id. - - We use the information in this column to find all of the events which - have an unresolved reference to this event and complete the holes in the - event_refs graph which could not be completed without this event. - - When a new event is written to the database the event_horizon column is - queried seeking the event's ID. Each entry in event_horizon is the index - of an event which we previously wrote to the database without knowing the - index of the event currently being written (an out-of-order write). - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - events__event_horizon__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, //uses conf item - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - 0, - - // expect queries hit - false, - - // block size - size_t(events__event_horizon__block__size), - - // meta_block size - size_t(events__event_horizon__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// event_sender -// - -decltype(ircd::m::dbs::desc::events__event_sender__block__size) -ircd::m::dbs::desc::events__event_sender__block__size -{ - { "name", "ircd.m.dbs.events._event_sender.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_sender__meta_block__size) -ircd::m::dbs::desc::events__event_sender__meta_block__size -{ - { "name", "ircd.m.dbs.events._event_sender.meta_block.size" }, - { "default", 4096L }, -}; - -decltype(ircd::m::dbs::desc::events__event_sender__cache__size) -ircd::m::dbs::desc::events__event_sender__cache__size -{ - { - { "name", "ircd.m.dbs.events._event_sender.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__event_sender__cache__size}; - db::capacity(db::cache(event_sender), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_sender__cache_comp__size) -ircd::m::dbs::desc::events__event_sender__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._event_sender.cache_comp.size" }, - { "default", long(0_MiB) }, - }, [] - { - const size_t &value{events__event_sender__cache_comp__size}; - db::capacity(db::cache_compressed(event_sender), value); - } -}; - -// sender_key - -ircd::string_view -ircd::m::dbs::event_sender_key(const mutable_buffer &out_, - const user::id &user_id, - const event::idx &event_idx) -{ - assert(size(out_) >= EVENT_SENDER_KEY_MAX_SIZE); - assert(!event_idx || user_id); - - mutable_buffer out{out_}; - consume(out, copy(out, user_id)); - - if(user_id && event_idx) - { - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, byte_view(event_idx))); - } - - return { data(out_), data(out) }; -} - -std::tuple -ircd::m::dbs::event_sender_key(const string_view &amalgam) -{ - const auto &parts - { - split(amalgam, '\0') - }; - - assert(empty(parts.first)); - return - { - byte_view(parts.second), - }; -} - -bool -ircd::m::dbs::is_event_sender_key(const string_view &key) -{ - return empty(key) || startswith(key, '@'); -} - -// sender_origin_key - -ircd::string_view -ircd::m::dbs::event_sender_origin_key(const mutable_buffer &out, - const user::id &user_id, - const event::idx &event_idx) -{ - return event_sender_origin_key(out, user_id.host(), user_id.local(), event_idx); -} - -ircd::string_view -ircd::m::dbs::event_sender_origin_key(const mutable_buffer &out_, - const string_view &origin, - const string_view &localpart, - const event::idx &event_idx) -{ - assert(size(out_) >= EVENT_SENDER_KEY_MAX_SIZE); - assert(!event_idx || localpart); - assert(!localpart || startswith(localpart, '@')); - - mutable_buffer out{out_}; - consume(out, copy(out, origin)); - consume(out, copy(out, localpart)); - - if(localpart && event_idx) - { - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, byte_view(event_idx))); - } - - return { data(out_), data(out) }; -} - -std::tuple -ircd::m::dbs::event_sender_origin_key(const string_view &amalgam) -{ - const auto &parts - { - split(amalgam, '\0') - }; - - assert(!empty(parts.first) && !empty(parts.second)); - assert(startswith(parts.first, '@')); - - return - { - parts.first, - byte_view(parts.second), - }; -} - -bool -ircd::m::dbs::is_event_sender_origin_key(const string_view &key) -{ - return !startswith(key, '@'); -} - -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__event_sender__pfx -{ - "_event_sender", - [](const string_view &key) - { - return startswith(key, '@')? - has(key, '\0'): - has(key, '@'); - }, - - [](const string_view &key) - { - const auto &[prefix, suffix] - { - // Split @localpart:hostpart\0event_idx by '\0' - startswith(key, '@')? - split(key, '\0'): - - // Split hostpart@localpart\0event_idx by '@' - split(key, '@') - }; - - return prefix; - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_sender -{ - // name - "_event_sender", - - // explanation - R"(Index of senders to their events. - - mxid | event_idx => -- - origin | localpart, event_idx => -- - - The senders of events are indexed by this column. This allows for all - events from a sender to be iterated. Additionally, all events from a - server and all known servers can be iterated from this column. - - key #1: - The first type of key is made from a user mxid and an event_idx concat. - - key #2: - The second type of key is made from a user mxid and an event_id, where - the mxid is part-swapped so the origin comes first, and the @localpart - comes after. - - Note that the indexers of this column ignores the actual "origin" field - of an event. Only the "sender" data is used here. - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - events__event_sender__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, //uses conf item - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - 0, - - // expect queries hit - false, - - // block size - size_t(events__event_sender__block__size), - - // meta_block size - size_t(events__event_sender__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// event_type -// - -decltype(ircd::m::dbs::desc::events__event_type__block__size) -ircd::m::dbs::desc::events__event_type__block__size -{ - { "name", "ircd.m.dbs.events._event_type.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_type__meta_block__size) -ircd::m::dbs::desc::events__event_type__meta_block__size -{ - { "name", "ircd.m.dbs.events._event_type.meta_block.size" }, - { "default", 4096L }, -}; - -decltype(ircd::m::dbs::desc::events__event_type__cache__size) -ircd::m::dbs::desc::events__event_type__cache__size -{ - { - { "name", "ircd.m.dbs.events._event_type.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__event_type__cache__size}; - db::capacity(db::cache(event_type), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_type__cache_comp__size) -ircd::m::dbs::desc::events__event_type__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._event_type.cache_comp.size" }, - { "default", long(0_MiB) }, - }, [] - { - const size_t &value{events__event_type__cache_comp__size}; - db::capacity(db::cache_compressed(event_type), value); - } -}; - -ircd::string_view -ircd::m::dbs::event_type_key(const mutable_buffer &out_, - const string_view &type, - const event::idx &event_idx) -{ - assert(size(out_) >= EVENT_TYPE_KEY_MAX_SIZE); - - mutable_buffer out{out_}; - consume(out, copy(out, type)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, byte_view(event_idx))); - return { data(out_), data(out) }; -} - -std::tuple -ircd::m::dbs::event_type_key(const string_view &amalgam) -{ - assert(size(amalgam) == sizeof(event::idx) + 1); - const auto &key - { - amalgam.substr(1) - }; - - assert(size(key) == sizeof(event::idx)); - return - { - byte_view(key) - }; -} - -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__event_type__pfx -{ - "_event_type", - [](const string_view &key) - { - return has(key, '\0'); - }, - - [](const string_view &key) - { - const auto &parts - { - split(key, '\0') - }; - - return parts.first; - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_type -{ - // name - "_event_type", - - // explanation - R"(Index of types of events. - - type | event_idx => -- - - The types of events are indexed by this column. All events of a specific type can be - iterated efficiently. The type string forms the prefix domain. - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - events__event_type__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, //uses conf item - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - 0, - - // expect queries hit - false, - - // block size - size_t(events__event_type__block__size), - - // meta_block size - size_t(events__event_type__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// event_state -// - -decltype(ircd::m::dbs::desc::events__event_state__block__size) -ircd::m::dbs::desc::events__event_state__block__size -{ - { "name", "ircd.m.dbs.events._event_state.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_state__meta_block__size) -ircd::m::dbs::desc::events__event_state__meta_block__size -{ - { "name", "ircd.m.dbs.events._event_state.meta_block.size" }, - { "default", 2048L }, -}; - -decltype(ircd::m::dbs::desc::events__event_state__cache__size) -ircd::m::dbs::desc::events__event_state__cache__size -{ - { - { "name", "ircd.m.dbs.events._event_state.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__event_state__cache__size}; - db::capacity(db::cache(event_state), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_state__cache_comp__size) -ircd::m::dbs::desc::events__event_state__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._event_state.cache_comp.size" }, - { "default", long(0_MiB) }, - }, [] - { - const size_t &value{events__event_state__cache_comp__size}; - db::capacity(db::cache_compressed(event_state), value); - } -}; - -ircd::string_view -ircd::m::dbs::event_state_key(const mutable_buffer &out_, - const event_state_tuple &tuple) -{ - assert(size(out_) >= EVENT_STATE_KEY_MAX_SIZE); - - const auto &[state_key, type, room_id, depth, event_idx] - { - tuple - }; - - if(!state_key) - return {}; - - mutable_buffer out{out_}; - consume(out, copy(out, state_key)); - if(!type) - return {data(out_), data(out)}; - - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, type)); - if(!room_id) - return {data(out_), data(out)}; - - assert(m::valid(m::id::ROOM, room_id)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, room_id)); - if(depth < 0) - return {data(out_), data(out)}; - - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, byte_view(depth))); - if(!event_idx) - return {data(out_), data(out)}; - - consume(out, copy(out, byte_view(event_idx))); - return {data(out_), data(out)}; -} - -ircd::m::dbs::event_state_tuple -ircd::m::dbs::event_state_key(const string_view &amalgam) -{ - assert(!startswith(amalgam, '\0')); - const auto &[state_key, r0] - { - split(amalgam, "\0"_sv) - }; - - const auto &[type, r1] - { - split(r0, "\0"_sv) - }; - - const auto &[room_id, r2] - { - split(r1, "\0"_sv) - }; - - assert(!room_id || m::valid(m::id::ROOM, room_id)); - return event_state_tuple - { - state_key, - type, - room_id, - r2.size() >= 8? - int64_t(byte_view(r2.substr(0, 8))): - -1L, - r2.size() >= 16? - event::idx(byte_view(r2.substr(8))): - 0UL, - }; -} - -const ircd::db::comparator -ircd::m::dbs::desc::events__event_state__cmp -{ - "_event_state", - - // less - [](const string_view &a, const string_view &b) - { - const event_state_tuple key[2] - { - event_state_key(a), - event_state_key(b), - }; - - const auto &[state_key_a, type_a, room_id_a, depth_a, event_idx_a] - { - key[0] - }; - - const auto &[state_key_b, type_b, room_id_b, depth_b, event_idx_b] - { - key[1] - }; - - if(state_key_a != state_key_b) - return state_key_a < state_key_b; - - if(type_a != type_b) - return type_a < type_b; - - if(room_id_a != room_id_b) - return room_id_a < room_id_b; - - if(depth_a != depth_b) - return depth_a > depth_b; - - if(event_idx_a != event_idx_b) - return event_idx_a > event_idx_b; - - return false; - }, - - // equal - [](const string_view &a, const string_view &b) - { - return a == b; - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_state -{ - // name - "_event_state", - - // explanation - R"(Index of states of events. - - state_key, type, room_id, depth, event_idx => -- - - The state transitions of events are indexed by this column, - based on the state_key property. - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - events__event_state__cmp, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, //uses conf item - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - 0, - - // expect queries hit - false, - - // block size - size_t(events__event_state__block__size), - - // meta_block size - size_t(events__event_state__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// room_head -// - -decltype(ircd::m::dbs::desc::events__room_head__block__size) -ircd::m::dbs::desc::events__room_head__block__size -{ - { "name", "ircd.m.dbs.events._room_head.block.size" }, - { "default", 4096L }, -}; - -decltype(ircd::m::dbs::desc::events__room_head__meta_block__size) -ircd::m::dbs::desc::events__room_head__meta_block__size -{ - { "name", "ircd.m.dbs.events._room_head.meta_block.size" }, - { "default", 4096L }, -}; - -decltype(ircd::m::dbs::desc::events__room_head__cache__size) -ircd::m::dbs::desc::events__room_head__cache__size -{ - { - { "name", "ircd.m.dbs.events._room_head.cache.size" }, - { "default", long(8_MiB) }, - }, [] - { - const size_t &value{events__room_head__cache__size}; - db::capacity(db::cache(room_head), value); - } -}; - -/// prefix transform for room_id,event_id in room_id -/// -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__room_head__pfx -{ - "_room_head", - [](const string_view &key) - { - return has(key, "\0"_sv); - }, - - [](const string_view &key) - { - return split(key, "\0"_sv).first; - } -}; - -ircd::string_view -ircd::m::dbs::room_head_key(const mutable_buffer &out_, - const id::room &room_id, - const id::event &event_id) -{ - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, event_id)); - return { data(out_), data(out) }; -} - -ircd::string_view -ircd::m::dbs::room_head_key(const string_view &amalgam) -{ - const auto &key - { - lstrip(amalgam, "\0"_sv) - }; - - return - { - key - }; -} - -/// This column stores unreferenced (head) events for a room. -/// -const ircd::db::descriptor -ircd::m::dbs::desc::events__room_head -{ - // name - "_room_head", - - // explanation - R"(Unreferenced events in a room. - - [room_id | event_id => event_idx] - - The key is a room_id and event_id concatenation. The value is an event_idx - of the event_id in the key. The key amalgam was specifically selected to - allow for DELETES sent to the WAL "in the blind" for all prev_events when - any new event is saved to the database, without making any read IO's to - look up anything about the prev reference to remove. - - This is a fast-moving column where unreferenced events are inserted and - then deleted the first time another event is seen which references it so - it collects a lot of DELETE commands in the WAL and has to be compacted - often to reduce them out. - - )", - - // typing (key, value) - { - typeid(string_view), typeid(uint64_t) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - events__room_head__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - 0, //no compresed cache - - // bloom filter bits - 0, //table too ephemeral for bloom generation/usefulness - - // expect queries hit - false, - - // block size - size_t(events__room_head__block__size), - - // meta_block size - size_t(events__room_head__meta_block__size), - - // compression - {}, // no compression for this column - - // compactor - {}, - - // compaction priority algorithm - "kByCompensatedSize"s, -}; - -// -// room_events -// - -decltype(ircd::m::dbs::desc::events__room_events__block__size) -ircd::m::dbs::desc::events__room_events__block__size -{ - { "name", "ircd.m.dbs.events._room_events.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__room_events__meta_block__size) -ircd::m::dbs::desc::events__room_events__meta_block__size -{ - { "name", "ircd.m.dbs.events._room_events.meta_block.size" }, - { "default", 16384L }, -}; - -decltype(ircd::m::dbs::desc::events__room_events__cache__size) -ircd::m::dbs::desc::events__room_events__cache__size -{ - { - { "name", "ircd.m.dbs.events._room_events.cache.size" }, - { "default", long(32_MiB) }, - }, [] - { - const size_t &value{events__room_events__cache__size}; - db::capacity(db::cache(room_events), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_events__cache_comp__size) -ircd::m::dbs::desc::events__room_events__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._room_events.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__room_events__cache_comp__size}; - db::capacity(db::cache_compressed(room_events), value); - } -}; - -/// Prefix transform for the events__room_events. The prefix here is a room_id -/// and the suffix is the depth+event_id concatenation. -/// for efficient sequences -/// -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__room_events__pfx -{ - "_room_events", - - [](const string_view &key) - { - return has(key, "\0"_sv); - }, - - [](const string_view &key) - { - return split(key, "\0"_sv).first; - } -}; - -/// Comparator for the events__room_events. The goal here is to sort the -/// events within a room by their depth from highest to lowest, so the -/// highest depth is hit first when a room is sought from this column. -/// -const ircd::db::comparator -ircd::m::dbs::desc::events__room_events__cmp -{ - "_room_events", - - // less - [](const string_view &a, const string_view &b) - { - static const auto &pt - { - events__room_events__pfx - }; - - // Extract the prefix from the keys - const string_view pre[2] - { - pt.get(a), - pt.get(b), - }; - - if(size(pre[0]) != size(pre[1])) - return size(pre[0]) < size(pre[1]); - - if(pre[0] != pre[1]) - return pre[0] < pre[1]; - - // After the prefix is the depth + event_idx - const string_view post[2] - { - a.substr(size(pre[0])), - b.substr(size(pre[1])), - }; - - // These conditions are matched on some queries when the user only - // supplies a room id. - - if(empty(post[0])) - return true; - - if(empty(post[1])) - return false; - - // Distill out the depth and event_idx integers - const std::tuple pair[2] - { - room_events_key(post[0]), - room_events_key(post[1]) - }; - - // When two events are at the same depth sort by index (the sequence - // number given as they were admitted into the system) otherwise - // sort by depth. Note this is a reverse order comparison. - return std::get<0>(pair[1]) != std::get<0>(pair[0])? - std::get<0>(pair[1]) < std::get<0>(pair[0]): - std::get<1>(pair[1]) < std::get<1>(pair[0]); - }, - - // equal - [](const string_view &a, const string_view &b) - { - return a == b; - } -}; - -ircd::string_view -ircd::m::dbs::room_events_key(const mutable_buffer &out_, - const id::room &room_id, - const uint64_t &depth) -{ - const const_buffer depth_cb - { - reinterpret_cast(&depth), sizeof(depth) - }; - - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, depth_cb)); - return { data(out_), data(out) }; -} - -ircd::string_view -ircd::m::dbs::room_events_key(const mutable_buffer &out_, - const id::room &room_id, - const uint64_t &depth, - const event::idx &event_idx) -{ - const const_buffer depth_cb - { - reinterpret_cast(&depth), sizeof(depth) - }; - - const const_buffer event_idx_cb - { - reinterpret_cast(&event_idx), sizeof(event_idx) - }; - - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, depth_cb)); - consume(out, copy(out, event_idx_cb)); - return { data(out_), data(out) }; -} - -std::tuple -ircd::m::dbs::room_events_key(const string_view &amalgam) -{ - assert(size(amalgam) >= 1 + 8 + 8 || size(amalgam) == 1 + 8); - assert(amalgam.front() == '\0'); - - const uint64_t &depth - { - *reinterpret_cast(data(amalgam) + 1) - }; - - const event::idx &event_idx - { - size(amalgam) >= 1 + 8 + 8? - *reinterpret_cast(data(amalgam) + 1 + 8): - std::numeric_limits::max() - }; - - // Make sure these are copied rather than ever returning references in - // a tuple because the chance the integers will be aligned is low. - return { depth, event_idx }; -} - -/// This column stores events in sequence in a room. Consider the following: -/// -/// [room_id | depth + event_idx] -/// -/// The key is composed from three parts: -/// -/// - `room_id` is the official prefix, bounding the sequence. That means we -/// make a blind query with just a room_id and get to the beginning of the -/// sequence, then iterate until we stop before the next room_id (upper bound). -/// -/// - `depth` is the ordering. Within the sequence, all elements are ordered by -/// depth from HIGHEST TO LOWEST. The sequence will start at the highest depth. -/// NOTE: Depth is a fixed 8 byte binary integer. -/// -/// - `event_idx` is the key suffix. This column serves to sequence all events -/// within a room ordered by depth. There may be duplicate room_id|depth -/// prefixing but the event_idx suffix gives the key total uniqueness. -/// NOTE: event_idx is a fixed 8 byte binary integer. -/// -const ircd::db::descriptor -ircd::m::dbs::desc::events__room_events -{ - // name - "_room_events", - - // explanation - R"(Indexes events in timeline sequence for a room - - [room_id | depth + event_idx] - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - events__room_events__cmp, - - // prefix transform - events__room_events__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - 0, // no bloom filter because of possible comparator issues - - // expect queries hit - true, - - // block size - size_t(events__room_events__block__size), - - // meta_block size - size_t(events__room_events__meta_block__size), -}; - -// -// room_type -// - -decltype(ircd::m::dbs::desc::events__room_type__block__size) -ircd::m::dbs::desc::events__room_type__block__size -{ - { "name", "ircd.m.dbs.events._room_type.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__room_type__meta_block__size) -ircd::m::dbs::desc::events__room_type__meta_block__size -{ - { "name", "ircd.m.dbs.events._room_type.meta_block.size" }, - { "default", 8192L }, -}; - -decltype(ircd::m::dbs::desc::events__room_type__cache__size) -ircd::m::dbs::desc::events__room_type__cache__size -{ - { - { "name", "ircd.m.dbs.events._room_type.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__room_type__cache__size}; - db::capacity(db::cache(room_type), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_type__cache_comp__size) -ircd::m::dbs::desc::events__room_type__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._room_type.cache_comp.size" }, - { "default", long(8_MiB) }, - }, [] - { - const size_t &value{events__room_type__cache_comp__size}; - db::capacity(db::cache_compressed(room_type), value); - } -}; - -/// Prefix transform for the events__room_type. The prefix here is a room_id -/// and the suffix is the type+depth+event_id concatenation. -/// for efficient sequences -/// -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__room_type__pfx -{ - "_room_type", - - [](const string_view &key) - { - return has(key, "\0"_sv); - }, - - [](const string_view &key) - { - return split(key, "\0"_sv).first; - } -}; - -/// Comparator for the events__room_type. The goal here is to sort the -/// events within a room by their depth from highest to lowest, so the -/// highest depth is hit first when a room is sought from this column. -/// -const ircd::db::comparator -ircd::m::dbs::desc::events__room_type__cmp -{ - "_room_type", - - // less - [](const string_view &a, const string_view &b) - { - static const auto &pt - { - events__room_type__pfx - }; - - // Extract the prefix from the keys - const string_view pre[2] - { - pt.get(a), - pt.get(b), - }; - - // Prefix size comparison has highest priority for rocksdb - if(size(pre[0]) < size(pre[1])) - return true; - - // Prefix size comparison has highest priority for rocksdb - if(size(pre[0]) > size(pre[1])) - return false; - - // Prefix lexical comparison sorts prefixes of the same size - if(pre[0] < pre[1]) - return true; - - // Prefix lexical comparison sorts prefixes of the same size - if(pre[0] > pre[1]) - return false; - - // After the prefix is the \0,type,\0,depth,event_idx - const string_view post[2] - { - a.substr(size(pre[0])), - b.substr(size(pre[1])), - }; - - // These conditions are matched on some queries when the user only - // supplies a room id. - if(empty(post[0])) - return true; - - if(empty(post[1])) - return false; - - const auto &[type_a, depth_a, event_idx_a] - { - room_type_key(post[0]) - }; - - const auto &[type_b, depth_b, event_idx_b] - { - room_type_key(post[1]) - }; - - if(type_a < type_b) - return true; - - if(type_a > type_b) - return false; - - // reverse depth to start from highest first like room_events - if(depth_a < depth_b) - return false; - - // reverse depth to start from highest first like room_events - if(depth_a > depth_b) - return true; - - // reverse event_idx to start from highest first like room_events) - if(event_idx_a < event_idx_b) - return false; - - if(event_idx_a > event_idx_b) - return true; - - // equal is not less; so false - return false; - }, - - // equal - [](const string_view &a, const string_view &b) - { - return a == b; - } -}; - -ircd::m::dbs::room_type_tuple -ircd::m::dbs::room_type_key(const string_view &amalgam_) -{ - assert(size(amalgam_) >= 1 + 1 + 8 + 8); - - assert(amalgam_.front() == '\0'); - const string_view &amalgam - { - amalgam_.substr(1) - }; - - assert(amalgam.size() >= 1 + 8 + 1); - const auto &[type, trail] - { - split(amalgam, '\0') - }; - - assert(trail.size() >= 8 + 8); - return room_type_tuple - { - type, - likely(trail.size() >= 8)? - uint64_t(byte_view(trail.substr(0, 8))): - -1UL, - likely(trail.size() >= 16)? - event::idx(byte_view(trail.substr(8))): - 0UL, - }; -} - -ircd::string_view -ircd::m::dbs::room_type_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &type, - const uint64_t &depth, - const event::idx &event_idx) -{ - assert(room_id); - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - - if(!type) - return { data(out_), data(out) }; - - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, type)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, byte_view(depth))); - consume(out, copy(out, byte_view(event_idx))); - return { data(out_), data(out) }; -} - -/// This column stores events by type in sequence in a room. Consider the -/// following: -/// -/// [room_id | type, depth, event_idx] -/// -const ircd::db::descriptor -ircd::m::dbs::desc::events__room_type -{ - // name - "_room_type", - - // explanation - R"(Indexes events per type in timeline sequence for a room - - [room_id | type, depth, event_idx] - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - events__room_type__cmp, - - // prefix transform - events__room_type__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - 0, // no bloom filter because of possible comparator issues - - // expect queries hit - true, - - // block size - size_t(events__room_type__block__size), - - // meta_block size - size_t(events__room_type__meta_block__size), -}; - -// -// joined sequential -// - -decltype(ircd::m::dbs::desc::events__room_joined__block__size) -ircd::m::dbs::desc::events__room_joined__block__size -{ - { "name", "ircd.m.dbs.events._room_joined.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__room_joined__meta_block__size) -ircd::m::dbs::desc::events__room_joined__meta_block__size -{ - { "name", "ircd.m.dbs.events._room_joined.meta_block.size" }, - { "default", 8192L }, -}; - -decltype(ircd::m::dbs::desc::events__room_joined__cache__size) -ircd::m::dbs::desc::events__room_joined__cache__size -{ - { - { "name", "ircd.m.dbs.events._room_joined.cache.size" }, - { "default", long(8_MiB) }, - }, [] - { - const size_t &value{events__room_joined__cache__size}; - db::capacity(db::cache(room_joined), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_joined__cache_comp__size) -ircd::m::dbs::desc::events__room_joined__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._room_joined.cache_comp.size" }, - { "default", long(8_MiB) }, - }, [] - { - const size_t &value{events__room_joined__cache_comp__size}; - db::capacity(db::cache_compressed(room_joined), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_joined__bloom__bits) -ircd::m::dbs::desc::events__room_joined__bloom__bits -{ - { "name", "ircd.m.dbs.events._room_joined.bloom.bits" }, - { "default", 6L }, -}; - -/// Prefix transform for the events__room_joined -/// -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__room_joined__pfx -{ - "_room_joined", - - [](const string_view &key) - { - return has(key, "\0"_sv); - }, - - [](const string_view &key) - { - return split(key, "\0"_sv).first; - } -}; - -ircd::string_view -ircd::m::dbs::room_joined_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &origin) -{ - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, origin)); - return { data(out_), data(out) }; -} - -ircd::string_view -ircd::m::dbs::room_joined_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &origin, - const id::user &member) -{ - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, origin)); - consume(out, copy(out, member)); - return { data(out_), data(out) }; -} - -std::tuple -ircd::m::dbs::room_joined_key(const string_view &amalgam) -{ - const auto &key - { - lstrip(amalgam, "\0"_sv) - }; - - const auto &s - { - split(key, "@"_sv) - }; - - return - { - { s.first }, - !empty(s.second)? - string_view{begin(s.second) - 1, end(s.second)}: - string_view{} - }; -} - -const ircd::db::descriptor -ircd::m::dbs::desc::events__room_joined -{ - // name - "_room_joined", - - // explanation - R"(Specifically indexes joined members of a room for fast iteration. - - [room_id | origin + mxid] => event_idx - - )", - - // typing (key, value) - { - typeid(string_view), typeid(uint64_t) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - events__room_joined__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events__room_joined__bloom__bits), - - // expect queries hit - false, - - // block size - size_t(events__room_joined__block__size), - - // meta_block size - size_t(events__room_joined__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// room present state sequential -// - -decltype(ircd::m::dbs::desc::events__room_state__block__size) -ircd::m::dbs::desc::events__room_state__block__size -{ - { "name", "ircd.m.dbs.events._room_state.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__room_state__meta_block__size) -ircd::m::dbs::desc::events__room_state__meta_block__size -{ - { "name", "ircd.m.dbs.events._room_state.meta_block.size" }, - { "default", 8192L }, -}; - -decltype(ircd::m::dbs::desc::events__room_state__cache__size) -ircd::m::dbs::desc::events__room_state__cache__size -{ - { - { "name", "ircd.m.dbs.events._room_state.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__room_state__cache__size}; - db::capacity(db::cache(room_state), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_state__cache_comp__size) -ircd::m::dbs::desc::events__room_state__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._room_state.cache_comp.size" }, - { "default", long(8_MiB) }, - }, [] - { - const size_t &value{events__room_state__cache_comp__size}; - db::capacity(db::cache_compressed(room_state), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_state__bloom__bits) -ircd::m::dbs::desc::events__room_state__bloom__bits -{ - { "name", "ircd.m.dbs.events._room_state.bloom.bits" }, - { "default", 10L }, -}; - -/// prefix transform for type,state_key in room_id -/// -/// This transform is special for concatenating room_id with type and state_key -/// in that order with prefix being the room_id (this may change to room_id+ -/// type -/// -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__room_state__pfx -{ - "_room_state", - [](const string_view &key) - { - return has(key, "\0"_sv); - }, - - [](const string_view &key) - { - return split(key, "\0"_sv).first; - } -}; - -ircd::string_view -ircd::m::dbs::room_state_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &type) -{ - return room_state_key(out_, room_id, type, string_view{}); -} - -ircd::string_view -ircd::m::dbs::room_state_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &type, - const string_view &state_key) -{ - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - assert(room_id); - - if(likely(defined(type))) - { - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, type)); - } - - if(defined(state_key)) - { - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, state_key)); - } - - return { data(out_), data(out) }; -} - -std::tuple -ircd::m::dbs::room_state_key(const string_view &amalgam) -{ - const auto &key - { - lstrip(amalgam, "\0"_sv) - }; - - const auto &s - { - split(key, "\0"_sv) - }; - - return - { - s.first, s.second - }; -} - -const ircd::db::descriptor -ircd::m::dbs::desc::events__room_state -{ - // name - "_room_state", - - // explanation - R"(The present state of the room. - - [room_id | type + state_key] => event_idx - - This column is also known as the "present state table." It contains the - very important present state of the room for this server. The key contains - plaintext room_id, type and state_key elements for direct point-lookup as - well as iteration. The value is the index of the apropos state event. - - )", - - // typing (key, value) - { - typeid(string_view), typeid(uint64_t) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - events__room_state__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events__room_state__bloom__bits), - - // expect queries hit - false, - - // block size - size_t(events__room_state__block__size), - - // meta_block size - size_t(events__room_state__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// room all states sequential -// - -decltype(ircd::m::dbs::desc::events__room_state_space__block__size) -ircd::m::dbs::desc::events__room_state_space__block__size -{ - { "name", "ircd.m.dbs.events._room_state_space.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__room_state_space__meta_block__size) -ircd::m::dbs::desc::events__room_state_space__meta_block__size -{ - { "name", "ircd.m.dbs.events._room_state_space.meta_block.size" }, - { "default", 8192L }, -}; - -decltype(ircd::m::dbs::desc::events__room_state_space__cache__size) -ircd::m::dbs::desc::events__room_state_space__cache__size -{ - { - { "name", "ircd.m.dbs.events._room_state_space.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - const size_t &value{events__room_state_space__cache__size}; - db::capacity(db::cache(room_state_space), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_state_space__cache_comp__size) -ircd::m::dbs::desc::events__room_state_space__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events._room_state_space.cache_comp.size" }, - { "default", long(8_MiB) }, - }, [] - { - const size_t &value{events__room_state_space__cache_comp__size}; - db::capacity(db::cache_compressed(room_state_space), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_state_space__bloom__bits) -ircd::m::dbs::desc::events__room_state_space__bloom__bits -{ - { "name", "ircd.m.dbs.events._room_state_space.bloom.bits" }, - { "default", 10L }, -}; - -ircd::string_view -ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, - const id::room &room_id) -{ - return room_state_space_key(out_, room_id, string_view{}, string_view{}, -1L, 0L); -} - -ircd::string_view -ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &type) -{ - return room_state_space_key(out_, room_id, type, string_view{}, -1L, 0L); -} - -ircd::string_view -ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &type, - const string_view &state_key) -{ - return room_state_space_key(out_, room_id, type, state_key, -1L, 0L); -} - -ircd::string_view -ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, - const id::room &room_id, - const string_view &type, - const string_view &state_key, - const int64_t &depth, - const event::idx &event_idx) -{ - mutable_buffer out{out_}; - consume(out, copy(out, room_id)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, type)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, state_key)); - consume(out, copy(out, "\0"_sv)); - consume(out, copy(out, byte_view(depth))); - consume(out, copy(out, byte_view(event_idx))); - return { data(out_), data(out) }; -} - -ircd::m::dbs::room_state_space_key_parts -ircd::m::dbs::room_state_space_key(const string_view &amalgam) -{ - const auto &key - { - lstrip(amalgam, "\0"_sv) - }; - - const auto &[type, after_type] - { - split(key, "\0"_sv) - }; - - const auto &[state_key, after_state_key] - { - split(after_type, "\0"_sv) - }; - - const int64_t &depth - { - size(after_state_key) >= 8? - int64_t(byte_view(after_state_key.substr(0, 8))): - -1L - }; - - const event::idx &event_idx - { - size(after_state_key) >= 16? - event::idx(byte_view(after_state_key.substr(8, 8))): - 0UL - }; - - return - { - type, state_key, depth, event_idx - }; -} - -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__room_state_space__pfx -{ - "_room_state_space", - [](const string_view &key) - { - return has(key, "\0"_sv); - }, - - [](const string_view &key) - { - return split(key, "\0"_sv).first; - } -}; - -const ircd::db::comparator -ircd::m::dbs::desc::events__room_state_space__cmp -{ - "_room_state_space", - - // less - [](const string_view &a, const string_view &b) - { - static const auto &pt - { - events__room_state_space__pfx - }; - - const string_view pre[2] - { - pt.get(a), - pt.get(b), - }; - - if(size(pre[0]) != size(pre[1])) - return size(pre[0]) < size(pre[1]); - - if(pre[0] != pre[1]) - return pre[0] < pre[1]; - - const string_view post[2] - { - a.substr(size(pre[0])), - b.substr(size(pre[1])), - }; - - // These conditions are matched on some queries when the user only - // supplies a room_id. - if(empty(post[0])) - return true; - - if(empty(post[1])) - return false; - - // Decompose the postfix of the key for granular sorting - const room_state_space_key_parts k[2] - { - room_state_space_key(post[0]), - room_state_space_key(post[1]) - }; - - // type - if(std::get<0>(k[0]) < std::get<0>(k[1])) - return true; - else if(std::get<0>(k[0]) > std::get<0>(k[1])) - return false; - - // state_key - if(std::get<1>(k[0]) < std::get<1>(k[1])) - return true; - else if(std::get<1>(k[0]) > std::get<1>(k[1])) - return false; - - // depth (ORDER IS DESCENDING!) - if(uint64_t(std::get<2>(k[0])) > uint64_t(std::get<2>(k[1]))) - return true; - else if(uint64_t(std::get<2>(k[0])) < uint64_t(std::get<2>(k[1]))) - return false; - - // event_idx (ORDER IS DESCENDING!) - if(std::get<3>(k[0]) > std::get<3>(k[1])) - return true; - else if(std::get<3>(k[0]) < std::get<3>(k[1])) - return false; - - return false; - }, - - // equal - [](const string_view &a, const string_view &b) - { - return a == b; - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__room_state_space -{ - // name - "_room_state_space", - - // explanation - R"(All states of the room. - - )", - - // typing (key, value) - { - typeid(string_view), typeid(uint64_t) - }, - - // options - {}, - - // comparator - events__room_state_space__cmp, - - // prefix transform - events__room_state_space__pfx, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events__room_state_space__bloom__bits), - - // expect queries hit - false, - - // block size - size_t(events__room_state_space__block__size), - - // meta_block size - size_t(events__room_state_space__meta_block__size), - - // compression - "kLZ4Compression;kSnappyCompression"s, - - // compactor - {}, - - // compaction priority algorithm - "kOldestSmallestSeqFirst"s, -}; - -// -// Direct column descriptors -// - -decltype(ircd::m::dbs::desc::events___event__bloom__bits) -ircd::m::dbs::desc::events___event__bloom__bits -{ - { "name", "ircd.m.dbs.events.__event.bloom.bits" }, - { "default", 8L }, -}; - -// -// event_id -// - -decltype(ircd::m::dbs::desc::events__event_id__block__size) -ircd::m::dbs::desc::events__event_id__block__size -{ - { "name", "ircd.m.dbs.events.event_id.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_id__meta_block__size) -ircd::m::dbs::desc::events__event_id__meta_block__size -{ - { "name", "ircd.m.dbs.events.event_id.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__event_id__cache__size) -ircd::m::dbs::desc::events__event_id__cache__size -{ - { - { "name", "ircd.m.dbs.events.event_id.cache.size" }, - { "default", long(32_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__event_id__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__event_id__cache_comp__size) -ircd::m::dbs::desc::events__event_id__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.event_id.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__event_id__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_event_id -{ - // name - "event_id", - - // explanation - R"(Stores the event_id property of an event. - - As with all direct event columns the key is an event_idx and the value - is the data for the event. It should be mentioned for this column - specifically that event_id's are already saved in the _event_idx column - however that is a mapping of event_id to event_idx whereas this is a - mapping of event_idx to event_id. - - 10.4 - MUST NOT exceed 255 bytes. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__event_id__block__size), - - // meta_block size - size_t(events__event_id__meta_block__size), -}; - -// -// type -// - -decltype(ircd::m::dbs::desc::events__type__block__size) -ircd::m::dbs::desc::events__type__block__size -{ - { "name", "ircd.m.dbs.events.type.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__type__meta_block__size) -ircd::m::dbs::desc::events__type__meta_block__size -{ - { "name", "ircd.m.dbs.events.type.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__type__cache__size) -ircd::m::dbs::desc::events__type__cache__size -{ - { - { "name", "ircd.m.dbs.events.type.cache.size" }, - { "default", long(32_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__type__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__type__cache_comp__size) -ircd::m::dbs::desc::events__type__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.type.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__type__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_type -{ - // name - "type", - - // explanation - R"(Stores the type property of an event. - - 10.1 - The type of event. This SHOULD be namespaced similar to Java package naming conventions - e.g. 'com.example.subdomain.event.type'. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_idx number. - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__type__block__size), - - // meta_block size - size_t(events__type__meta_block__size), -}; - -// -// content -// - -decltype(ircd::m::dbs::desc::events__content__block__size) -ircd::m::dbs::desc::events__content__block__size -{ - { "name", "ircd.m.dbs.events.content.block.size" }, - { "default", 2048L }, -}; - -decltype(ircd::m::dbs::desc::events__content__meta_block__size) -ircd::m::dbs::desc::events__content__meta_block__size -{ - { "name", "ircd.m.dbs.events.content.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__content__cache__size) -ircd::m::dbs::desc::events__content__cache__size -{ - { - { "name", "ircd.m.dbs.events.content.cache.size" }, - { "default", long(48_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__content__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__content__cache_comp__size) -ircd::m::dbs::desc::events__content__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.content.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__content__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_content -{ - // name - "content", - - // explanation - R"(Stores the content property of an event. - - 10.1 - The fields in this object will vary depending on the type of event. When interacting - with the REST API, this is the HTTP body. - - ### developer note: - Since events must not exceed 64 KiB the maximum size for the content is the remaining - space after all the other fields for the event are rendered. - - key is event_idx number. - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__content__block__size), - - // meta_block size - size_t(events__content__meta_block__size), -}; - -// -// room_id -// - -decltype(ircd::m::dbs::desc::events__room_id__block__size) -ircd::m::dbs::desc::events__room_id__block__size -{ - { "name", "ircd.m.dbs.events.room_id.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__room_id__meta_block__size) -ircd::m::dbs::desc::events__room_id__meta_block__size -{ - { "name", "ircd.m.dbs.events.room_id.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__room_id__cache__size) -ircd::m::dbs::desc::events__room_id__cache__size -{ - { - { "name", "ircd.m.dbs.events.room_id.cache.size" }, - { "default", long(32_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__room_id__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__room_id__cache_comp__size) -ircd::m::dbs::desc::events__room_id__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.room_id.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__room_id__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_room_id -{ - // name - "room_id", - - // explanation - R"(Stores the room_id property of an event. - - 10.2 (apropos room events) - Required. The ID of the room associated with this event. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_idx number. - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__room_id__block__size), - - // meta_block size - size_t(events__room_id__meta_block__size), -}; - -// -// sender -// - -decltype(ircd::m::dbs::desc::events__sender__block__size) -ircd::m::dbs::desc::events__sender__block__size -{ - { "name", "ircd.m.dbs.events.sender.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__sender__meta_block__size) -ircd::m::dbs::desc::events__sender__meta_block__size -{ - { "name", "ircd.m.dbs.events.sender.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__sender__cache__size) -ircd::m::dbs::desc::events__sender__cache__size -{ - { - { "name", "ircd.m.dbs.events.sender.cache.size" }, - { "default", long(32_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__sender__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__sender__cache_comp__size) -ircd::m::dbs::desc::events__sender__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.sender.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__sender__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_sender -{ - // name - "sender", - - // explanation - R"(Stores the sender property of an event. - - 10.2 (apropos room events) - Required. Contains the fully-qualified ID of the user who sent this event. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_idx number. - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__sender__block__size), - - // meta_block size - size_t(events__sender__meta_block__size), -}; - -// -// state_key -// - -decltype(ircd::m::dbs::desc::events__state_key__block__size) -ircd::m::dbs::desc::events__state_key__block__size -{ - { "name", "ircd.m.dbs.events.state_key.block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__state_key__meta_block__size) -ircd::m::dbs::desc::events__state_key__meta_block__size -{ - { "name", "ircd.m.dbs.events.state_key.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__state_key__cache__size) -ircd::m::dbs::desc::events__state_key__cache__size -{ - { - { "name", "ircd.m.dbs.events.state_key.cache.size" }, - { "default", long(32_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__state_key__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__state_key__cache_comp__size) -ircd::m::dbs::desc::events__state_key__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.state_key.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__state_key__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_state_key -{ - // name - "state_key", - - // explanation - R"(Stores the state_key property of an event. - - 10.3 (apropos room state events) - A unique key which defines the overwriting semantics for this piece of room state. - This value is often a zero-length string. The presence of this key makes this event a - State Event. The key MUST NOT start with '_'. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_idx number. - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__state_key__block__size), - - // meta_block size - size_t(events__state_key__meta_block__size), -}; - -// -// origin_server_ts -// - -decltype(ircd::m::dbs::desc::events__origin_server_ts__block__size) -ircd::m::dbs::desc::events__origin_server_ts__block__size -{ - { "name", "ircd.m.dbs.events.origin_server_ts.block.size" }, - { "default", 256L }, -}; - -decltype(ircd::m::dbs::desc::events__origin_server_ts__meta_block__size) -ircd::m::dbs::desc::events__origin_server_ts__meta_block__size -{ - { "name", "ircd.m.dbs.events.origin_server_ts.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__origin_server_ts__cache__size) -ircd::m::dbs::desc::events__origin_server_ts__cache__size -{ - { - { "name", "ircd.m.dbs.events.origin_server_ts.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__origin_server_ts__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__origin_server_ts__cache_comp__size) -ircd::m::dbs::desc::events__origin_server_ts__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.origin_server_ts.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__origin_server_ts__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_origin_server_ts -{ - // name - "origin_server_ts", - - // explanation - R"(Stores the origin_server_ts property of an event. - - FEDERATION 4.1 - Timestamp in milliseconds on origin homeserver when this PDU was created. - - ### developer note: - key is event_idx number. - value is a machine integer (binary) - - TODO: consider unsigned rather than time_t because of millisecond precision - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(time_t) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__origin_server_ts__block__size), - - // meta_block size - size_t(events__origin_server_ts__meta_block__size), -}; - -// -// depth -// - -decltype(ircd::m::dbs::desc::events__depth__block__size) -ircd::m::dbs::desc::events__depth__block__size -{ - { "name", "ircd.m.dbs.events.depth.block.size" }, - { "default", 256L }, -}; - -decltype(ircd::m::dbs::desc::events__depth__meta_block__size) -ircd::m::dbs::desc::events__depth__meta_block__size -{ - { "name", "ircd.m.dbs.events.depth.meta_block.size" }, - { "default", 512L }, -}; - -decltype(ircd::m::dbs::desc::events__depth__cache__size) -ircd::m::dbs::desc::events__depth__cache__size -{ - { - { "name", "ircd.m.dbs.events.depth.cache.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__depth__cache__size}; - db::capacity(db::cache(column), value); - } -}; - -decltype(ircd::m::dbs::desc::events__depth__cache_comp__size) -ircd::m::dbs::desc::events__depth__cache_comp__size -{ - { - { "name", "ircd.m.dbs.events.depth.cache_comp.size" }, - { "default", long(16_MiB) }, - }, [] - { - auto &column(event_column.at(json::indexof())); - const size_t &value{events__depth__cache_comp__size}; - db::capacity(db::cache_compressed(column), value); - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_depth -{ - // name - "depth", - - // explanation - R"(Stores the depth property of an event. - - ### developer note: - key is event_idx number. value is long integer - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(int64_t) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - bool(events_cache_enable)? -1 : 0, - - // cache size for compressed assets - bool(events_cache_comp_enable)? -1 : 0, - - // bloom filter bits - size_t(events___event__bloom__bits), - - // expect queries hit - true, - - // block size - size_t(events__depth__block__size), - - // meta_block size - size_t(events__depth__meta_block__size), -}; - -// -// Other column descriptions -// - -namespace ircd::m::dbs::desc -{ - // Deprecated / dropped columns. - // - // These have to be retained for users that have yet to open their - // database with a newly released schema which has dropped a column - // from the schema. If the legacy descriptor is not provided here then - // the database will not know how to open the descriptor in order to - // conduct the drop. - - extern const ircd::db::descriptor events_auth_events; - extern const ircd::db::descriptor events_hashes; - extern const ircd::db::descriptor events_membership; - extern const ircd::db::descriptor events_origin; - extern const ircd::db::descriptor events_prev_events; - extern const ircd::db::descriptor events_prev_state; - extern const ircd::db::descriptor events_redacts; - extern const ircd::db::descriptor events_signatures; - extern const ircd::db::descriptor events__event_auth; - extern const ircd::db::comparator events__event_auth__cmp; - extern const ircd::db::prefix_transform events__event_auth__pfx; - extern const ircd::db::descriptor events__event_bad; - extern const ircd::db::descriptor events__state_node; - - // - // Required by RocksDB - // - - extern const ircd::db::descriptor events__default; -}; - -const ircd::db::prefix_transform -ircd::m::dbs::desc::events__event_auth__pfx -{ - "_event_auth", - [](const string_view &key) - { - return size(key) >= sizeof(event::idx) * 2; - }, - - [](const string_view &key) - { - assert(size(key) >= sizeof(event::idx)); - return string_view - { - data(key), data(key) + sizeof(event::idx) - }; - } -}; - -const ircd::db::comparator -ircd::m::dbs::desc::events__event_auth__cmp -{ - "_event_auth", - - // less - [](const string_view &a, const string_view &b) - { - static const size_t half(sizeof(event::idx)); - static const size_t full(half * 2); - - assert(size(a) >= half); - assert(size(b) >= half); - const event::idx *const key[2] - { - reinterpret_cast(data(a)), - reinterpret_cast(data(b)), - }; - - return - key[0][0] < key[1][0]? true: - key[0][0] > key[1][0]? false: - size(a) < size(b)? true: - size(a) > size(b)? false: - size(a) == half? false: - key[0][1] < key[1][1]? true: - false; - }, - - // equal - [](const string_view &a, const string_view &b) - { - return size(a) == size(b) && memcmp(data(a), data(b), size(a)) == 0; - } -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_auth -{ - // name - "_event_auth", - - // explanation - R"(Inverse reference graph of events. - - event_idx | ref, event_idx => -- - - The first part of the key is the event being referenced. The second part - of the key is the event which refers to the first event somewhere in its - prev_events references. The event_idx in the second part of the key also - contains a dbs::ref type in its highest order byte so we can store - different kinds of references. - - The prefix transform is in effect; an event may be referenced multiple - times. We can find all the events we have which reference a target, and - why. The database must already contain both events (hence they have - event::idx numbers). - - The value is currently unused/empty; we may eventually store metadata with - information about this reference (i.e. is depth adjacent? is the ref - redundant with another in the same event and should not be made? etc). - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - events__event_auth__cmp, - - // prefix transform - events__event_auth__pfx, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__event_bad -{ - // name - "_event_bad", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(string_view), typeid(uint64_t) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_auth_events -{ - // name - "auth_events", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_hashes -{ - // name - "hashes", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_membership -{ - // name - "membership", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_origin -{ - // name - "origin", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_prev_events -{ - // name - "prev_events", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_prev_state -{ - // name - "prev_state", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(ircd::string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_redacts -{ - // name - "redacts", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events_signatures -{ - // name - "signatures", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(uint64_t), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__state_node -{ - // name - "_state_node", - - // explanation - R"( - - This column is deprecated and has been dropped from the schema. This - descriptor will erase its presence in the database upon next open. - - )", - - // typing (key, value) - { - typeid(ircd::string_view), typeid(ircd::string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - true, -}; - -const ircd::db::descriptor -ircd::m::dbs::desc::events__default -{ - // name - "default", - - // explanation - R"(This column is unused but required by the database software. - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, - - // drop column - false, - - // cache size - 0_MiB, - - // cache size for compressed assets - 0_MiB, - - // bloom filter bits - 0, - - // expect queries hit - false, -}; - -// -// Description vector -// - -const ircd::db::description -ircd::m::dbs::desc::events -{ - // Requirement of RocksDB/LevelDB - events__default, - - // - // These columns directly represent event fields indexed by event_idx - // number and the value is the actual event values. Some values may be - // JSON, like content. - // - - events_content, - events_depth, - events_event_id, - events_origin, - events_origin_server_ts, - events_room_id, - events_sender, - events_state_key, - events_type, - - // - // These columns are metadata oriented around the event data. - // - - // event_id => uint64_t - // Mapping of event_id to index number. - events__event_idx, - - // event_idx => json - // Mapping of event_idx to full json - events__event_json, - - // event_idx | event_idx - // Reverse mapping of the event reference graph. - events__event_refs, - - // event_idx | event_idx - // Mapping of unresolved event refs. - events__event_horizon, - - // origin | sender, event_idx - // Mapping of senders to event_idx's they are the sender of. - events__event_sender, - - // type | event_idx - // Mapping of type strings to event_idx's of that type. - events__event_type, - - // state_key, type, room_id, depth, event_idx - // Mapping of event states, indexed for application features. - events__event_state, - - // (room_id, (depth, event_idx)) - // Sequence of all events for a room, ever. - events__room_events, - - // (room_id, (type, depth, event_idx)) - // Sequence of all events by type for a room. - events__room_type, - - // (room_id, (origin, user_id)) - // Sequence of all PRESENTLY JOINED joined for a room. - events__room_joined, - - // (room_id, (type, state_key)) => (event_idx) - // Sequence of the PRESENT STATE of the room. - events__room_state, - - // (room_id, (type, state_key, depth, event_idx)) - // Sequence of all states of the room. - events__room_state_space, - - // (room_id, event_id) => (event_idx) - // Mapping of all current head events for a room. - events__room_head, - - // - // These columns are legacy; they have been dropped from the schema. - // - - events_auth_events, - events_hashes, - events_membership, - events_prev_events, - events_prev_state, - events_redacts, - events_signatures, - events__event_auth, - events__event_bad, - events__state_node, -}; diff --git a/matrix/dbs_desc.cc b/matrix/dbs_desc.cc new file mode 100644 index 000000000..81327a795 --- /dev/null +++ b/matrix/dbs_desc.cc @@ -0,0 +1,547 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +namespace ircd::m::dbs::desc +{ + // Deprecated / dropped columns. + // + // These have to be retained for users that have yet to open their + // database with a newly released schema which has dropped a column + // from the schema. If the legacy descriptor is not provided here then + // the database will not know how to open the descriptor in order to + // conduct the drop. + + extern const ircd::db::descriptor events_auth_events; + extern const ircd::db::descriptor events_hashes; + extern const ircd::db::descriptor events_membership; + extern const ircd::db::descriptor events_origin; + extern const ircd::db::descriptor events_prev_events; + extern const ircd::db::descriptor events_prev_state; + extern const ircd::db::descriptor events_redacts; + extern const ircd::db::descriptor events_signatures; + extern const ircd::db::descriptor events__event_auth; + extern const ircd::db::comparator events__event_auth__cmp; + extern const ircd::db::prefix_transform events__event_auth__pfx; + extern const ircd::db::descriptor events__event_bad; + extern const ircd::db::descriptor events__state_node; + + // + // Required by RocksDB + // + + extern const ircd::db::descriptor events__default; +}; + +const ircd::db::prefix_transform +ircd::m::dbs::desc::events__event_auth__pfx +{ + "_event_auth", + nullptr, + nullptr, +}; + +const ircd::db::comparator +ircd::m::dbs::desc::events__event_auth__cmp +{ + "_event_auth", + nullptr, + nullptr, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events__event_auth +{ + // name + "_event_auth", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + events__event_auth__cmp, + + // prefix transform + events__event_auth__pfx, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events__event_bad +{ + // name + "_event_bad", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(uint64_t) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_auth_events +{ + // name + "auth_events", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_hashes +{ + // name + "hashes", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_membership +{ + // name + "membership", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_origin +{ + // name + "origin", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_prev_events +{ + // name + "prev_events", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_prev_state +{ + // name + "prev_state", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_redacts +{ + // name + "redacts", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events_signatures +{ + // name + "signatures", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events__state_node +{ + // name + "_state_node", + + // explanation + R"( + + This column is deprecated and has been dropped from the schema. This + descriptor will erase its presence in the database upon next open. + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + true, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::events__default +{ + // name + "default", + + // explanation + R"(This column is unused but required by the database software. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + 0_MiB, + + // cache size for compressed assets + 0_MiB, + + // bloom filter bits + 0, + + // expect queries hit + false, +}; + +// +// Description vector +// + +decltype(ircd::m::dbs::desc::events) +ircd::m::dbs::desc::events +{ + // Requirement of RocksDB/LevelDB + events__default, + + // + // These columns directly represent event fields indexed by event_idx + // number and the value is the actual event values. Some values may be + // JSON, like content. + // + + content, + depth, + event_id, + origin_server_ts, + room_id, + sender, + state_key, + type, + + // + // These columns are metadata oriented around the event data. + // + + // event_id => uint64_t + // Mapping of event_id to index number. + event_idx, + + // event_idx => json + // Mapping of event_idx to full json + event_json, + + // event_idx | event_idx + // Reverse mapping of the event reference graph. + event_refs, + + // event_idx | event_idx + // Mapping of unresolved event refs. + event_horizon, + + // origin | sender, event_idx + // Mapping of senders to event_idx's they are the sender of. + event_sender, + + // type | event_idx + // Mapping of type strings to event_idx's of that type. + event_type, + + // state_key, type, room_id, depth, event_idx + // Mapping of event states, indexed for application features. + event_state, + + // (room_id, (depth, event_idx)) + // Sequence of all events for a room, ever. + room_events, + + // (room_id, (type, depth, event_idx)) + // Sequence of all events by type for a room. + room_type, + + // (room_id, (origin, user_id)) + // Sequence of all PRESENTLY JOINED joined for a room. + room_joined, + + // (room_id, (type, state_key)) => (event_idx) + // Sequence of the PRESENT STATE of the room. + room_state, + + // (room_id, (type, state_key, depth, event_idx)) + // Sequence of all states of the room. + room_state_space, + + // (room_id, event_id) => (event_idx) + // Mapping of all current head events for a room. + room_head, + + // + // These columns are legacy; they have been dropped from the schema. + // + + events_auth_events, + events_hashes, + events_membership, + events_origin, + events_prev_events, + events_prev_state, + events_redacts, + events_signatures, + events__event_auth, + events__event_bad, + events__state_node, +}; diff --git a/matrix/dbs_event_column.cc b/matrix/dbs_event_column.cc new file mode 100644 index 000000000..0c4c9f984 --- /dev/null +++ b/matrix/dbs_event_column.cc @@ -0,0 +1,868 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +/// Linkage for a cache of the columns of the events database which directly +/// correspond to a property in the matrix event object. This array allows +/// for constant time access to a column the same way one can make constant +/// time access to a property in m::event. +decltype(ircd::m::dbs::event_column) +ircd::m::dbs::event_column; + +decltype(ircd::m::dbs::desc::_event__bloom__bits) +ircd::m::dbs::desc::_event__bloom__bits +{ + { "name", "ircd.m.dbs.__event.bloom.bits" }, + { "default", 8L }, +}; + +decltype(ircd::m::dbs::desc::event_id__block__size) +ircd::m::dbs::desc::event_id__block__size +{ + { "name", "ircd.m.dbs.event_id.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_id__meta_block__size) +ircd::m::dbs::desc::event_id__meta_block__size +{ + { "name", "ircd.m.dbs.event_id.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_id__cache__size) +ircd::m::dbs::desc::event_id__cache__size +{ + { + { "name", "ircd.m.dbs.event_id.cache.size" }, + { "default", long(32_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{event_id__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::event_id__cache_comp__size) +ircd::m::dbs::desc::event_id__cache_comp__size +{ + { + { "name", "ircd.m.dbs.event_id.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{event_id__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::event_id +{ + // name + "event_id", + + // explanation + R"(Stores the event_id property of an event. + + As with all direct event columns the key is an event_idx and the value + is the data for the event. It should be mentioned for this column + specifically that event_id's are already saved in the _event_idx column + however that is a mapping of event_id to event_idx whereas this is a + mapping of event_idx to event_id. + + 10.4 + MUST NOT exceed 255 bytes. + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(event_id__block__size), + + // meta_block size + size_t(event_id__meta_block__size), +}; + +// +// type +// + +decltype(ircd::m::dbs::desc::type__block__size) +ircd::m::dbs::desc::type__block__size +{ + { "name", "ircd.m.dbs.type.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::type__meta_block__size) +ircd::m::dbs::desc::type__meta_block__size +{ + { "name", "ircd.m.dbs.type.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::type__cache__size) +ircd::m::dbs::desc::type__cache__size +{ + { + { "name", "ircd.m.dbs.type.cache.size" }, + { "default", long(32_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{type__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::type__cache_comp__size) +ircd::m::dbs::desc::type__cache_comp__size +{ + { + { "name", "ircd.m.dbs.type.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{type__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::type +{ + // name + "type", + + // explanation + R"(Stores the type property of an event. + + 10.1 + The type of event. This SHOULD be namespaced similar to Java package naming conventions + e.g. 'com.example.subdomain.event.type'. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_idx number. + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(type__block__size), + + // meta_block size + size_t(type__meta_block__size), +}; + +// +// content +// + +decltype(ircd::m::dbs::desc::content__block__size) +ircd::m::dbs::desc::content__block__size +{ + { "name", "ircd.m.dbs.content.block.size" }, + { "default", 2048L }, +}; + +decltype(ircd::m::dbs::desc::content__meta_block__size) +ircd::m::dbs::desc::content__meta_block__size +{ + { "name", "ircd.m.dbs.content.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::content__cache__size) +ircd::m::dbs::desc::content__cache__size +{ + { + { "name", "ircd.m.dbs.content.cache.size" }, + { "default", long(48_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{content__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::content__cache_comp__size) +ircd::m::dbs::desc::content__cache_comp__size +{ + { + { "name", "ircd.m.dbs.content.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{content__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::content +{ + // name + "content", + + // explanation + R"(Stores the content property of an event. + + 10.1 + The fields in this object will vary depending on the type of event. When interacting + with the REST API, this is the HTTP body. + + ### developer note: + Since events must not exceed 64 KiB the maximum size for the content is the remaining + space after all the other fields for the event are rendered. + + key is event_idx number. + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(content__block__size), + + // meta_block size + size_t(content__meta_block__size), +}; + +// +// room_id +// + +decltype(ircd::m::dbs::desc::room_id__block__size) +ircd::m::dbs::desc::room_id__block__size +{ + { "name", "ircd.m.dbs.room_id.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::room_id__meta_block__size) +ircd::m::dbs::desc::room_id__meta_block__size +{ + { "name", "ircd.m.dbs.room_id.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::room_id__cache__size) +ircd::m::dbs::desc::room_id__cache__size +{ + { + { "name", "ircd.m.dbs.room_id.cache.size" }, + { "default", long(32_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{room_id__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::room_id__cache_comp__size) +ircd::m::dbs::desc::room_id__cache_comp__size +{ + { + { "name", "ircd.m.dbs.room_id.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{room_id__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::room_id +{ + // name + "room_id", + + // explanation + R"(Stores the room_id property of an event. + + 10.2 (apropos room events) + Required. The ID of the room associated with this event. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_idx number. + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(room_id__block__size), + + // meta_block size + size_t(room_id__meta_block__size), +}; + +// +// sender +// + +decltype(ircd::m::dbs::desc::sender__block__size) +ircd::m::dbs::desc::sender__block__size +{ + { "name", "ircd.m.dbs.sender.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::sender__meta_block__size) +ircd::m::dbs::desc::sender__meta_block__size +{ + { "name", "ircd.m.dbs.sender.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::sender__cache__size) +ircd::m::dbs::desc::sender__cache__size +{ + { + { "name", "ircd.m.dbs.sender.cache.size" }, + { "default", long(32_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{sender__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::sender__cache_comp__size) +ircd::m::dbs::desc::sender__cache_comp__size +{ + { + { "name", "ircd.m.dbs.sender.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{sender__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::sender +{ + // name + "sender", + + // explanation + R"(Stores the sender property of an event. + + 10.2 (apropos room events) + Required. Contains the fully-qualified ID of the user who sent this event. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_idx number. + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(sender__block__size), + + // meta_block size + size_t(sender__meta_block__size), +}; + +// +// state_key +// + +decltype(ircd::m::dbs::desc::state_key__block__size) +ircd::m::dbs::desc::state_key__block__size +{ + { "name", "ircd.m.dbs.state_key.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::state_key__meta_block__size) +ircd::m::dbs::desc::state_key__meta_block__size +{ + { "name", "ircd.m.dbs.state_key.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::state_key__cache__size) +ircd::m::dbs::desc::state_key__cache__size +{ + { + { "name", "ircd.m.dbs.state_key.cache.size" }, + { "default", long(32_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{state_key__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::state_key__cache_comp__size) +ircd::m::dbs::desc::state_key__cache_comp__size +{ + { + { "name", "ircd.m.dbs.state_key.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{state_key__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::state_key +{ + // name + "state_key", + + // explanation + R"(Stores the state_key property of an event. + + 10.3 (apropos room state events) + A unique key which defines the overwriting semantics for this piece of room state. + This value is often a zero-length string. The presence of this key makes this event a + State Event. The key MUST NOT start with '_'. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_idx number. + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(state_key__block__size), + + // meta_block size + size_t(state_key__meta_block__size), +}; + +// +// origin_server_ts +// + +decltype(ircd::m::dbs::desc::origin_server_ts__block__size) +ircd::m::dbs::desc::origin_server_ts__block__size +{ + { "name", "ircd.m.dbs.origin_server_ts.block.size" }, + { "default", 256L }, +}; + +decltype(ircd::m::dbs::desc::origin_server_ts__meta_block__size) +ircd::m::dbs::desc::origin_server_ts__meta_block__size +{ + { "name", "ircd.m.dbs.origin_server_ts.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::origin_server_ts__cache__size) +ircd::m::dbs::desc::origin_server_ts__cache__size +{ + { + { "name", "ircd.m.dbs.origin_server_ts.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{origin_server_ts__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::origin_server_ts__cache_comp__size) +ircd::m::dbs::desc::origin_server_ts__cache_comp__size +{ + { + { "name", "ircd.m.dbs.origin_server_ts.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{origin_server_ts__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::origin_server_ts +{ + // name + "origin_server_ts", + + // explanation + R"(Stores the origin_server_ts property of an event. + + FEDERATION 4.1 + Timestamp in milliseconds on origin homeserver when this PDU was created. + + ### developer note: + key is event_idx number. + value is a machine integer (binary) + + TODO: consider unsigned rather than time_t because of millisecond precision + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(time_t) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(origin_server_ts__block__size), + + // meta_block size + size_t(origin_server_ts__meta_block__size), +}; + +// +// depth +// + +decltype(ircd::m::dbs::desc::depth__block__size) +ircd::m::dbs::desc::depth__block__size +{ + { "name", "ircd.m.dbs.depth.block.size" }, + { "default", 256L }, +}; + +decltype(ircd::m::dbs::desc::depth__meta_block__size) +ircd::m::dbs::desc::depth__meta_block__size +{ + { "name", "ircd.m.dbs.depth.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::depth__cache__size) +ircd::m::dbs::desc::depth__cache__size +{ + { + { "name", "ircd.m.dbs.depth.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{depth__cache__size}; + db::capacity(db::cache(column), value); + } +}; + +decltype(ircd::m::dbs::desc::depth__cache_comp__size) +ircd::m::dbs::desc::depth__cache_comp__size +{ + { + { "name", "ircd.m.dbs.depth.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + auto &column(event_column.at(json::indexof())); + const size_t &value{depth__cache_comp__size}; + db::capacity(db::cache_compressed(column), value); + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::depth +{ + // name + "depth", + + // explanation + R"(Stores the depth property of an event. + + ### developer note: + key is event_idx number. value is long integer + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(int64_t) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(_event__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(depth__block__size), + + // meta_block size + size_t(depth__meta_block__size), +}; + +void +ircd::m::dbs::_index_event_cols(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_COLS)); + assert(opts.event_idx); + const byte_view key + { + opts.event_idx + }; + + size_t i{0}; + for_each(event, [&txn, &opts, &key, &i] + (const auto &, auto&& val) + { + auto &column + { + event_column.at(i++) + }; + + if(!column) + return; + + if(value_required(opts.op) && !defined(json::value(val))) + return; + + db::txn::append + { + txn, column, db::column::delta + { + opts.op, + string_view{key}, + value_required(opts.op)? + byte_view{val}: + byte_view{} + } + }; + }); +} diff --git a/matrix/dbs_event_horizon.cc b/matrix/dbs_event_horizon.cc new file mode 100644 index 000000000..4a797068d --- /dev/null +++ b/matrix/dbs_event_horizon.cc @@ -0,0 +1,310 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::event_horizon) +ircd::m::dbs::event_horizon; + +decltype(ircd::m::dbs::desc::event_horizon__block__size) +ircd::m::dbs::desc::event_horizon__block__size +{ + { "name", "ircd.m.dbs._event_horizon.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_horizon__meta_block__size) +ircd::m::dbs::desc::event_horizon__meta_block__size +{ + { "name", "ircd.m.dbs._event_horizon.meta_block.size" }, + { "default", 1024L }, +}; + +decltype(ircd::m::dbs::desc::event_horizon__cache__size) +ircd::m::dbs::desc::event_horizon__cache__size +{ + { + { "name", "ircd.m.dbs._event_horizon.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{event_horizon__cache__size}; + db::capacity(db::cache(dbs::event_horizon), value); + } +}; + +decltype(ircd::m::dbs::desc::event_horizon__cache_comp__size) +ircd::m::dbs::desc::event_horizon__cache_comp__size +{ + { + { "name", "ircd.m.dbs._event_horizon.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{event_horizon__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::event_horizon), value); + } +}; + +const ircd::db::prefix_transform +ircd::m::dbs::desc::event_horizon__pfx +{ + "_event_horizon", + + [](const string_view &key) + { + return has(key, '\0'); + }, + + [](const string_view &key) + { + assert(size(key) >= sizeof(event::idx)); + return split(key, '\0').first; + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::event_horizon +{ + // name + "_event_horizon", + + // explanation + R"(Unresolved references in the reverse reference graph of events. + + event_id | event_idx => -- + + The first part of the key is an event_id which the server does not have. + The suffix of the key is the index number of an event which the server + does have and it contains a reference to event_id. + + We use the information in this column to find all of the events which + have an unresolved reference to this event and complete the holes in the + event_refs graph which could not be completed without this event. + + When a new event is written to the database the event_horizon column is + queried seeking the event's ID. Each entry in event_horizon is the index + of an event which we previously wrote to the database without knowing the + index of the event currently being written (an out-of-order write). + + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + event_horizon__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, + + // expect queries hit + false, + + // block size + size_t(event_horizon__block__size), + + // meta_block size + size_t(event_horizon__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_horizon_resolve(db::txn &txn, + const event &event, + const write_opts &opts) +{ + char buf[EVENT_HORIZON_KEY_MAX_SIZE]; + assert(opts.appendix.test(appendix::EVENT_HORIZON_RESOLVE)); + assert(opts.event_idx != 0); + assert(event.event_id); + const string_view &key + { + event_horizon_key(buf, event.event_id) + }; + + auto it + { + dbs::event_horizon.begin(key) + }; + + for(; it; ++it) + { + const auto parts + { + event_horizon_key(it->first) + }; + + const auto &event_idx + { + std::get<0>(parts) + }; + + assert(event_idx != 0); + assert(event_idx != opts.event_idx); + const event::fetch _event + { + event_idx, std::nothrow + }; + + if(!_event.valid) + { + log::dwarning + { + log, "Horizon resolve for %s @%lu not possible @%lu", + string_view{event.event_id}, + opts.event_idx, + event_idx, + }; + + continue; + } + + log::debug + { + log, "Horizon resolve for %s @%lu; remisé %s @%lu", + string_view{event.event_id}, + opts.event_idx, + string_view{_event.event_id}, + event_idx, + }; + + // Make the references on behalf of the future event + write_opts _opts; + _opts.op = opts.op; + _opts.event_idx = event_idx; + _opts.appendix.reset(); + _opts.appendix.set(appendix::EVENT_REFS); + _opts.appendix.set(appendix::ROOM_REDACT); + _opts.event_refs = opts.horizon_resolve; + _opts.interpose = &txn; + write(txn, _event, _opts); + + // Delete the event_horizon entry after resolving. + thread_local char buf[EVENT_HORIZON_KEY_MAX_SIZE]; + const string_view &key + { + event_horizon_key(buf, event.event_id, event_idx) + }; + + db::txn::append + { + txn, dbs::event_horizon, + { + opts.op == db::op::SET? + db::op::DELETE: + db::op::SET, + key + } + }; + } +} + +void +ircd::m::dbs::_index_event_horizon(db::txn &txn, + const event &event, + const write_opts &opts, + const m::event::id &unresolved_id) +{ + thread_local char buf[EVENT_HORIZON_KEY_MAX_SIZE]; + assert(opts.appendix.test(appendix::EVENT_HORIZON)); + assert(opts.event_idx != 0 && unresolved_id); + const string_view &key + { + event_horizon_key(buf, unresolved_id, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_horizon, + { + opts.op, key + } + }; +} + +// +// key +// + +std::tuple +ircd::m::dbs::event_horizon_key(const string_view &amalgam) +{ + assert(size(amalgam) == 1 + sizeof(event::idx)); + assert(amalgam[0] == '\0'); + + const byte_view &event_idx + { + amalgam.substr(1) + }; + + return + { + static_cast(event_idx) + }; +} + +ircd::string_view +ircd::m::dbs::event_horizon_key(const mutable_buffer &out, + const event::id &event_id) +{ + return event_horizon_key(out, event_id, 0UL); +} + +ircd::string_view +ircd::m::dbs::event_horizon_key(const mutable_buffer &out, + const event::id &event_id, + const event::idx &event_idx) +{ + mutable_buffer buf(out); + consume(buf, copy(buf, event_id)); + + if(event_idx) + { + consume(buf, copy(buf, "\0"_sv)); + consume(buf, copy(buf, byte_view(event_idx))); + } + + const string_view ret + { + data(out), data(buf) + }; + + assert(size(ret) == size(event_id) || size(ret) == size(event_id) + sizeof(event::idx) + 1); + return ret; +} diff --git a/matrix/dbs_event_idx.cc b/matrix/dbs_event_idx.cc new file mode 100644 index 000000000..bcb6f794f --- /dev/null +++ b/matrix/dbs_event_idx.cc @@ -0,0 +1,170 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::event_idx) +ircd::m::dbs::event_idx; + +decltype(ircd::m::dbs::desc::event_idx__block__size) +ircd::m::dbs::desc::event_idx__block__size +{ + { "name", "ircd.m.dbs._event_idx.block.size" }, + { "default", 256L }, +}; + +decltype(ircd::m::dbs::desc::event_idx__meta_block__size) +ircd::m::dbs::desc::event_idx__meta_block__size +{ + { "name", "ircd.m.dbs._event_idx.meta_block.size" }, + { "default", 2048L }, +}; + +decltype(ircd::m::dbs::desc::event_idx__cache__size) +ircd::m::dbs::desc::event_idx__cache__size +{ + { + { "name", "ircd.m.dbs._event_idx.cache.size" }, + { "default", long(64_MiB) }, + }, [] + { + const size_t &value{event_idx__cache__size}; + db::capacity(db::cache(dbs::event_idx), value); + } +}; + +decltype(ircd::m::dbs::desc::event_idx__cache_comp__size) +ircd::m::dbs::desc::event_idx__cache_comp__size +{ + { + { "name", "ircd.m.dbs._event_idx.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{event_idx__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::event_idx), value); + } +}; + +decltype(ircd::m::dbs::desc::event_idx__bloom__bits) +ircd::m::dbs::desc::event_idx__bloom__bits +{ + { "name", "ircd.m.dbs._event_idx.bloom.bits" }, + { "default", 10L }, +}; + +decltype(ircd::m::dbs::desc::event_idx) +ircd::m::dbs::desc::event_idx +{ + // name + "_event_idx", + + // explanation + R"(Maps matrix event_id strings into internal index numbers. + + event_id => event_idx + + The key is an event_id and the value is the index number to be used as the + key to all the event data columns. The index number is referred to as the + event_idx and is a fixed 8 byte unsigned integer. All other columns which + may key on an event_id string instead use this event_idx index number. The + index number was generated sequentially based on the order the event was + written to the database. Index numbers start at 1 because 0 is used as a + sentinel value and is not valid. The index numbers throughout the database + generally do not have gaps and can be iterated, however gaps may exist when + an event is erased from the database (which is rare for the matrix + application). + + )", + + // typing (key, value) + { + typeid(string_view), typeid(uint64_t) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(event_idx__bloom__bits), + + // expect queries hit + false, + + // block size + size_t(event_idx__block__size), + + // meta_block size + size_t(event_idx__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_event_id(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_ID)); + assert(opts.event_idx); + assert(event.event_id); + + db::txn::append + { + txn, dbs::event_idx, + { + opts.op, + string_view{event.event_id}, + byte_view(opts.event_idx) + } + }; + + // For a v1 event, the "event_id" property will be saved into the `event_id` + // column by the direct property->column indexer. + if(json::get<"event_id"_>(event)) + return; + + // For v3+ events, the direct column indexer won't see any "event_id" + // property. In this case we insert the `event.event_id` manually into + // that column here. + db::txn::append + { + txn, event_column.at(json::indexof()), + { + opts.op, + byte_view(opts.event_idx), + string_view{event.event_id}, + } + }; +} diff --git a/matrix/dbs_event_json.cc b/matrix/dbs_event_json.cc new file mode 100644 index 000000000..dd885c244 --- /dev/null +++ b/matrix/dbs_event_json.cc @@ -0,0 +1,188 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::event_json) +ircd::m::dbs::event_json; + +decltype(ircd::m::dbs::desc::event_json__block__size) +ircd::m::dbs::desc::event_json__block__size +{ + { "name", "ircd.m.dbs._event_json.block.size" }, + { "default", long(1_KiB) }, +}; + +decltype(ircd::m::dbs::desc::event_json__meta_block__size) +ircd::m::dbs::desc::event_json__meta_block__size +{ + { "name", "ircd.m.dbs._event_json.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_json__cache__size) +ircd::m::dbs::desc::event_json__cache__size +{ + { + { "name", "ircd.m.dbs._event_json.cache.size" }, + { "default", long(64_MiB) }, + }, [] + { + const size_t &value{event_json__cache__size}; + db::capacity(db::cache(dbs::event_json), value); + } +}; + +decltype(ircd::m::dbs::desc::event_json__cache_comp__size) +ircd::m::dbs::desc::event_json__cache_comp__size +{ + { + { "name", "ircd.m.dbs._event_json.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{event_json__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::event_json), value); + } +}; + +decltype(ircd::m::dbs::desc::event_json__bloom__bits) +ircd::m::dbs::desc::event_json__bloom__bits +{ + { "name", "ircd.m.dbs._event_json.bloom.bits" }, + { "default", 9L }, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::event_json +{ + // name + "_event_json", + + // explanation + R"(Full JSON object of an event. + + event_idx => event_json + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(event_json__bloom__bits), + + // expect queries hit + true, + + // block size + size_t(event_json__block__size), + + // meta_block size + size_t(event_json__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestLargestSeqFirst"s, + + // target_file_size + { + 2_GiB, // base + 1L, // multiplier + }, + + // max_bytes_for_level[8] + { + { 128_MiB, 1L }, // max_bytes_for_level_base + { 0L, 0L }, // max_bytes_for_level[0] + { 0L, 1L }, // max_bytes_for_level[1] + { 0L, 1L }, // max_bytes_for_level[2] + { 0L, 3L }, // max_bytes_for_level[3] + { 0L, 7L }, // max_bytes_for_level[4] + { 0L, 15L }, // max_bytes_for_level[5] + { 0L, 31L }, // max_bytes_for_level[6] + }, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_event_json(db::txn &txn, + const event &event, + const write_opts &opts) +{ + const ctx::critical_assertion ca; + thread_local char buf[m::event::MAX_SIZE]; + assert(opts.appendix.test(appendix::EVENT_JSON)); + assert(opts.event_idx); + + const string_view &key + { + byte_view(opts.event_idx) + }; + + const string_view &val + { + // If an already-strung json::object is carried by the event and + // the opts allow us, we use it directly. This is not the default + // path unless the developer knows the source JSON is good enough + // to store directly. + opts.op == db::op::SET && event.source && opts.json_source? + string_view{event.source}: + + // If an already-strung json::object is carried by the event we + // re-stringify it into a temporary buffer. This is the common case + // because the original source might be crap JSON w/ spaces etc. + opts.op == db::op::SET && event.source? + json::stringify(mutable_buffer{buf}, event.source): + + // If no source was given with the event we can generate it. + opts.op == db::op::SET? + json::stringify(mutable_buffer{buf}, event): + + // Empty value; generally for a non-SET db::op + string_view{} + }; + + db::txn::append + { + txn, event_json, + { + opts.op, // db::op + key, // key + val, // val + } + }; +} diff --git a/matrix/dbs_event_refs.cc b/matrix/dbs_event_refs.cc new file mode 100644 index 000000000..89636b81d --- /dev/null +++ b/matrix/dbs_event_refs.cc @@ -0,0 +1,782 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +namespace ircd::m::dbs +{ + static void _index_event_refs_m_room_redaction(db::txn &, const event &, const write_opts &); //query + static void _index_event_refs_m_receipt_m_read(db::txn &, const event &, const write_opts &); //query + static void _index_event_refs_m_relates_m_reply(db::txn &, const event &, const write_opts &); //query + static void _index_event_refs_m_relates(db::txn &, const event &, const write_opts &); //query + static void _index_event_refs_state(db::txn &, const event &, const write_opts &); // query + static void _index_event_refs_auth(db::txn &, const event &, const write_opts &); //query + static void _index_event_refs_prev(db::txn &, const event &, const write_opts &); //query + static bool event_refs__cmp_less(const string_view &a, const string_view &b); +} + +decltype(ircd::m::dbs::event_refs) +ircd::m::dbs::event_refs; + +decltype(ircd::m::dbs::desc::event_refs__block__size) +ircd::m::dbs::desc::event_refs__block__size +{ + { "name", "ircd.m.dbs._event_refs.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_refs__meta_block__size) +ircd::m::dbs::desc::event_refs__meta_block__size +{ + { "name", "ircd.m.dbs._event_refs.meta_block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_refs__cache__size) +ircd::m::dbs::desc::event_refs__cache__size +{ + { + { "name", "ircd.m.dbs._event_refs.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{event_refs__cache__size}; + db::capacity(db::cache(dbs::event_refs), value); + } +}; + +decltype(ircd::m::dbs::desc::event_refs__cache_comp__size) +ircd::m::dbs::desc::event_refs__cache_comp__size +{ + { + { "name", "ircd.m.dbs._event_refs.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{event_refs__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::event_refs), value); + } +}; + +const ircd::db::prefix_transform +ircd::m::dbs::desc::event_refs__pfx +{ + "_event_refs", + [](const string_view &key) + { + return size(key) >= sizeof(event::idx) * 2; + }, + + [](const string_view &key) + { + assert(size(key) >= sizeof(event::idx)); + return string_view + { + data(key), data(key) + sizeof(event::idx) + }; + } +}; + +const ircd::db::comparator +ircd::m::dbs::desc::event_refs__cmp +{ + "_event_refs", + event_refs__cmp_less, + std::equal_to{}, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::event_refs +{ + // name + "_event_refs", + + // explanation + R"(Inverse reference graph of events. + + event_idx | ref, event_idx => -- + + The first part of the key is the event being referenced. The second part + of the key is the event which refers to the first event somewhere in its + prev_events references. The event_idx in the second part of the key also + contains a dbs::ref type in its highest order byte so we can store + different kinds of references. + + The prefix transform is in effect; an event may be referenced multiple + times. We can find all the events we have which reference a target, and + why. The database must already contain both events (hence they have + event::idx numbers). + + The value is currently unused/empty; we may eventually store metadata with + information about this reference (i.e. is depth adjacent? is the ref + redundant with another in the same event and should not be made? etc). + + )", + + // typing (key, value) + { + typeid(uint64_t), typeid(string_view) + }, + + // options + {}, + + // comparator + event_refs__cmp, + + // prefix transform + event_refs__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, + + // expect queries hit + true, + + // block size + size_t(event_refs__block__size), + + // meta_block size + size_t(event_refs__meta_block__size), + + // compression + {}, // no compression for this column + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexers +// + +void +ircd::m::dbs::_index_event_refs(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + + if(opts.event_refs.test(uint(ref::NEXT))) + _index_event_refs_prev(txn, event, opts); + + if(opts.event_refs.test(uint(ref::NEXT_AUTH))) + _index_event_refs_auth(txn, event, opts); + + if(opts.event_refs.test(uint(ref::NEXT_STATE)) || + opts.event_refs.test(uint(ref::PREV_STATE))) + _index_event_refs_state(txn, event, opts); + + if(opts.event_refs.test(uint(ref::M_RECEIPT__M_READ))) + _index_event_refs_m_receipt_m_read(txn, event, opts); + + if(opts.event_refs.test(uint(ref::M_RELATES))) + _index_event_refs_m_relates(txn, event, opts); + + if(opts.event_refs.test(uint(ref::M_RELATES))) + _index_event_refs_m_relates_m_reply(txn, event, opts); + + if(opts.event_refs.test(uint(ref::M_ROOM_REDACTION))) + _index_event_refs_m_room_redaction(txn, event, opts); +} + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_refs_prev(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + assert(opts.event_refs.test(uint(ref::NEXT))); + + const event::prev prev{event}; + for(size_t i(0); i < prev.prev_events_count(); ++i) + { + const event::id &prev_id + { + prev.prev_event(i) + }; + + const event::idx &prev_idx + { + find_event_idx(prev_id, opts) + }; + + if(opts.appendix.test(appendix::EVENT_HORIZON) && !prev_idx) + { + _index_event_horizon(txn, event, opts, prev_id); + continue; + } + else if(!prev_idx) + { + log::dwarning + { + log, "No index found to ref %s PREV of %s", + string_view{prev_id}, + string_view{event.event_id}, + }; + + continue; + } + + thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; + assert(opts.event_idx != 0 && prev_idx != 0); + assert(opts.event_idx != prev_idx); + const string_view &key + { + event_refs_key(buf, prev_idx, ref::NEXT, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; + } +} + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_refs_auth(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + assert(opts.event_refs.test(uint(ref::NEXT_AUTH))); + + if(!m::room::auth::is_power_event(event)) + return; + + const event::prev prev{event}; + for(size_t i(0); i < prev.auth_events_count(); ++i) + { + const event::id &auth_id + { + prev.auth_event(i) + }; + + const event::idx &auth_idx + { + find_event_idx(auth_id, opts) + }; + + if(unlikely(!auth_idx)) + { + if(opts.appendix.test(appendix::EVENT_HORIZON)) + _index_event_horizon(txn, event, opts, auth_id); + + log::error + { + log, "No index found to ref %s AUTH of %s", + string_view{auth_id}, + string_view{event.event_id} + }; + + continue; + } + + thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; + assert(opts.event_idx != 0 && auth_idx != 0); + assert(opts.event_idx != auth_idx); + const string_view &key + { + event_refs_key(buf, auth_idx, ref::NEXT_AUTH, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; + } +} + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_refs_state(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + assert(opts.event_refs.test(uint(ref::NEXT_STATE)) || + opts.event_refs.test(uint(ref::PREV_STATE))); + + if(!json::get<"room_id"_>(event)) + return; + + if(!json::get<"state_key"_>(event)) + return; + + const m::room room + { + at<"room_id"_>(event) //TODO: ABA ABA ABA ABA + }; + + const m::room::state state + { + room + }; + + const event::idx &prev_state_idx + { + opts.allow_queries? + state.get(std::nothrow, at<"type"_>(event), at<"state_key"_>(event)): // query + 0UL + }; + + // No previous state; nothing to do. + if(!prev_state_idx) + return; + + // If the previous state's event_idx is greater than the event_idx of the + // event we're transacting this is almost surely a replay/rewrite. Bail + // out for now rather than corrupting the graph. + if(unlikely(prev_state_idx >= opts.event_idx)) + return; + + thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; + assert(opts.event_idx != 0 && prev_state_idx != 0); + assert(opts.event_idx != prev_state_idx); + + if(opts.event_refs.test(uint(ref::NEXT_STATE))) + { + const string_view &key + { + event_refs_key(buf, prev_state_idx, ref::NEXT_STATE, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; + } + + if(opts.event_refs.test(uint(ref::PREV_STATE))) + { + const string_view &key + { + event_refs_key(buf, opts.event_idx, ref::PREV_STATE, prev_state_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; + } +} + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_refs_m_receipt_m_read(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + assert(opts.event_refs.test(uint(ref::M_RECEIPT__M_READ))); + + if(json::get<"type"_>(event) != "ircd.read") + return; + + if(!my_host(json::get<"origin"_>(event))) + return; + + //TODO: disallow local forge? + + const json::string &event_id + { + json::get<"content"_>(event).get("event_id") + }; + + const event::idx &event_idx + { + find_event_idx(event_id, opts) + }; + + if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) + { + _index_event_horizon(txn, event, opts, event_id); + return; + } + else if(!event_idx) + { + log::dwarning + { + log, "No index found to ref %s M_RECEIPT__M_READ of %s", + string_view{event_id}, + string_view{event.event_id} + }; + + return; + } + + thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; + assert(opts.event_idx != 0 && event_idx != 0); + assert(opts.event_idx != event_idx); + const string_view &key + { + event_refs_key(buf, event_idx, ref::M_RECEIPT__M_READ, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; +} + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_refs_m_relates(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + assert(opts.event_refs.test(uint(ref::M_RELATES))); + + if(!json::get<"content"_>(event).has("m.relates_to")) + return; + + if(json::type(json::get<"content"_>(event).get("m.relates_to")) != json::OBJECT) + return; + + const json::object &m_relates_to + { + json::get<"content"_>(event).get("m.relates_to") + }; + + const json::string &event_id + { + m_relates_to.get("event_id") + }; + + if(!event_id) + return; + + if(!valid(m::id::EVENT, event_id)) + { + log::derror + { + log, "Cannot index m.relates_to in %s; '%s' is not an event_id.", + string_view{event.event_id}, + string_view{event_id} + }; + + return; + } + + const event::idx &event_idx + { + find_event_idx(event_id, opts) + }; + + if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) + { + // If we don't have the event being related to yet, place a marker in + // the event_horizon indicating need for re-evaluation later. + _index_event_horizon(txn, event, opts, event_id); + return; + } + else if(!event_idx) + { + log::derror + { + log, "Cannot index m.relates_to in %s; referenced %s not found.", + string_view{event.event_id}, + string_view{event_id} + }; + + return; + } + + thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; + assert(opts.event_idx != 0 && event_idx != 0); + assert(opts.event_idx != event_idx); + const string_view &key + { + event_refs_key(buf, event_idx, ref::M_RELATES, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; +} + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_refs_m_relates_m_reply(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + assert(opts.event_refs.test(uint(ref::M_RELATES))); + + if(json::get<"type"_>(event) != "m.room.message") + return; + + if(!json::get<"content"_>(event).has("m.relates_to")) + return; + + if(json::type(json::get<"content"_>(event).get("m.relates_to")) != json::OBJECT) + return; + + const json::object &m_relates_to + { + json::get<"content"_>(event).get("m.relates_to") + }; + + if(!m_relates_to.has("m.in_reply_to")) + return; + + if(json::type(m_relates_to.get("m.in_reply_to")) != json::OBJECT) + { + log::derror + { + log, "Cannot index m.in_reply_to in %s; not an OBJECT.", + string_view{event.event_id} + }; + + return; + } + + const json::object &m_in_reply_to + { + m_relates_to.get("m.in_reply_to") + }; + + const json::string &event_id + { + m_in_reply_to.get("event_id") + }; + + if(!valid(m::id::EVENT, event_id)) + { + log::derror + { + log, "Cannot index m.in_reply_to in %s; '%s' is not an event_id.", + string_view{event.event_id}, + string_view{event_id} + }; + + return; + } + + const event::idx &event_idx + { + find_event_idx(event_id, opts) + }; + + if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) + { + _index_event_horizon(txn, event, opts, event_id); + return; + } + else if(!event_idx) + { + log::dwarning + { + log, "Cannot index m.in_reply_to in %s; referenced %s not found.", + string_view{event.event_id}, + string_view{event_id} + }; + + return; + } + + thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; + assert(opts.event_idx != 0 && event_idx != 0); + assert(opts.event_idx != event_idx); + const string_view &key + { + event_refs_key(buf, event_idx, ref::M_RELATES, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; +} + +// NOTE: QUERY +void +ircd::m::dbs::_index_event_refs_m_room_redaction(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_REFS)); + assert(opts.event_refs.test(uint(ref::M_ROOM_REDACTION))); + + if(json::get<"type"_>(event) != "m.room.redaction") + return; + + if(!valid(m::id::EVENT, json::get<"redacts"_>(event))) + return; + + const auto &event_id + { + json::get<"redacts"_>(event) + }; + + const event::idx &event_idx + { + find_event_idx(event_id, opts) + }; + + if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx) + { + _index_event_horizon(txn, event, opts, event_id); + return; + } + else if(!event_idx) + { + log::dwarning + { + log, "Cannot index m.room.redaction in %s; referenced %s not found.", + string_view{event.event_id}, + string_view{event_id} + }; + + return; + } + + thread_local char buf[EVENT_REFS_KEY_MAX_SIZE]; + assert(opts.event_idx != 0 && event_idx != 0); + assert(opts.event_idx != event_idx); + const string_view &key + { + event_refs_key(buf, event_idx, ref::M_ROOM_REDACTION, opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_refs, + { + opts.op, key + } + }; +} + +// +// cmp +// + +bool +ircd::m::dbs::event_refs__cmp_less(const string_view &a, + const string_view &b) +{ + static const size_t half(sizeof(event::idx)); + static const size_t full(half * 2); + + assert(size(a) >= half); + assert(size(b) >= half); + const event::idx *const key[2] + { + reinterpret_cast(data(a)), + reinterpret_cast(data(b)), + }; + + return + key[0][0] < key[1][0]? true: + key[0][0] > key[1][0]? false: + size(a) < size(b)? true: + size(a) > size(b)? false: + size(a) == half? false: + key[0][1] < key[1][1]? true: + false; +} + +// +// key +// + +std::tuple +ircd::m::dbs::event_refs_key(const string_view &amalgam) +{ + const event::idx key + { + byte_view{amalgam} + }; + + return + { + ref(key >> ref_shift), key & ~ref_mask + }; +} + +ircd::string_view +ircd::m::dbs::event_refs_key(const mutable_buffer &out, + const event::idx &tgt, + const ref &type, + const event::idx &src) +{ + assert((src & ref_mask) == 0); + assert(size(out) >= sizeof(event::idx) * 2); + event::idx *const &key + { + reinterpret_cast(data(out)) + }; + + key[0] = tgt; + key[1] = src; + key[1] |= uint64_t(type) << ref_shift; + return string_view + { + data(out), data(out) + sizeof(event::idx) * 2 + }; +} + +// +// util +// + +ircd::string_view +ircd::m::dbs::reflect(const ref &type) +{ + switch(type) + { + case ref::NEXT: return "NEXT"; + case ref::NEXT_AUTH: return "NEXT_AUTH"; + case ref::NEXT_STATE: return "NEXT_STATE"; + case ref::PREV_STATE: return "PREV_STATE"; + case ref::M_RECEIPT__M_READ: return "M_RECEIPT__M_READ"; + case ref::M_RELATES: return "M_RELATES"; + case ref::M_ROOM_REDACTION: return "M_ROOM_REDACTION"; + } + + return "????"; +} diff --git a/matrix/dbs_event_sender.cc b/matrix/dbs_event_sender.cc new file mode 100644 index 000000000..3c13132c5 --- /dev/null +++ b/matrix/dbs_event_sender.cc @@ -0,0 +1,297 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::event_sender) +ircd::m::dbs::event_sender; + +decltype(ircd::m::dbs::desc::event_sender__block__size) +ircd::m::dbs::desc::event_sender__block__size +{ + { "name", "ircd.m.dbs._event_sender.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_sender__meta_block__size) +ircd::m::dbs::desc::event_sender__meta_block__size +{ + { "name", "ircd.m.dbs._event_sender.meta_block.size" }, + { "default", 4096L }, +}; + +decltype(ircd::m::dbs::desc::event_sender__cache__size) +ircd::m::dbs::desc::event_sender__cache__size +{ + { + { "name", "ircd.m.dbs._event_sender.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{event_sender__cache__size}; + db::capacity(db::cache(dbs::event_sender), value); + } +}; + +decltype(ircd::m::dbs::desc::event_sender__cache_comp__size) +ircd::m::dbs::desc::event_sender__cache_comp__size +{ + { + { "name", "ircd.m.dbs._event_sender.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{event_sender__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::event_sender), value); + } +}; + +const ircd::db::prefix_transform +ircd::m::dbs::desc::event_sender__pfx +{ + "_event_sender", + [](const string_view &key) + { + return startswith(key, '@')? + has(key, '\0'): + has(key, '@'); + }, + + [](const string_view &key) + { + const auto &[prefix, suffix] + { + // Split @localpart:hostpart\0event_idx by '\0' + startswith(key, '@')? + split(key, '\0'): + + // Split hostpart@localpart\0event_idx by '@' + split(key, '@') + }; + + return prefix; + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::event_sender +{ + // name + "_event_sender", + + // explanation + R"(Index of senders to their events. + + mxid | event_idx => -- + origin | localpart, event_idx => -- + + The senders of events are indexed by this column. This allows for all + events from a sender to be iterated. Additionally, all events from a + server and all known servers can be iterated from this column. + + key #1: + The first type of key is made from a user mxid and an event_idx concat. + + key #2: + The second type of key is made from a user mxid and an event_id, where + the mxid is part-swapped so the origin comes first, and the @localpart + comes after. + + Note that the indexers of this column ignores the actual "origin" field + of an event. Only the "sender" data is used here. + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + event_sender__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, + + // expect queries hit + false, + + // block size + size_t(event_sender__block__size), + + // meta_block size + size_t(event_sender__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_event_sender(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_SENDER)); + assert(json::get<"sender"_>(event)); + assert(opts.event_idx); + + thread_local char buf[2][EVENT_SENDER_KEY_MAX_SIZE]; + const string_view &sender_key + { + event_sender_key(buf[0], at<"sender"_>(event), opts.event_idx) + }; + + const string_view &sender_origin_key + { + event_sender_origin_key(buf[1], at<"sender"_>(event), opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_sender, + { + opts.op, sender_key + } + }; + + db::txn::append + { + txn, dbs::event_sender, + { + opts.op, sender_origin_key + } + }; +} + +// +// key +// + +// sender_key + +std::tuple +ircd::m::dbs::event_sender_key(const string_view &amalgam) +{ + const auto &parts + { + split(amalgam, '\0') + }; + + assert(empty(parts.first)); + return + { + byte_view(parts.second), + }; +} + +ircd::string_view +ircd::m::dbs::event_sender_key(const mutable_buffer &out_, + const user::id &user_id, + const event::idx &event_idx) +{ + assert(size(out_) >= EVENT_SENDER_KEY_MAX_SIZE); + assert(!event_idx || user_id); + + mutable_buffer out{out_}; + consume(out, copy(out, user_id)); + + if(user_id && event_idx) + { + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, byte_view(event_idx))); + } + + return { data(out_), data(out) }; +} + +bool +ircd::m::dbs::is_event_sender_key(const string_view &key) +{ + return empty(key) || startswith(key, '@'); +} + +// sender_origin_key + +std::tuple +ircd::m::dbs::event_sender_origin_key(const string_view &amalgam) +{ + const auto &parts + { + split(amalgam, '\0') + }; + + assert(!empty(parts.first) && !empty(parts.second)); + assert(startswith(parts.first, '@')); + + return + { + parts.first, + byte_view(parts.second), + }; +} + +ircd::string_view +ircd::m::dbs::event_sender_origin_key(const mutable_buffer &out, + const user::id &user_id, + const event::idx &event_idx) +{ + return event_sender_origin_key(out, user_id.host(), user_id.local(), event_idx); +} + +ircd::string_view +ircd::m::dbs::event_sender_origin_key(const mutable_buffer &out_, + const string_view &origin, + const string_view &localpart, + const event::idx &event_idx) +{ + assert(size(out_) >= EVENT_SENDER_KEY_MAX_SIZE); + assert(!event_idx || localpart); + assert(!localpart || startswith(localpart, '@')); + + mutable_buffer out{out_}; + consume(out, copy(out, origin)); + consume(out, copy(out, localpart)); + + if(localpart && event_idx) + { + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, byte_view(event_idx))); + } + + return { data(out_), data(out) }; +} + +bool +ircd::m::dbs::is_event_sender_origin_key(const string_view &key) +{ + return !startswith(key, '@'); +} diff --git a/matrix/dbs_event_state.cc b/matrix/dbs_event_state.cc new file mode 100644 index 000000000..b7661aaf5 --- /dev/null +++ b/matrix/dbs_event_state.cc @@ -0,0 +1,278 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +namespace ircd::m::dbs +{ + static bool event_state__cmp_lt(const string_view &, const string_view &); +} + +decltype(ircd::m::dbs::event_state) +ircd::m::dbs::event_state; + +decltype(ircd::m::dbs::desc::event_state__block__size) +ircd::m::dbs::desc::event_state__block__size +{ + { "name", "ircd.m.dbs._event_state.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_state__meta_block__size) +ircd::m::dbs::desc::event_state__meta_block__size +{ + { "name", "ircd.m.dbs._event_state.meta_block.size" }, + { "default", long(2_KiB) }, +}; + +decltype(ircd::m::dbs::desc::event_state__cache__size) +ircd::m::dbs::desc::event_state__cache__size +{ + { + { "name", "ircd.m.dbs._event_state.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{event_state__cache__size}; + db::capacity(db::cache(dbs::event_state), value); + } +}; + +decltype(ircd::m::dbs::desc::event_state__cache_comp__size) +ircd::m::dbs::desc::event_state__cache_comp__size +{ + { + { "name", "ircd.m.dbs._event_state.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{event_state__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::event_state), value); + } +}; + +const ircd::db::comparator +ircd::m::dbs::desc::event_state__cmp +{ + "_event_state", + event_state__cmp_lt, + std::equal_to{}, +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::event_state +{ + // name + "_event_state", + + // explanation + R"(Index of states of events. + + state_key, type, room_id, depth, event_idx => -- + + The state transitions of events are indexed by this column, + based on the state_key property. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + event_state__cmp, + + // prefix transform + {}, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, + + // expect queries hit + false, + + // block size + size_t(event_state__block__size), + + // meta_block size + size_t(event_state__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_event_state(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_STATE)); + assert(json::get<"type"_>(event)); + assert(opts.event_idx); + + if(!defined(json::get<"state_key"_>(event))) + return; + + thread_local char buf[EVENT_STATE_KEY_MAX_SIZE]; + db::txn::append + { + txn, dbs::event_state, + { + opts.op, event_state_key(buf, event_state_tuple + { + at<"state_key"_>(event), + at<"type"_>(event), + at<"room_id"_>(event), + at<"depth"_>(event), + opts.event_idx, + }) + } + }; +} + +// +// cmp +// + +bool +ircd::m::dbs::event_state__cmp_lt(const string_view &a, + const string_view &b) +{ + const event_state_tuple key[2] + { + event_state_key(a), + event_state_key(b), + }; + + const auto &[state_key_a, type_a, room_id_a, depth_a, event_idx_a] + { + key[0] + }; + + const auto &[state_key_b, type_b, room_id_b, depth_b, event_idx_b] + { + key[1] + }; + + if(state_key_a != state_key_b) + return state_key_a < state_key_b; + + if(type_a != type_b) + return type_a < type_b; + + if(room_id_a != room_id_b) + return room_id_a < room_id_b; + + if(depth_a != depth_b) + return depth_a > depth_b; + + if(event_idx_a != event_idx_b) + return event_idx_a > event_idx_b; + + return false; +} + +// +// key +// + +ircd::m::dbs::event_state_tuple +ircd::m::dbs::event_state_key(const string_view &amalgam) +{ + assert(!startswith(amalgam, '\0')); + const auto &[state_key, r0] + { + split(amalgam, "\0"_sv) + }; + + const auto &[type, r1] + { + split(r0, "\0"_sv) + }; + + const auto &[room_id, r2] + { + split(r1, "\0"_sv) + }; + + assert(!room_id || m::valid(m::id::ROOM, room_id)); + return event_state_tuple + { + state_key, + type, + room_id, + r2.size() >= 8? + int64_t(byte_view(r2.substr(0, 8))): + -1L, + r2.size() >= 16? + event::idx(byte_view(r2.substr(8))): + 0UL, + }; +} + +ircd::string_view +ircd::m::dbs::event_state_key(const mutable_buffer &out_, + const event_state_tuple &tuple) +{ + assert(size(out_) >= EVENT_STATE_KEY_MAX_SIZE); + + const auto &[state_key, type, room_id, depth, event_idx] + { + tuple + }; + + if(!state_key) + return {}; + + mutable_buffer out{out_}; + consume(out, copy(out, state_key)); + if(!type) + return {data(out_), data(out)}; + + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, type)); + if(!room_id) + return {data(out_), data(out)}; + + assert(m::valid(m::id::ROOM, room_id)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, room_id)); + if(depth < 0) + return {data(out_), data(out)}; + + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, byte_view(depth))); + if(!event_idx) + return {data(out_), data(out)}; + + consume(out, copy(out, byte_view(event_idx))); + return {data(out_), data(out)}; +} diff --git a/matrix/dbs_event_type.cc b/matrix/dbs_event_type.cc new file mode 100644 index 000000000..16836d152 --- /dev/null +++ b/matrix/dbs_event_type.cc @@ -0,0 +1,191 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::event_type) +ircd::m::dbs::event_type; + +decltype(ircd::m::dbs::desc::event_type__block__size) +ircd::m::dbs::desc::event_type__block__size +{ + { "name", "ircd.m.dbs._event_type.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::event_type__meta_block__size) +ircd::m::dbs::desc::event_type__meta_block__size +{ + { "name", "ircd.m.dbs._event_type.meta_block.size" }, + { "default", long(4_KiB) }, +}; + +decltype(ircd::m::dbs::desc::event_type__cache__size) +ircd::m::dbs::desc::event_type__cache__size +{ + { + { "name", "ircd.m.dbs._event_type.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{event_type__cache__size}; + db::capacity(db::cache(dbs::event_type), value); + } +}; + +decltype(ircd::m::dbs::desc::event_type__cache_comp__size) +ircd::m::dbs::desc::event_type__cache_comp__size +{ + { + { "name", "ircd.m.dbs._event_type.cache_comp.size" }, + { "default", long(0_MiB) }, + }, [] + { + const size_t &value{event_type__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::event_type), value); + } +}; + +const ircd::db::prefix_transform +ircd::m::dbs::desc::event_type__pfx +{ + "_event_type", + + [](const string_view &key) + { + return has(key, '\0'); + }, + + [](const string_view &key) + { + return split(key, '\0').first; + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::event_type +{ + // name + "_event_type", + + // explanation + R"(Index of types of events. + + type | event_idx => -- + + The types of events are indexed by this column. All events of a specific type can be + iterated efficiently. The type string forms the prefix domain. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + event_type__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, //uses conf item + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, + + // expect queries hit + false, + + // block size + size_t(event_type__block__size), + + // meta_block size + size_t(event_type__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_event_type(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::EVENT_TYPE)); + assert(json::get<"type"_>(event)); + assert(opts.event_idx); + + thread_local char buf[EVENT_TYPE_KEY_MAX_SIZE]; + const string_view &key + { + event_type_key(buf, at<"type"_>(event), opts.event_idx) + }; + + db::txn::append + { + txn, dbs::event_type, + { + opts.op, key + } + }; +} + +// +// key +// + +std::tuple +ircd::m::dbs::event_type_key(const string_view &amalgam) +{ + assert(size(amalgam) == sizeof(event::idx) + 1); + const auto &key + { + amalgam.substr(1) + }; + + assert(size(key) == sizeof(event::idx)); + return + { + byte_view(key) + }; +} + +ircd::string_view +ircd::m::dbs::event_type_key(const mutable_buffer &out_, + const string_view &type, + const event::idx &event_idx) +{ + assert(size(out_) >= EVENT_TYPE_KEY_MAX_SIZE); + + mutable_buffer out{out_}; + consume(out, copy(out, type)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, byte_view(event_idx))); + return { data(out_), data(out) }; +} diff --git a/matrix/dbs_room_events.cc b/matrix/dbs_room_events.cc new file mode 100644 index 000000000..fbba15a02 --- /dev/null +++ b/matrix/dbs_room_events.cc @@ -0,0 +1,312 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +namespace ircd::m::dbs +{ + static bool room_events__cmp_lt(const string_view &, const string_view &); +} + +/// Linkage for a reference to the room_events column +decltype(ircd::m::dbs::room_events) +ircd::m::dbs::room_events; + +decltype(ircd::m::dbs::desc::room_events__block__size) +ircd::m::dbs::desc::room_events__block__size +{ + { "name", "ircd.m.dbs._room_events.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::room_events__meta_block__size) +ircd::m::dbs::desc::room_events__meta_block__size +{ + { "name", "ircd.m.dbs._room_events.meta_block.size" }, + { "default", long(16_KiB) }, +}; + +decltype(ircd::m::dbs::desc::room_events__cache__size) +ircd::m::dbs::desc::room_events__cache__size +{ + { + { "name", "ircd.m.dbs._room_events.cache.size" }, + { "default", long(32_MiB) }, + }, [] + { + const size_t &value{room_events__cache__size}; + db::capacity(db::cache(dbs::room_events), value); + } +}; + +decltype(ircd::m::dbs::desc::room_events__cache_comp__size) +ircd::m::dbs::desc::room_events__cache_comp__size +{ + { + { "name", "ircd.m.dbs._room_events.cache_comp.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{room_events__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::room_events), value); + } +}; + +/// Prefix transform for the room_events. The prefix here is a room_id +/// and the suffix is the depth+event_id concatenation. +/// for efficient sequences +/// +const ircd::db::prefix_transform +ircd::m::dbs::desc::room_events__pfx +{ + "_room_events", + + [](const string_view &key) + { + return has(key, "\0"_sv); + }, + + [](const string_view &key) + { + return split(key, "\0"_sv).first; + } +}; + +/// Comparator for the room_events. The goal here is to sort the +/// events within a room by their depth from highest to lowest, so the +/// highest depth is hit first when a room is sought from this column. +/// +const ircd::db::comparator +ircd::m::dbs::desc::room_events__cmp +{ + "_room_events", + room_events__cmp_lt, + std::equal_to{}, +}; + +/// This column stores events in sequence in a room. Consider the following: +/// +/// [room_id | depth + event_idx] +/// +/// The key is composed from three parts: +/// +/// - `room_id` is the official prefix, bounding the sequence. That means we +/// make a blind query with just a room_id and get to the beginning of the +/// sequence, then iterate until we stop before the next room_id (upper bound). +/// +/// - `depth` is the ordering. Within the sequence, all elements are ordered by +/// depth from HIGHEST TO LOWEST. The sequence will start at the highest depth. +/// NOTE: Depth is a fixed 8 byte binary integer. +/// +/// - `event_idx` is the key suffix. This column serves to sequence all events +/// within a room ordered by depth. There may be duplicate room_id|depth +/// prefixing but the event_idx suffix gives the key total uniqueness. +/// NOTE: event_idx is a fixed 8 byte binary integer. +/// +const ircd::db::descriptor +ircd::m::dbs::desc::room_events +{ + // name + "_room_events", + + // explanation + R"(Indexes events in timeline sequence for a room + + [room_id | depth + event_idx] + + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + room_events__cmp, + + // prefix transform + room_events__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, // no bloom filter because of possible comparator issues + + // expect queries hit + true, + + // block size + size_t(room_events__block__size), + + // meta_block size + size_t(room_events__meta_block__size), +}; + +// +// indexer +// + +/// Adds the entry for the room_events column into the txn. +void +ircd::m::dbs::_index_room_events(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::ROOM_EVENTS)); + + thread_local char buf[ROOM_EVENTS_KEY_MAX_SIZE]; + const ctx::critical_assertion ca; + const string_view &key + { + room_events_key(buf, at<"room_id"_>(event), at<"depth"_>(event), opts.event_idx) + }; + + db::txn::append + { + txn, room_events, + { + opts.op, // db::op + key, // key, + } + }; +} + +// +// cmp +// + +bool +ircd::m::dbs::room_events__cmp_lt(const string_view &a, + const string_view &b) +{ + static const auto &pt + { + desc::room_events__pfx + }; + + // Extract the prefix from the keys + const string_view pre[2] + { + pt.get(a), + pt.get(b), + }; + + if(size(pre[0]) != size(pre[1])) + return size(pre[0]) < size(pre[1]); + + if(pre[0] != pre[1]) + return pre[0] < pre[1]; + + // After the prefix is the depth + event_idx + const string_view post[2] + { + a.substr(size(pre[0])), + b.substr(size(pre[1])), + }; + + // These conditions are matched on some queries when the user only + // supplies a room id. + + if(empty(post[0])) + return true; + + if(empty(post[1])) + return false; + + // Distill out the depth and event_idx integers + const std::tuple pair[2] + { + room_events_key(post[0]), + room_events_key(post[1]) + }; + + // When two events are at the same depth sort by index (the sequence + // number given as they were admitted into the system) otherwise + // sort by depth. Note this is a reverse order comparison. + return std::get<0>(pair[1]) != std::get<0>(pair[0])? + std::get<0>(pair[1]) < std::get<0>(pair[0]): + std::get<1>(pair[1]) < std::get<1>(pair[0]); +} + +// +// key +// + +std::tuple +ircd::m::dbs::room_events_key(const string_view &amalgam) +{ + assert(size(amalgam) >= 1 + 8 + 8 || size(amalgam) == 1 + 8); + assert(amalgam.front() == '\0'); + + const uint64_t &depth + { + *reinterpret_cast(data(amalgam) + 1) + }; + + const event::idx &event_idx + { + size(amalgam) >= 1 + 8 + 8? + *reinterpret_cast(data(amalgam) + 1 + 8): + std::numeric_limits::max() + }; + + // Make sure these are copied rather than ever returning references in + // a tuple because the chance the integers will be aligned is low. + return { depth, event_idx }; +} + +ircd::string_view +ircd::m::dbs::room_events_key(const mutable_buffer &out_, + const id::room &room_id, + const uint64_t &depth) +{ + const const_buffer depth_cb + { + reinterpret_cast(&depth), sizeof(depth) + }; + + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, depth_cb)); + return { data(out_), data(out) }; +} + +ircd::string_view +ircd::m::dbs::room_events_key(const mutable_buffer &out_, + const id::room &room_id, + const uint64_t &depth, + const event::idx &event_idx) +{ + const const_buffer depth_cb + { + reinterpret_cast(&depth), sizeof(depth) + }; + + const const_buffer event_idx_cb + { + reinterpret_cast(&event_idx), sizeof(event_idx) + }; + + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, depth_cb)); + consume(out, copy(out, event_idx_cb)); + return { data(out_), data(out) }; +} diff --git a/matrix/dbs_room_head.cc b/matrix/dbs_room_head.cc new file mode 100644 index 000000000..07503bb12 --- /dev/null +++ b/matrix/dbs_room_head.cc @@ -0,0 +1,229 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::room_head) +ircd::m::dbs::room_head; + +decltype(ircd::m::dbs::desc::room_head__block__size) +ircd::m::dbs::desc::room_head__block__size +{ + { "name", "ircd.m.dbs._room_head.block.size" }, + { "default", long(4_KiB) }, +}; + +decltype(ircd::m::dbs::desc::room_head__meta_block__size) +ircd::m::dbs::desc::room_head__meta_block__size +{ + { "name", "ircd.m.dbs._room_head.meta_block.size" }, + { "default", long(4_KiB) }, +}; + +decltype(ircd::m::dbs::desc::room_head__cache__size) +ircd::m::dbs::desc::room_head__cache__size +{ + { + { "name", "ircd.m.dbs._room_head.cache.size" }, + { "default", long(8_MiB) }, + }, [] + { + const size_t &value{room_head__cache__size}; + db::capacity(db::cache(dbs::room_head), value); + } +}; + +/// prefix transform for room_id,event_id in room_id +/// +const ircd::db::prefix_transform +ircd::m::dbs::desc::room_head__pfx +{ + "_room_head", + + [](const string_view &key) + { + return has(key, "\0"_sv); + }, + + [](const string_view &key) + { + return split(key, "\0"_sv).first; + } +}; + +/// This column stores unreferenced (head) events for a room. +/// +const ircd::db::descriptor +ircd::m::dbs::desc::room_head +{ + // name + "_room_head", + + // explanation + R"(Unreferenced events in a room. + + [room_id | event_id => event_idx] + + The key is a room_id and event_id concatenation. The value is an event_idx + of the event_id in the key. The key amalgam was specifically selected to + allow for DELETES sent to the WAL "in the blind" for all prev_events when + any new event is saved to the database, without making any read IO's to + look up anything about the prev reference to remove. + + This is a fast-moving column where unreferenced events are inserted and + then deleted the first time another event is seen which references it so + it collects a lot of DELETE commands in the WAL and has to be compacted + often to reduce them out. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(uint64_t) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + room_head__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + 0, //no compresed cache + + // bloom filter bits + 0, //table too ephemeral for bloom generation/usefulness + + // expect queries hit + false, + + // block size + size_t(room_head__block__size), + + // meta_block size + size_t(room_head__meta_block__size), + + // compression + {}, // no compression for this column + + // compactor + {}, + + // compaction priority algorithm + "kByCompensatedSize"s, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_room_head(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::ROOM_HEAD)); + assert(opts.event_idx); + assert(event.event_id); + + const ctx::critical_assertion ca; + thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE]; + const string_view &key + { + room_head_key(buf, at<"room_id"_>(event), event.event_id) + }; + + db::txn::append + { + txn, room_head, + { + opts.op, + key, + byte_view{opts.event_idx} + } + }; +} + +void +ircd::m::dbs::_index_room_head_resolve(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::ROOM_HEAD_RESOLVE)); + + //TODO: If op is DELETE and we are deleting this event and thereby + //TODO: potentially creating a gap in the reference graph (just for us + //TODO: though) can we *re-add* the prev_events to the head? + + if(opts.op != db::op::SET) + return; + + const event::prev prev{event}; + for(size_t i(0); i < prev.prev_events_count(); ++i) + { + const auto &event_id + { + prev.prev_event(i) + }; + + thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE]; + const ctx::critical_assertion ca; + const string_view &key + { + room_head_key(buf, at<"room_id"_>(event), event_id) + }; + + db::txn::append + { + txn, room_head, + { + db::op::DELETE, + key, + } + }; + } +} + +// +// key +// + +ircd::string_view +ircd::m::dbs::room_head_key(const string_view &amalgam) +{ + const auto &key + { + lstrip(amalgam, "\0"_sv) + }; + + return + { + key + }; +} + +ircd::string_view +ircd::m::dbs::room_head_key(const mutable_buffer &out_, + const id::room &room_id, + const id::event &event_id) +{ + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, event_id)); + return { data(out_), data(out) }; +} diff --git a/matrix/dbs_room_joined.cc b/matrix/dbs_room_joined.cc new file mode 100644 index 000000000..e3389744b --- /dev/null +++ b/matrix/dbs_room_joined.cc @@ -0,0 +1,244 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::room_joined) +ircd::m::dbs::room_joined; + +decltype(ircd::m::dbs::desc::room_joined__block__size) +ircd::m::dbs::desc::room_joined__block__size +{ + { "name", "ircd.m.dbs._room_joined.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::room_joined__meta_block__size) +ircd::m::dbs::desc::room_joined__meta_block__size +{ + { "name", "ircd.m.dbs._room_joined.meta_block.size" }, + { "default", long(8_KiB) }, +}; + +decltype(ircd::m::dbs::desc::room_joined__cache__size) +ircd::m::dbs::desc::room_joined__cache__size +{ + { + { "name", "ircd.m.dbs._room_joined.cache.size" }, + { "default", long(8_MiB) }, + }, [] + { + const size_t &value{room_joined__cache__size}; + db::capacity(db::cache(dbs::room_joined), value); + } +}; + +decltype(ircd::m::dbs::desc::room_joined__cache_comp__size) +ircd::m::dbs::desc::room_joined__cache_comp__size +{ + { + { "name", "ircd.m.dbs._room_joined.cache_comp.size" }, + { "default", long(8_MiB) }, + }, [] + { + const size_t &value{room_joined__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::room_joined), value); + } +}; + +decltype(ircd::m::dbs::desc::room_joined__bloom__bits) +ircd::m::dbs::desc::room_joined__bloom__bits +{ + { "name", "ircd.m.dbs._room_joined.bloom.bits" }, + { "default", 6L }, +}; + +/// Prefix transform for the room_joined +/// +const ircd::db::prefix_transform +ircd::m::dbs::desc::room_joined__pfx +{ + "_room_joined", + + [](const string_view &key) + { + return has(key, "\0"_sv); + }, + + [](const string_view &key) + { + return split(key, "\0"_sv).first; + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::room_joined +{ + // name + "_room_joined", + + // explanation + R"(Specifically indexes joined members of a room for fast iteration. + + [room_id | origin + mxid] => event_idx + + )", + + // typing (key, value) + { + typeid(string_view), typeid(uint64_t) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + room_joined__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(room_joined__bloom__bits), + + // expect queries hit + false, + + // block size + size_t(room_joined__block__size), + + // meta_block size + size_t(room_joined__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +/// Adds the entry for the room_joined column into the txn. +void +ircd::m::dbs::_index_room_joined(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::ROOM_JOINED)); + assert(at<"type"_>(event) == "m.room.member"); + + thread_local char buf[ROOM_JOINED_KEY_MAX_SIZE]; + const ctx::critical_assertion ca; + const string_view &key + { + room_joined_key(buf, at<"room_id"_>(event), at<"origin"_>(event), at<"state_key"_>(event)) + }; + + const string_view &membership + { + m::membership(event) + }; + + assert(!empty(membership)); + + db::op op; + if(opts.op == db::op::SET) switch(hash(membership)) + { + case hash("join"): + op = db::op::SET; + break; + + case hash("ban"): + case hash("leave"): + op = db::op::DELETE; + break; + + default: + return; + } + else if(opts.op == db::op::DELETE) + op = opts.op; + else + return; + + db::txn::append + { + txn, room_joined, + { + op, + key, + } + }; +} + +// +// key +// + +std::tuple +ircd::m::dbs::room_joined_key(const string_view &amalgam) +{ + const auto &key + { + lstrip(amalgam, "\0"_sv) + }; + + const auto &s + { + split(key, "@"_sv) + }; + + return + { + { s.first }, + !empty(s.second)? + string_view{begin(s.second) - 1, end(s.second)}: + string_view{} + }; +} + +ircd::string_view +ircd::m::dbs::room_joined_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &origin) +{ + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, origin)); + return { data(out_), data(out) }; +} + +ircd::string_view +ircd::m::dbs::room_joined_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &origin, + const id::user &member) +{ + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, origin)); + consume(out, copy(out, member)); + return { data(out_), data(out) }; +} diff --git a/matrix/dbs_room_state.cc b/matrix/dbs_room_state.cc new file mode 100644 index 000000000..99f9d5160 --- /dev/null +++ b/matrix/dbs_room_state.cc @@ -0,0 +1,234 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::dbs::room_state) +ircd::m::dbs::room_state; + +decltype(ircd::m::dbs::desc::room_state__block__size) +ircd::m::dbs::desc::room_state__block__size +{ + { "name", "ircd.m.dbs._room_state.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::room_state__meta_block__size) +ircd::m::dbs::desc::room_state__meta_block__size +{ + { "name", "ircd.m.dbs._room_state.meta_block.size" }, + { "default", 8192L }, +}; + +decltype(ircd::m::dbs::desc::room_state__cache__size) +ircd::m::dbs::desc::room_state__cache__size +{ + { + { "name", "ircd.m.dbs._room_state.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{room_state__cache__size}; + db::capacity(db::cache(dbs::room_state), value); + } +}; + +decltype(ircd::m::dbs::desc::room_state__cache_comp__size) +ircd::m::dbs::desc::room_state__cache_comp__size +{ + { + { "name", "ircd.m.dbs._room_state.cache_comp.size" }, + { "default", long(8_MiB) }, + }, [] + { + const size_t &value{room_state__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::room_state), value); + } +}; + +decltype(ircd::m::dbs::desc::room_state__bloom__bits) +ircd::m::dbs::desc::room_state__bloom__bits +{ + { "name", "ircd.m.dbs._room_state.bloom.bits" }, + { "default", 10L }, +}; + +/// prefix transform for type,state_key in room_id +/// +/// This transform is special for concatenating room_id with type and state_key +/// in that order with prefix being the room_id (this may change to room_id+ +/// type +/// +const ircd::db::prefix_transform +ircd::m::dbs::desc::room_state__pfx +{ + "_room_state", + + [](const string_view &key) + { + return has(key, "\0"_sv); + }, + + [](const string_view &key) + { + return split(key, "\0"_sv).first; + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::room_state +{ + // name + "_room_state", + + // explanation + R"(The present state of the room. + + [room_id | type + state_key] => event_idx + + This column is also known as the "present state table." It contains the + very important present state of the room for this server. The key contains + plaintext room_id, type and state_key elements for direct point-lookup as + well as iteration. The value is the index of the apropos state event. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(uint64_t) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + room_state__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(room_state__bloom__bits), + + // expect queries hit + false, + + // block size + size_t(room_state__block__size), + + // meta_block size + size_t(room_state__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_room_state(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::ROOM_STATE)); + + const ctx::critical_assertion ca; + thread_local char buf[ROOM_STATE_KEY_MAX_SIZE]; + const string_view &key + { + room_state_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"state_key"_>(event)) + }; + + const string_view val + { + byte_view(opts.event_idx) + }; + + db::txn::append + { + txn, room_state, + { + opts.op, + key, + value_required(opts.op)? val : string_view{}, + } + }; +} + +// +// key +// + +ircd::string_view +ircd::m::dbs::room_state_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &type) +{ + return room_state_key(out_, room_id, type, string_view{}); +} + +ircd::string_view +ircd::m::dbs::room_state_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &type, + const string_view &state_key) +{ + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + assert(room_id); + + if(likely(defined(type))) + { + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, type)); + } + + if(defined(state_key)) + { + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, state_key)); + } + + return { data(out_), data(out) }; +} + +std::tuple +ircd::m::dbs::room_state_key(const string_view &amalgam) +{ + const auto &key + { + lstrip(amalgam, "\0"_sv) + }; + + const auto &s + { + split(key, "\0"_sv) + }; + + return + { + s.first, s.second + }; +} diff --git a/matrix/dbs_room_state_space.cc b/matrix/dbs_room_state_space.cc new file mode 100644 index 000000000..5990611d0 --- /dev/null +++ b/matrix/dbs_room_state_space.cc @@ -0,0 +1,331 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +namespace ircd::m::dbs +{ + static bool room_state_space__cmp_lt(const string_view &, const string_view &); +} + +decltype(ircd::m::dbs::room_state_space) +ircd::m::dbs::room_state_space; + +decltype(ircd::m::dbs::desc::room_state_space__block__size) +ircd::m::dbs::desc::room_state_space__block__size +{ + { "name", "ircd.m.dbs._room_state_space.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::room_state_space__meta_block__size) +ircd::m::dbs::desc::room_state_space__meta_block__size +{ + { "name", "ircd.m.dbs._room_state_space.meta_block.size" }, + { "default", long(8_KiB) }, +}; + +decltype(ircd::m::dbs::desc::room_state_space__cache__size) +ircd::m::dbs::desc::room_state_space__cache__size +{ + { + { "name", "ircd.m.dbs._room_state_space.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{room_state_space__cache__size}; + db::capacity(db::cache(dbs::room_state_space), value); + } +}; + +decltype(ircd::m::dbs::desc::room_state_space__cache_comp__size) +ircd::m::dbs::desc::room_state_space__cache_comp__size +{ + { + { "name", "ircd.m.dbs._room_state_space.cache_comp.size" }, + { "default", long(8_MiB) }, + }, [] + { + const size_t &value{room_state_space__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::room_state_space), value); + } +}; + +decltype(ircd::m::dbs::desc::room_state_space__bloom__bits) +ircd::m::dbs::desc::room_state_space__bloom__bits +{ + { "name", "ircd.m.dbs._room_state_space.bloom.bits" }, + { "default", 10L }, +}; + +const ircd::db::comparator +ircd::m::dbs::desc::room_state_space__cmp +{ + "_room_state_space", + room_state_space__cmp_lt, + std::equal_to{}, +}; + +const ircd::db::prefix_transform +ircd::m::dbs::desc::room_state_space__pfx +{ + "_room_state_space", + + [](const string_view &key) + { + return has(key, "\0"_sv); + }, + + [](const string_view &key) + { + return split(key, "\0"_sv).first; + } +}; + +const ircd::db::descriptor +ircd::m::dbs::desc::room_state_space +{ + // name + "_room_state_space", + + // explanation + R"(All states of the room. + + )", + + // typing (key, value) + { + typeid(string_view), typeid(uint64_t) + }, + + // options + {}, + + // comparator + room_state_space__cmp, + + // prefix transform + room_state_space__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + size_t(room_state_space__bloom__bits), + + // expect queries hit + false, + + // block size + size_t(room_state_space__block__size), + + // meta_block size + size_t(room_state_space__meta_block__size), + + // compression + "kLZ4Compression;kSnappyCompression"s, + + // compactor + {}, + + // compaction priority algorithm + "kOldestSmallestSeqFirst"s, +}; + +// +// indexer +// + +void +ircd::m::dbs::_index_room_state_space(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::ROOM_STATE_SPACE)); + + const ctx::critical_assertion ca; + thread_local char buf[ROOM_STATE_SPACE_KEY_MAX_SIZE]; + const string_view &key + { + room_state_space_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"state_key"_>(event), at<"depth"_>(event), opts.event_idx) + }; + + db::txn::append + { + txn, room_state_space, + { + opts.op, + key, + } + }; +} + +// +// cmp +// + +bool +ircd::m::dbs::room_state_space__cmp_lt(const string_view &a, + const string_view &b) +{ + static const auto &pt + { + desc::room_state_space__pfx + }; + + const string_view pre[2] + { + pt.get(a), + pt.get(b), + }; + + if(size(pre[0]) != size(pre[1])) + return size(pre[0]) < size(pre[1]); + + if(pre[0] != pre[1]) + return pre[0] < pre[1]; + + const string_view post[2] + { + a.substr(size(pre[0])), + b.substr(size(pre[1])), + }; + + // These conditions are matched on some queries when the user only + // supplies a room_id. + if(empty(post[0])) + return true; + + if(empty(post[1])) + return false; + + // Decompose the postfix of the key for granular sorting + const room_state_space_key_parts k[2] + { + room_state_space_key(post[0]), + room_state_space_key(post[1]) + }; + + // type + if(std::get<0>(k[0]) < std::get<0>(k[1])) + return true; + else if(std::get<0>(k[0]) > std::get<0>(k[1])) + return false; + + // state_key + if(std::get<1>(k[0]) < std::get<1>(k[1])) + return true; + else if(std::get<1>(k[0]) > std::get<1>(k[1])) + return false; + + // depth (ORDER IS DESCENDING!) + if(uint64_t(std::get<2>(k[0])) > uint64_t(std::get<2>(k[1]))) + return true; + else if(uint64_t(std::get<2>(k[0])) < uint64_t(std::get<2>(k[1]))) + return false; + + // event_idx (ORDER IS DESCENDING!) + if(std::get<3>(k[0]) > std::get<3>(k[1])) + return true; + else if(std::get<3>(k[0]) < std::get<3>(k[1])) + return false; + + return false; +} + +// +// key +// + +ircd::m::dbs::room_state_space_key_parts +ircd::m::dbs::room_state_space_key(const string_view &amalgam) +{ + const auto &key + { + lstrip(amalgam, "\0"_sv) + }; + + const auto &[type, after_type] + { + split(key, "\0"_sv) + }; + + const auto &[state_key, after_state_key] + { + split(after_type, "\0"_sv) + }; + + const int64_t &depth + { + size(after_state_key) >= 8? + int64_t(byte_view(after_state_key.substr(0, 8))): + -1L + }; + + const event::idx &event_idx + { + size(after_state_key) >= 16? + event::idx(byte_view(after_state_key.substr(8, 8))): + 0UL + }; + + return + { + type, state_key, depth, event_idx + }; +} + +ircd::string_view +ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, + const id::room &room_id) +{ + return room_state_space_key(out_, room_id, string_view{}, string_view{}, -1L, 0L); +} + +ircd::string_view +ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &type) +{ + return room_state_space_key(out_, room_id, type, string_view{}, -1L, 0L); +} + +ircd::string_view +ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &type, + const string_view &state_key) +{ + return room_state_space_key(out_, room_id, type, state_key, -1L, 0L); +} + +ircd::string_view +ircd::m::dbs::room_state_space_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &type, + const string_view &state_key, + const int64_t &depth, + const event::idx &event_idx) +{ + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, type)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, state_key)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, byte_view(depth))); + consume(out, copy(out, byte_view(event_idx))); + return { data(out_), data(out) }; +} diff --git a/matrix/dbs_room_type.cc b/matrix/dbs_room_type.cc new file mode 100644 index 000000000..910c5e872 --- /dev/null +++ b/matrix/dbs_room_type.cc @@ -0,0 +1,314 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +namespace ircd::m::dbs +{ + static bool room_type__cmp_lt(const string_view &, const string_view &); +} + +decltype(ircd::m::dbs::room_type) +ircd::m::dbs::room_type; + +decltype(ircd::m::dbs::desc::room_type__block__size) +ircd::m::dbs::desc::room_type__block__size +{ + { "name", "ircd.m.dbs._room_type.block.size" }, + { "default", 512L }, +}; + +decltype(ircd::m::dbs::desc::room_type__meta_block__size) +ircd::m::dbs::desc::room_type__meta_block__size +{ + { "name", "ircd.m.dbs._room_type.meta_block.size" }, + { "default", 8192L }, +}; + +decltype(ircd::m::dbs::desc::room_type__cache__size) +ircd::m::dbs::desc::room_type__cache__size +{ + { + { "name", "ircd.m.dbs._room_type.cache.size" }, + { "default", long(16_MiB) }, + }, [] + { + const size_t &value{room_type__cache__size}; + db::capacity(db::cache(dbs::room_type), value); + } +}; + +decltype(ircd::m::dbs::desc::room_type__cache_comp__size) +ircd::m::dbs::desc::room_type__cache_comp__size +{ + { + { "name", "ircd.m.dbs._room_type.cache_comp.size" }, + { "default", long(8_MiB) }, + }, [] + { + const size_t &value{room_type__cache_comp__size}; + db::capacity(db::cache_compressed(dbs::room_type), value); + } +}; + +/// Prefix transform for the room_type. The prefix here is a room_id +/// and the suffix is the type+depth+event_id concatenation. +/// for efficient sequences +/// +const ircd::db::prefix_transform +ircd::m::dbs::desc::room_type__pfx +{ + "_room_type", + + [](const string_view &key) + { + return has(key, "\0"_sv); + }, + + [](const string_view &key) + { + return split(key, "\0"_sv).first; + } +}; + +/// Comparator for the room_type. The goal here is to sort the +/// events within a room by their depth from highest to lowest, so the +/// highest depth is hit first when a room is sought from this column. +/// +const ircd::db::comparator +ircd::m::dbs::desc::room_type__cmp +{ + "_room_type", + room_type__cmp_lt, + std::equal_to{}, +}; + +/// This column stores events by type in sequence in a room. Consider the +/// following: +/// +/// [room_id | type, depth, event_idx] +/// +const ircd::db::descriptor +ircd::m::dbs::desc::room_type +{ + // name + "_room_type", + + // explanation + R"(Indexes events per type in timeline sequence for a room + + [room_id | type, depth, event_idx] + + )", + + // typing (key, value) + { + typeid(string_view), typeid(string_view) + }, + + // options + {}, + + // comparator + room_type__cmp, + + // prefix transform + room_type__pfx, + + // drop column + false, + + // cache size + bool(cache_enable)? -1 : 0, + + // cache size for compressed assets + bool(cache_comp_enable)? -1 : 0, + + // bloom filter bits + 0, // no bloom filter because of possible comparator issues + + // expect queries hit + true, + + // block size + size_t(room_type__block__size), + + // meta_block size + size_t(room_type__meta_block__size), +}; + +// +// indexer +// + +/// Adds the entry for the room_type column into the txn. +void +ircd::m::dbs::_index_room_type(db::txn &txn, + const event &event, + const write_opts &opts) +{ + assert(opts.appendix.test(appendix::ROOM_TYPE)); + + thread_local char buf[ROOM_TYPE_KEY_MAX_SIZE]; + const ctx::critical_assertion ca; + const string_view &key + { + room_type_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"depth"_>(event), opts.event_idx) + }; + + db::txn::append + { + txn, room_type, + { + opts.op, // db::op + key, // key, + } + }; +} + +// +// cmp +// + +bool +ircd::m::dbs::room_type__cmp_lt(const string_view &a, + const string_view &b) +{ + static const auto &pt + { + desc::room_type__pfx + }; + + // Extract the prefix from the keys + const string_view pre[2] + { + pt.get(a), + pt.get(b), + }; + + // Prefix size comparison has highest priority for rocksdb + if(size(pre[0]) < size(pre[1])) + return true; + + // Prefix size comparison has highest priority for rocksdb + if(size(pre[0]) > size(pre[1])) + return false; + + // Prefix lexical comparison sorts prefixes of the same size + if(pre[0] < pre[1]) + return true; + + // Prefix lexical comparison sorts prefixes of the same size + if(pre[0] > pre[1]) + return false; + + // After the prefix is the \0,type,\0,depth,event_idx + const string_view post[2] + { + a.substr(size(pre[0])), + b.substr(size(pre[1])), + }; + + // These conditions are matched on some queries when the user only + // supplies a room id. + if(empty(post[0])) + return true; + + if(empty(post[1])) + return false; + + const auto &[type_a, depth_a, event_idx_a] + { + room_type_key(post[0]) + }; + + const auto &[type_b, depth_b, event_idx_b] + { + room_type_key(post[1]) + }; + + if(type_a < type_b) + return true; + + if(type_a > type_b) + return false; + + // reverse depth to start from highest first like room_events + if(depth_a < depth_b) + return false; + + // reverse depth to start from highest first like room_events + if(depth_a > depth_b) + return true; + + // reverse event_idx to start from highest first like room_events) + if(event_idx_a < event_idx_b) + return false; + + if(event_idx_a > event_idx_b) + return true; + + // equal is not less; so false + return false; +} + +// +// key +// + +ircd::m::dbs::room_type_tuple +ircd::m::dbs::room_type_key(const string_view &amalgam_) +{ + assert(size(amalgam_) >= 1 + 1 + 8 + 8); + + assert(amalgam_.front() == '\0'); + const string_view &amalgam + { + amalgam_.substr(1) + }; + + assert(amalgam.size() >= 1 + 8 + 1); + const auto &[type, trail] + { + split(amalgam, '\0') + }; + + assert(trail.size() >= 8 + 8); + return room_type_tuple + { + type, + likely(trail.size() >= 8)? + uint64_t(byte_view(trail.substr(0, 8))): + -1UL, + likely(trail.size() >= 16)? + event::idx(byte_view(trail.substr(8))): + 0UL, + }; +} + +ircd::string_view +ircd::m::dbs::room_type_key(const mutable_buffer &out_, + const id::room &room_id, + const string_view &type, + const uint64_t &depth, + const event::idx &event_idx) +{ + assert(room_id); + mutable_buffer out{out_}; + consume(out, copy(out, room_id)); + + if(!type) + return { data(out_), data(out) }; + + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, type)); + consume(out, copy(out, "\0"_sv)); + consume(out, copy(out, byte_view(depth))); + consume(out, copy(out, byte_view(event_idx))); + return { data(out_), data(out) }; +} diff --git a/matrix/events.cc b/matrix/events.cc index 8ba965a61..f82e488d2 100644 --- a/matrix/events.cc +++ b/matrix/events.cc @@ -486,7 +486,7 @@ ircd::m::events::type::for_each(const string_view &prefix, const auto &prefixer { - dbs::desc::events__event_type__pfx + dbs::desc::event_type__pfx }; string_view last; @@ -570,7 +570,7 @@ ircd::m::events::origin::for_each(const string_view &prefix, const auto &prefixer { - dbs::desc::events__event_sender__pfx + dbs::desc::event_sender__pfx }; if(unlikely(startswith(prefix, '@'))) @@ -654,7 +654,7 @@ ircd::m::events::sender::for_each(const string_view &prefix_, const auto &prefixer { - dbs::desc::events__event_sender__pfx + dbs::desc::event_sender__pfx }; // We MUST query the column with a key starting with '@' here. For a more