MirrorHub/construct
0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-11-29 02:02:38 +01:00
Code Issues Releases Wiki Activity

ircd:Ⓜ️:dbs: Split dbs unit per column; naming simplifications; major reorg.

Browse source
This commit is contained in:
Jason Volk 2020-03-25 11:06:29 -07:00
parent ddbabedf84
commit b8239d45cd
34 changed files with 5547 additions and 5388 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
doc/TUNING.md
View file

@ -74,7 +74,7 @@ output from the above command, use the following command where `<COLUMN>` is
replaced by one of the names under `COLUMN` in the above output:
```
conf ircd.m.dbs.events.<COLUMN>.cache.size
conf ircd.m.dbs.<COLUMN>.cache.size
```
To alter a cache size, set the configuration item with a byte value. In the
@ -82,7 +82,7 @@ example below we will set the `_event_json` cache size to 256 MiB. This change
will take effect immediately and the cache will grow or shrink to that size.
```
conf set ircd.m.dbs.events._event_json.cache.size 268435456
conf set ircd.m.dbs._event_json.cache.size 268435456
```
> Tip: The best metric to figure out which caches are inadequate is not

14
include/ircd/m/dbs/dbs.h
View file

@ -19,10 +19,10 @@ namespace ircd::m::dbs
enum class ref :uint8_t;
// General confs
extern conf::item<bool> events_cache_enable;
extern conf::item<bool> events_cache_comp_enable;
extern conf::item<size_t> events_mem_write_buffer_size;
extern conf::item<size_t> events_sst_write_buffer_size;
extern conf::item<bool> cache_enable;
extern conf::item<bool> cache_comp_enable;
extern conf::item<size_t> mem_write_buffer_size;
extern conf::item<size_t> sst_write_buffer_size;
// Database instance
extern std::shared_ptr<db::database> events;
@ -197,3 +197,9 @@ struct ircd::m::dbs::init
init(const string_view &servername, std::string dbopts = {});
~init() noexcept;
};
// Internal utils (here for now)
namespace ircd::m::dbs
{
event::idx find_event_idx(const event::id &, const write_opts &);
}

84
include/ircd/m/dbs/event_column.h
View file

@ -25,6 +25,8 @@ namespace ircd::m::dbs
event::size()
};
void _index_event_cols(db::txn &, const event &, const write_opts &);
// There is one position in this array corresponding to each property
// in the m::event tuple, however, the db::column in this position may
// be default-initialized if this column is not used.
@ -33,53 +35,53 @@ namespace ircd::m::dbs
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events___event__bloom__bits;
extern conf::item<size_t> _event__bloom__bits;
extern conf::item<size_t> events__content__block__size;
extern conf::item<size_t> events__content__meta_block__size;
extern conf::item<size_t> events__content__cache__size;
extern conf::item<size_t> events__content__cache_comp__size;
extern const db::descriptor events_content;
extern conf::item<size_t> content__block__size;
extern conf::item<size_t> content__meta_block__size;
extern conf::item<size_t> content__cache__size;
extern conf::item<size_t> content__cache_comp__size;
extern const db::descriptor content;
extern conf::item<size_t> events__depth__block__size;
extern conf::item<size_t> events__depth__meta_block__size;
extern conf::item<size_t> events__depth__cache__size;
extern conf::item<size_t> events__depth__cache_comp__size;
extern const db::descriptor events_depth;
extern conf::item<size_t> depth__block__size;
extern conf::item<size_t> depth__meta_block__size;
extern conf::item<size_t> depth__cache__size;
extern conf::item<size_t> depth__cache_comp__size;
extern const db::descriptor depth;
extern conf::item<size_t> events__event_id__block__size;
extern conf::item<size_t> events__event_id__meta_block__size;
extern conf::item<size_t> events__event_id__cache__size;
extern conf::item<size_t> events__event_id__cache_comp__size;
extern const db::descriptor events_event_id;
extern conf::item<size_t> event_id__block__size;
extern conf::item<size_t> event_id__meta_block__size;
extern conf::item<size_t> event_id__cache__size;
extern conf::item<size_t> event_id__cache_comp__size;
extern const db::descriptor event_id;
extern conf::item<size_t> events__origin_server_ts__block__size;
extern conf::item<size_t> events__origin_server_ts__meta_block__size;
extern conf::item<size_t> events__origin_server_ts__cache__size;
extern conf::item<size_t> events__origin_server_ts__cache_comp__size;
extern const db::descriptor events_origin_server_ts;
extern conf::item<size_t> origin_server_ts__block__size;
extern conf::item<size_t> origin_server_ts__meta_block__size;
extern conf::item<size_t> origin_server_ts__cache__size;
extern conf::item<size_t> origin_server_ts__cache_comp__size;
extern const db::descriptor origin_server_ts;
extern conf::item<size_t> events__room_id__block__size;
extern conf::item<size_t> events__room_id__meta_block__size;
extern conf::item<size_t> events__room_id__cache__size;
extern conf::item<size_t> events__room_id__cache_comp__size;
extern const db::descriptor events_room_id;
extern conf::item<size_t> room_id__block__size;
extern conf::item<size_t> room_id__meta_block__size;
extern conf::item<size_t> room_id__cache__size;
extern conf::item<size_t> room_id__cache_comp__size;
extern const db::descriptor room_id;
extern conf::item<size_t> events__sender__block__size;
extern conf::item<size_t> events__sender__meta_block__size;
extern conf::item<size_t> events__sender__cache__size;
extern conf::item<size_t> events__sender__cache_comp__size;
extern const db::descriptor events_sender;
extern conf::item<size_t> sender__block__size;
extern conf::item<size_t> sender__meta_block__size;
extern conf::item<size_t> sender__cache__size;
extern conf::item<size_t> sender__cache_comp__size;
extern const db::descriptor sender;
extern conf::item<size_t> events__state_key__block__size;
extern conf::item<size_t> events__state_key__meta_block__size;
extern conf::item<size_t> events__state_key__cache__size;
extern conf::item<size_t> events__state_key__cache_comp__size;
extern const db::descriptor events_state_key;
extern conf::item<size_t> state_key__block__size;
extern conf::item<size_t> state_key__meta_block__size;
extern conf::item<size_t> state_key__cache__size;
extern conf::item<size_t> state_key__cache_comp__size;
extern const db::descriptor state_key;
extern conf::item<size_t> events__type__block__size;
extern conf::item<size_t> events__type__meta_block__size;
extern conf::item<size_t> events__type__cache__size;
extern conf::item<size_t> events__type__cache_comp__size;
extern const db::descriptor events_type;
extern conf::item<size_t> type__block__size;
extern conf::item<size_t> type__meta_block__size;
extern conf::item<size_t> type__cache__size;
extern conf::item<size_t> type__cache_comp__size;
extern const db::descriptor type;
}

15
include/ircd/m/dbs/event_horizon.h
View file

@ -22,16 +22,19 @@ namespace ircd::m::dbs
string_view event_horizon_key(const mutable_buffer &out, const id::event &);
std::tuple<event::idx> event_horizon_key(const string_view &amalgam);
void _index_event_horizon_resolve(db::txn &, const event &, const write_opts &); //query
void _index_event_horizon(db::txn &, const event &, const write_opts &, const id::event &);
// event_id | event_idx
extern db::domain event_horizon;
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__event_horizon__block__size;
extern conf::item<size_t> events__event_horizon__meta_block__size;
extern conf::item<size_t> events__event_horizon__cache__size;
extern conf::item<size_t> events__event_horizon__cache_comp__size;
extern const db::prefix_transform events__event_horizon__pfx;
extern const db::descriptor events__event_horizon;
extern conf::item<size_t> event_horizon__block__size;
extern conf::item<size_t> event_horizon__meta_block__size;
extern conf::item<size_t> event_horizon__cache__size;
extern conf::item<size_t> event_horizon__cache_comp__size;
extern const db::prefix_transform event_horizon__pfx;
extern const db::descriptor event_horizon;
}

14
include/ircd/m/dbs/event_idx.h
View file

@ -13,15 +13,17 @@
namespace ircd::m::dbs
{
void _index_event_id(db::txn &, const event &, const write_opts &);
extern db::column event_idx; // event_id => event_idx
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__event_idx__block__size;
extern conf::item<size_t> events__event_idx__meta_block__size;
extern conf::item<size_t> events__event_idx__cache__size;
extern conf::item<size_t> events__event_idx__cache_comp__size;
extern conf::item<size_t> events__event_idx__bloom__bits;
extern const db::descriptor events__event_idx;
extern conf::item<size_t> event_idx__block__size;
extern conf::item<size_t> event_idx__meta_block__size;
extern conf::item<size_t> event_idx__cache__size;
extern conf::item<size_t> event_idx__cache_comp__size;
extern conf::item<size_t> event_idx__bloom__bits;
extern const db::descriptor event_idx;
}

14
include/ircd/m/dbs/event_json.h
View file

@ -13,16 +13,18 @@
namespace ircd::m::dbs
{
void _index_event_json(db::txn &, const event &, const write_opts &);
// event_idx => full json
extern db::column event_json;
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__event_json__block__size;
extern conf::item<size_t> events__event_json__meta_block__size;
extern conf::item<size_t> events__event_json__cache__size;
extern conf::item<size_t> events__event_json__cache_comp__size;
extern conf::item<size_t> events__event_json__bloom__bits;
extern const db::descriptor events__event_json;
extern conf::item<size_t> event_json__block__size;
extern conf::item<size_t> event_json__meta_block__size;
extern conf::item<size_t> event_json__cache__size;
extern conf::item<size_t> event_json__cache_comp__size;
extern conf::item<size_t> event_json__bloom__bits;
extern const db::descriptor event_json;
}

22
include/ircd/m/dbs/event_refs.h
View file

@ -28,9 +28,6 @@ namespace ircd::m::dbs
0xFFUL << ref_shift
};
// event_idx | ref_type, event_idx
extern db::domain event_refs;
string_view
event_refs_key(const mutable_buffer &out,
const event::idx &tgt,
@ -42,17 +39,22 @@ namespace ircd::m::dbs
string_view
reflect(const ref &);
void _index_event_refs(db::txn &, const event &, const write_opts &);
// event_idx | ref_type, event_idx
extern db::domain event_refs;
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__event_refs__block__size;
extern conf::item<size_t> events__event_refs__meta_block__size;
extern conf::item<size_t> events__event_refs__cache__size;
extern conf::item<size_t> events__event_refs__cache_comp__size;
extern const db::prefix_transform events__event_refs__pfx;
extern const db::comparator events__event_refs__cmp;
extern const db::descriptor events__event_refs;
extern conf::item<size_t> event_refs__block__size;
extern conf::item<size_t> event_refs__meta_block__size;
extern conf::item<size_t> event_refs__cache__size;
extern conf::item<size_t> event_refs__cache_comp__size;
extern const db::prefix_transform event_refs__pfx;
extern const db::comparator event_refs__cmp;
extern const db::descriptor event_refs;
}
/// Types of references indexed by event_refs. This is a single byte integer,

14
include/ircd/m/dbs/event_sender.h
View file

@ -29,6 +29,8 @@ namespace ircd::m::dbs
string_view event_sender_origin_key(const mutable_buffer &out, const id::user &, const event::idx &);
std::tuple<string_view, event::idx> event_sender_origin_key(const string_view &amalgam);
void _index_event_sender(db::txn &, const event &, const write_opts &);
// mxid | event_idx
// host | local, event_idx (see event_sender_origin.h)
extern db::domain event_sender;
@ -36,10 +38,10 @@ namespace ircd::m::dbs
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__event_sender__block__size;
extern conf::item<size_t> events__event_sender__meta_block__size;
extern conf::item<size_t> events__event_sender__cache__size;
extern conf::item<size_t> events__event_sender__cache_comp__size;
extern const db::prefix_transform events__event_sender__pfx;
extern const db::descriptor events__event_sender;
extern conf::item<size_t> event_sender__block__size;
extern conf::item<size_t> event_sender__meta_block__size;
extern conf::item<size_t> event_sender__cache__size;
extern conf::item<size_t> event_sender__cache_comp__size;
extern const db::prefix_transform event_sender__pfx;
extern const db::descriptor event_sender;
}

14
include/ircd/m/dbs/event_state.h
View file

@ -30,6 +30,8 @@ namespace ircd::m::dbs
string_view event_state_key(const mutable_buffer &out, const event_state_tuple &);
event_state_tuple event_state_key(const string_view &);
void _index_event_state(db::txn &, const event &, const write_opts &);
// state_key, type, room_id, depth, event_idx
extern db::domain event_state;
}
@ -37,10 +39,10 @@ namespace ircd::m::dbs
namespace ircd::m::dbs::desc
{
// events _event_state
extern conf::item<size_t> events__event_state__block__size;
extern conf::item<size_t> events__event_state__meta_block__size;
extern conf::item<size_t> events__event_state__cache__size;
extern conf::item<size_t> events__event_state__cache_comp__size;
extern const db::comparator events__event_state__cmp;
extern const db::descriptor events__event_state;
extern conf::item<size_t> event_state__block__size;
extern conf::item<size_t> event_state__meta_block__size;
extern conf::item<size_t> event_state__cache__size;
extern conf::item<size_t> event_state__cache_comp__size;
extern const db::comparator event_state__cmp;
extern const db::descriptor event_state;
}

14
include/ircd/m/dbs/event_type.h
View file

@ -21,6 +21,8 @@ namespace ircd::m::dbs
string_view event_type_key(const mutable_buffer &out, const string_view &, const event::idx & = 0);
std::tuple<event::idx> event_type_key(const string_view &amalgam);
void _index_event_type(db::txn &, const event &, const write_opts &);
// type | event_idx => -
extern db::domain event_type;
}
@ -28,10 +30,10 @@ namespace ircd::m::dbs
namespace ircd::m::dbs::desc
{
// events type
extern conf::item<size_t> events__event_type__block__size;
extern conf::item<size_t> events__event_type__meta_block__size;
extern conf::item<size_t> events__event_type__cache__size;
extern conf::item<size_t> events__event_type__cache_comp__size;
extern const db::prefix_transform events__event_type__pfx;
extern const db::descriptor events__event_type;
extern conf::item<size_t> event_type__block__size;
extern conf::item<size_t> event_type__meta_block__size;
extern conf::item<size_t> event_type__cache__size;
extern conf::item<size_t> event_type__cache_comp__size;
extern const db::prefix_transform event_type__pfx;
extern const db::descriptor event_type;
}

16
include/ircd/m/dbs/room_events.h
View file

@ -22,6 +22,8 @@ namespace ircd::m::dbs
string_view room_events_key(const mutable_buffer &out, const id::room &, const uint64_t &depth);
std::tuple<uint64_t, event::idx> room_events_key(const string_view &amalgam);
void _index_room_events(db::txn &, const event &, const write_opts &);
// room_id | depth, event_idx => node_id
extern db::domain room_events;
}
@ -29,11 +31,11 @@ namespace ircd::m::dbs
namespace ircd::m::dbs::desc
{
// room events sequence
extern conf::item<size_t> events__room_events__block__size;
extern conf::item<size_t> events__room_events__meta_block__size;
extern conf::item<size_t> events__room_events__cache__size;
extern conf::item<size_t> events__room_events__cache_comp__size;
extern const db::prefix_transform events__room_events__pfx;
extern const db::comparator events__room_events__cmp;
extern const db::descriptor events__room_events;
extern conf::item<size_t> room_events__block__size;
extern conf::item<size_t> room_events__meta_block__size;
extern conf::item<size_t> room_events__cache__size;
extern conf::item<size_t> room_events__cache_comp__size;
extern const db::prefix_transform room_events__pfx;
extern const db::comparator room_events__cmp;
extern const db::descriptor room_events;
}

13
include/ircd/m/dbs/room_head.h
View file

@ -21,15 +21,18 @@ namespace ircd::m::dbs
string_view room_head_key(const mutable_buffer &out, const id::room &, const id::event &);
string_view room_head_key(const string_view &amalgam);
void _index_room_head_resolve(db::txn &, const event &, const write_opts &);
void _index_room_head(db::txn &, const event &, const write_opts &);
// room_id | event_id => event_idx
extern db::domain room_head;
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__room_head__block__size;
extern conf::item<size_t> events__room_head__meta_block__size;
extern conf::item<size_t> events__room_head__cache__size;
extern const db::prefix_transform events__room_head__pfx;
extern const db::descriptor events__room_head;
extern conf::item<size_t> room_head__block__size;
extern conf::item<size_t> room_head__meta_block__size;
extern conf::item<size_t> room_head__cache__size;
extern const db::prefix_transform room_head__pfx;
extern const db::descriptor room_head;
}

16
include/ircd/m/dbs/room_joined.h
View file

@ -22,17 +22,19 @@ namespace ircd::m::dbs
string_view room_joined_key(const mutable_buffer &out, const id::room &, const string_view &origin);
std::tuple<string_view, string_view> room_joined_key(const string_view &amalgam);
void _index_room_joined(db::txn &, const event &, const write_opts &);
// room_id | origin, member => event_idx
extern db::domain room_joined;
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__room_joined__block__size;
extern conf::item<size_t> events__room_joined__meta_block__size;
extern conf::item<size_t> events__room_joined__cache__size;
extern conf::item<size_t> events__room_joined__cache_comp__size;
extern conf::item<size_t> events__room_joined__bloom__bits;
extern const db::prefix_transform events__room_joined__pfx;
extern const db::descriptor events__room_joined;
extern conf::item<size_t> room_joined__block__size;
extern conf::item<size_t> room_joined__meta_block__size;
extern conf::item<size_t> room_joined__cache__size;
extern conf::item<size_t> room_joined__cache_comp__size;
extern conf::item<size_t> room_joined__bloom__bits;
extern const db::prefix_transform room_joined__pfx;
extern const db::descriptor room_joined;
}

16
include/ircd/m/dbs/room_state.h
View file

@ -22,17 +22,19 @@ namespace ircd::m::dbs
string_view room_state_key(const mutable_buffer &out, const id::room &, const string_view &type);
std::tuple<string_view, string_view> room_state_key(const string_view &amalgam);
void _index_room_state(db::txn &, const event &, const write_opts &);
// room_id | type, state_key => event_idx
extern db::domain room_state;
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__room_state__block__size;
extern conf::item<size_t> events__room_state__meta_block__size;
extern conf::item<size_t> events__room_state__cache__size;
extern conf::item<size_t> events__room_state__cache_comp__size;
extern conf::item<size_t> events__room_state__bloom__bits;
extern const db::prefix_transform events__room_state__pfx;
extern const db::descriptor events__room_state;
extern conf::item<size_t> room_state__block__size;
extern conf::item<size_t> room_state__meta_block__size;
extern conf::item<size_t> room_state__cache__size;
extern conf::item<size_t> room_state__cache_comp__size;
extern conf::item<size_t> room_state__bloom__bits;
extern const db::prefix_transform room_state__pfx;
extern const db::descriptor room_state;
}

18
include/ircd/m/dbs/room_state_space.h
View file

@ -30,18 +30,20 @@ namespace ircd::m::dbs
string_view room_state_space_key(const mutable_buffer &out, const id::room &);
room_state_space_key_parts room_state_space_key(const string_view &amalgam);
void _index_room_state_space(db::txn &, const event &, const write_opts &);
// room_id | type, state_key, depth, event_idx => --
extern db::domain room_state_space;
}
namespace ircd::m::dbs::desc
{
extern conf::item<size_t> events__room_state_space__block__size;
extern conf::item<size_t> events__room_state_space__meta_block__size;
extern conf::item<size_t> events__room_state_space__cache__size;
extern conf::item<size_t> events__room_state_space__cache_comp__size;
extern conf::item<size_t> events__room_state_space__bloom__bits;
extern const db::prefix_transform events__room_state_space__pfx;
extern const db::comparator events__room_state_space__cmp;
extern const db::descriptor events__room_state_space;
extern conf::item<size_t> room_state_space__block__size;
extern conf::item<size_t> room_state_space__meta_block__size;
extern conf::item<size_t> room_state_space__cache__size;
extern conf::item<size_t> room_state_space__cache_comp__size;
extern conf::item<size_t> room_state_space__bloom__bits;
extern const db::prefix_transform room_state_space__pfx;
extern const db::comparator room_state_space__cmp;
extern const db::descriptor room_state_space;
}

16
include/ircd/m/dbs/room_type.h
View file

@ -35,6 +35,8 @@ namespace ircd::m::dbs
const uint64_t &depth = -1,
const event::idx & = -1);
void _index_room_type(db::txn &, const event &, const write_opts &);
// room_id | type, depth, event_idx
extern db::domain room_type;
}
@ -42,11 +44,11 @@ namespace ircd::m::dbs
namespace ircd::m::dbs::desc
{
// room events sequence
extern conf::item<size_t> events__room_type__block__size;
extern conf::item<size_t> events__room_type__meta_block__size;
extern conf::item<size_t> events__room_type__cache__size;
extern conf::item<size_t> events__room_type__cache_comp__size;
extern const db::prefix_transform events__room_type__pfx;
extern const db::comparator events__room_type__cmp;
extern const db::descriptor events__room_type;
extern conf::item<size_t> room_type__block__size;
extern conf::item<size_t> room_type__meta_block__size;
extern conf::item<size_t> room_type__cache__size;
extern conf::item<size_t> room_type__cache_comp__size;
extern const db::prefix_transform room_type__pfx;
extern const db::comparator room_type__cmp;
extern const db::descriptor room_type;
}

15
matrix/Makefile.am
View file

@ -60,6 +60,21 @@ libircd_matrix_la_SOURCES =#
libircd_matrix_la_SOURCES += name.cc
libircd_matrix_la_SOURCES += id.cc
libircd_matrix_la_SOURCES += dbs.cc
libircd_matrix_la_SOURCES += dbs_event_idx.cc
libircd_matrix_la_SOURCES += dbs_event_json.cc
libircd_matrix_la_SOURCES += dbs_event_column.cc
libircd_matrix_la_SOURCES += dbs_event_refs.cc
libircd_matrix_la_SOURCES += dbs_event_horizon.cc
libircd_matrix_la_SOURCES += dbs_event_sender.cc
libircd_matrix_la_SOURCES += dbs_event_type.cc
libircd_matrix_la_SOURCES += dbs_event_state.cc
libircd_matrix_la_SOURCES += dbs_room_events.cc
libircd_matrix_la_SOURCES += dbs_room_type.cc
libircd_matrix_la_SOURCES += dbs_room_state.cc
libircd_matrix_la_SOURCES += dbs_room_state_space.cc
libircd_matrix_la_SOURCES += dbs_room_joined.cc
libircd_matrix_la_SOURCES += dbs_room_head.cc
libircd_matrix_la_SOURCES += dbs_desc.cc
libircd_matrix_la_SOURCES += hook.cc
libircd_matrix_la_SOURCES += event.cc
libircd_matrix_la_SOURCES += event_cached.cc

5315
matrix/dbs.cc
View file

File diff suppressed because it is too large Load diff

547
matrix/dbs_desc.cc Normal file
View file

@ -0,0 +1,547 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::m::dbs::desc
{
// Deprecated / dropped columns.
//
// These have to be retained for users that have yet to open their
// database with a newly released schema which has dropped a column
// from the schema. If the legacy descriptor is not provided here then
// the database will not know how to open the descriptor in order to
// conduct the drop.
extern const ircd::db::descriptor events_auth_events;
extern const ircd::db::descriptor events_hashes;
extern const ircd::db::descriptor events_membership;
extern const ircd::db::descriptor events_origin;
extern const ircd::db::descriptor events_prev_events;
extern const ircd::db::descriptor events_prev_state;
extern const ircd::db::descriptor events_redacts;
extern const ircd::db::descriptor events_signatures;
extern const ircd::db::descriptor events__event_auth;
extern const ircd::db::comparator events__event_auth__cmp;
extern const ircd::db::prefix_transform events__event_auth__pfx;
extern const ircd::db::descriptor events__event_bad;
extern const ircd::db::descriptor events__state_node;
//
// Required by RocksDB
//
extern const ircd::db::descriptor events__default;
};
const ircd::db::prefix_transform
ircd::m::dbs::desc::events__event_auth__pfx
{
"_event_auth",
nullptr,
nullptr,
};
const ircd::db::comparator
ircd::m::dbs::desc::events__event_auth__cmp
{
"_event_auth",
nullptr,
nullptr,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events__event_auth
{
// name
"_event_auth",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
events__event_auth__cmp,
// prefix transform
events__event_auth__pfx,
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events__event_bad
{
// name
"_event_bad",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(string_view), typeid(uint64_t)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_auth_events
{
// name
"auth_events",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_hashes
{
// name
"hashes",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_membership
{
// name
"membership",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_origin
{
// name
"origin",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_prev_events
{
// name
"prev_events",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_prev_state
{
// name
"prev_state",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(ircd::string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_redacts
{
// name
"redacts",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events_signatures
{
// name
"signatures",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events__state_node
{
// name
"_state_node",
// explanation
R"(
This column is deprecated and has been dropped from the schema. This
descriptor will erase its presence in the database upon next open.
)",
// typing (key, value)
{
typeid(ircd::string_view), typeid(ircd::string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
true,
};
const ircd::db::descriptor
ircd::m::dbs::desc::events__default
{
// name
"default",
// explanation
R"(This column is unused but required by the database software.
)",
// typing (key, value)
{
typeid(string_view), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
0_MiB,
// cache size for compressed assets
0_MiB,
// bloom filter bits
0,
// expect queries hit
false,
};
//
// Description vector
//
decltype(ircd::m::dbs::desc::events)
ircd::m::dbs::desc::events
{
// Requirement of RocksDB/LevelDB
events__default,
//
// These columns directly represent event fields indexed by event_idx
// number and the value is the actual event values. Some values may be
// JSON, like content.
//
content,
depth,
event_id,
origin_server_ts,
room_id,
sender,
state_key,
type,
//
// These columns are metadata oriented around the event data.
//
// event_id => uint64_t
// Mapping of event_id to index number.
event_idx,
// event_idx => json
// Mapping of event_idx to full json
event_json,
// event_idx | event_idx
// Reverse mapping of the event reference graph.
event_refs,
// event_idx | event_idx
// Mapping of unresolved event refs.
event_horizon,
// origin | sender, event_idx
// Mapping of senders to event_idx's they are the sender of.
event_sender,
// type | event_idx
// Mapping of type strings to event_idx's of that type.
event_type,
// state_key, type, room_id, depth, event_idx
// Mapping of event states, indexed for application features.
event_state,
// (room_id, (depth, event_idx))
// Sequence of all events for a room, ever.
room_events,
// (room_id, (type, depth, event_idx))
// Sequence of all events by type for a room.
room_type,
// (room_id, (origin, user_id))
// Sequence of all PRESENTLY JOINED joined for a room.
room_joined,
// (room_id, (type, state_key)) => (event_idx)
// Sequence of the PRESENT STATE of the room.
room_state,
// (room_id, (type, state_key, depth, event_idx))
// Sequence of all states of the room.
room_state_space,
// (room_id, event_id) => (event_idx)
// Mapping of all current head events for a room.
room_head,
//
// These columns are legacy; they have been dropped from the schema.
//
events_auth_events,
events_hashes,
events_membership,
events_origin,
events_prev_events,
events_prev_state,
events_redacts,
events_signatures,
events__event_auth,
events__event_bad,
events__state_node,
};

868
matrix/dbs_event_column.cc Normal file
View file

@ -0,0 +1,868 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
/// Linkage for a cache of the columns of the events database which directly
/// correspond to a property in the matrix event object. This array allows
/// for constant time access to a column the same way one can make constant
/// time access to a property in m::event.
decltype(ircd::m::dbs::event_column)
ircd::m::dbs::event_column;
decltype(ircd::m::dbs::desc::_event__bloom__bits)
ircd::m::dbs::desc::_event__bloom__bits
{
{ "name", "ircd.m.dbs.__event.bloom.bits" },
{ "default", 8L },
};
decltype(ircd::m::dbs::desc::event_id__block__size)
ircd::m::dbs::desc::event_id__block__size
{
{ "name", "ircd.m.dbs.event_id.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_id__meta_block__size)
ircd::m::dbs::desc::event_id__meta_block__size
{
{ "name", "ircd.m.dbs.event_id.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_id__cache__size)
ircd::m::dbs::desc::event_id__cache__size
{
{
{ "name", "ircd.m.dbs.event_id.cache.size" },
{ "default", long(32_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "event_id"_>()));
const size_t &value{event_id__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::event_id__cache_comp__size)
ircd::m::dbs::desc::event_id__cache_comp__size
{
{
{ "name", "ircd.m.dbs.event_id.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "event_id"_>()));
const size_t &value{event_id__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::event_id
{
// name
"event_id",
// explanation
R"(Stores the event_id property of an event.
As with all direct event columns the key is an event_idx and the value
is the data for the event. It should be mentioned for this column
specifically that event_id's are already saved in the _event_idx column
however that is a mapping of event_id to event_idx whereas this is a
mapping of event_idx to event_id.
10.4
MUST NOT exceed 255 bytes.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(event_id__block__size),
// meta_block size
size_t(event_id__meta_block__size),
};
//
// type
//
decltype(ircd::m::dbs::desc::type__block__size)
ircd::m::dbs::desc::type__block__size
{
{ "name", "ircd.m.dbs.type.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::type__meta_block__size)
ircd::m::dbs::desc::type__meta_block__size
{
{ "name", "ircd.m.dbs.type.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::type__cache__size)
ircd::m::dbs::desc::type__cache__size
{
{
{ "name", "ircd.m.dbs.type.cache.size" },
{ "default", long(32_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "type"_>()));
const size_t &value{type__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::type__cache_comp__size)
ircd::m::dbs::desc::type__cache_comp__size
{
{
{ "name", "ircd.m.dbs.type.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "type"_>()));
const size_t &value{type__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::type
{
// name
"type",
// explanation
R"(Stores the type property of an event.
10.1
The type of event. This SHOULD be namespaced similar to Java package naming conventions
e.g. 'com.example.subdomain.event.type'.
10.4
MUST NOT exceed 255 bytes.
### developer note:
key is event_idx number.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(type__block__size),
// meta_block size
size_t(type__meta_block__size),
};
//
// content
//
decltype(ircd::m::dbs::desc::content__block__size)
ircd::m::dbs::desc::content__block__size
{
{ "name", "ircd.m.dbs.content.block.size" },
{ "default", 2048L },
};
decltype(ircd::m::dbs::desc::content__meta_block__size)
ircd::m::dbs::desc::content__meta_block__size
{
{ "name", "ircd.m.dbs.content.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::content__cache__size)
ircd::m::dbs::desc::content__cache__size
{
{
{ "name", "ircd.m.dbs.content.cache.size" },
{ "default", long(48_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "content"_>()));
const size_t &value{content__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::content__cache_comp__size)
ircd::m::dbs::desc::content__cache_comp__size
{
{
{ "name", "ircd.m.dbs.content.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "content"_>()));
const size_t &value{content__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::content
{
// name
"content",
// explanation
R"(Stores the content property of an event.
10.1
The fields in this object will vary depending on the type of event. When interacting
with the REST API, this is the HTTP body.
### developer note:
Since events must not exceed 64 KiB the maximum size for the content is the remaining
space after all the other fields for the event are rendered.
key is event_idx number.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(content__block__size),
// meta_block size
size_t(content__meta_block__size),
};
//
// room_id
//
decltype(ircd::m::dbs::desc::room_id__block__size)
ircd::m::dbs::desc::room_id__block__size
{
{ "name", "ircd.m.dbs.room_id.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::room_id__meta_block__size)
ircd::m::dbs::desc::room_id__meta_block__size
{
{ "name", "ircd.m.dbs.room_id.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::room_id__cache__size)
ircd::m::dbs::desc::room_id__cache__size
{
{
{ "name", "ircd.m.dbs.room_id.cache.size" },
{ "default", long(32_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "room_id"_>()));
const size_t &value{room_id__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::room_id__cache_comp__size)
ircd::m::dbs::desc::room_id__cache_comp__size
{
{
{ "name", "ircd.m.dbs.room_id.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "room_id"_>()));
const size_t &value{room_id__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::room_id
{
// name
"room_id",
// explanation
R"(Stores the room_id property of an event.
10.2 (apropos room events)
Required. The ID of the room associated with this event.
10.4
MUST NOT exceed 255 bytes.
### developer note:
key is event_idx number.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(room_id__block__size),
// meta_block size
size_t(room_id__meta_block__size),
};
//
// sender
//
decltype(ircd::m::dbs::desc::sender__block__size)
ircd::m::dbs::desc::sender__block__size
{
{ "name", "ircd.m.dbs.sender.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::sender__meta_block__size)
ircd::m::dbs::desc::sender__meta_block__size
{
{ "name", "ircd.m.dbs.sender.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::sender__cache__size)
ircd::m::dbs::desc::sender__cache__size
{
{
{ "name", "ircd.m.dbs.sender.cache.size" },
{ "default", long(32_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "sender"_>()));
const size_t &value{sender__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::sender__cache_comp__size)
ircd::m::dbs::desc::sender__cache_comp__size
{
{
{ "name", "ircd.m.dbs.sender.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "sender"_>()));
const size_t &value{sender__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::sender
{
// name
"sender",
// explanation
R"(Stores the sender property of an event.
10.2 (apropos room events)
Required. Contains the fully-qualified ID of the user who sent this event.
10.4
MUST NOT exceed 255 bytes.
### developer note:
key is event_idx number.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(sender__block__size),
// meta_block size
size_t(sender__meta_block__size),
};
//
// state_key
//
decltype(ircd::m::dbs::desc::state_key__block__size)
ircd::m::dbs::desc::state_key__block__size
{
{ "name", "ircd.m.dbs.state_key.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::state_key__meta_block__size)
ircd::m::dbs::desc::state_key__meta_block__size
{
{ "name", "ircd.m.dbs.state_key.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::state_key__cache__size)
ircd::m::dbs::desc::state_key__cache__size
{
{
{ "name", "ircd.m.dbs.state_key.cache.size" },
{ "default", long(32_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "state_key"_>()));
const size_t &value{state_key__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::state_key__cache_comp__size)
ircd::m::dbs::desc::state_key__cache_comp__size
{
{
{ "name", "ircd.m.dbs.state_key.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "state_key"_>()));
const size_t &value{state_key__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::state_key
{
// name
"state_key",
// explanation
R"(Stores the state_key property of an event.
10.3 (apropos room state events)
A unique key which defines the overwriting semantics for this piece of room state.
This value is often a zero-length string. The presence of this key makes this event a
State Event. The key MUST NOT start with '_'.
10.4
MUST NOT exceed 255 bytes.
### developer note:
key is event_idx number.
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(state_key__block__size),
// meta_block size
size_t(state_key__meta_block__size),
};
//
// origin_server_ts
//
decltype(ircd::m::dbs::desc::origin_server_ts__block__size)
ircd::m::dbs::desc::origin_server_ts__block__size
{
{ "name", "ircd.m.dbs.origin_server_ts.block.size" },
{ "default", 256L },
};
decltype(ircd::m::dbs::desc::origin_server_ts__meta_block__size)
ircd::m::dbs::desc::origin_server_ts__meta_block__size
{
{ "name", "ircd.m.dbs.origin_server_ts.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::origin_server_ts__cache__size)
ircd::m::dbs::desc::origin_server_ts__cache__size
{
{
{ "name", "ircd.m.dbs.origin_server_ts.cache.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "origin_server_ts"_>()));
const size_t &value{origin_server_ts__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::origin_server_ts__cache_comp__size)
ircd::m::dbs::desc::origin_server_ts__cache_comp__size
{
{
{ "name", "ircd.m.dbs.origin_server_ts.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "origin_server_ts"_>()));
const size_t &value{origin_server_ts__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::origin_server_ts
{
// name
"origin_server_ts",
// explanation
R"(Stores the origin_server_ts property of an event.
FEDERATION 4.1
Timestamp in milliseconds on origin homeserver when this PDU was created.
### developer note:
key is event_idx number.
value is a machine integer (binary)
TODO: consider unsigned rather than time_t because of millisecond precision
)",
// typing (key, value)
{
typeid(uint64_t), typeid(time_t)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(origin_server_ts__block__size),
// meta_block size
size_t(origin_server_ts__meta_block__size),
};
//
// depth
//
decltype(ircd::m::dbs::desc::depth__block__size)
ircd::m::dbs::desc::depth__block__size
{
{ "name", "ircd.m.dbs.depth.block.size" },
{ "default", 256L },
};
decltype(ircd::m::dbs::desc::depth__meta_block__size)
ircd::m::dbs::desc::depth__meta_block__size
{
{ "name", "ircd.m.dbs.depth.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::depth__cache__size)
ircd::m::dbs::desc::depth__cache__size
{
{
{ "name", "ircd.m.dbs.depth.cache.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "depth"_>()));
const size_t &value{depth__cache__size};
db::capacity(db::cache(column), value);
}
};
decltype(ircd::m::dbs::desc::depth__cache_comp__size)
ircd::m::dbs::desc::depth__cache_comp__size
{
{
{ "name", "ircd.m.dbs.depth.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
auto &column(event_column.at(json::indexof<event, "depth"_>()));
const size_t &value{depth__cache_comp__size};
db::capacity(db::cache_compressed(column), value);
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::depth
{
// name
"depth",
// explanation
R"(Stores the depth property of an event.
### developer note:
key is event_idx number. value is long integer
)",
// typing (key, value)
{
typeid(uint64_t), typeid(int64_t)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(_event__bloom__bits),
// expect queries hit
true,
// block size
size_t(depth__block__size),
// meta_block size
size_t(depth__meta_block__size),
};
void
ircd::m::dbs::_index_event_cols(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_COLS));
assert(opts.event_idx);
const byte_view<string_view> key
{
opts.event_idx
};
size_t i{0};
for_each(event, [&txn, &opts, &key, &i]
(const auto &, auto&& val)
{
auto &column
{
event_column.at(i++)
};
if(!column)
return;
if(value_required(opts.op) && !defined(json::value(val)))
return;
db::txn::append
{
txn, column, db::column::delta
{
opts.op,
string_view{key},
value_required(opts.op)?
byte_view<string_view>{val}:
byte_view<string_view>{}
}
};
});
}

310
matrix/dbs_event_horizon.cc Normal file
View file

@ -0,0 +1,310 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::event_horizon)
ircd::m::dbs::event_horizon;
decltype(ircd::m::dbs::desc::event_horizon__block__size)
ircd::m::dbs::desc::event_horizon__block__size
{
{ "name", "ircd.m.dbs._event_horizon.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_horizon__meta_block__size)
ircd::m::dbs::desc::event_horizon__meta_block__size
{
{ "name", "ircd.m.dbs._event_horizon.meta_block.size" },
{ "default", 1024L },
};
decltype(ircd::m::dbs::desc::event_horizon__cache__size)
ircd::m::dbs::desc::event_horizon__cache__size
{
{
{ "name", "ircd.m.dbs._event_horizon.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{event_horizon__cache__size};
db::capacity(db::cache(dbs::event_horizon), value);
}
};
decltype(ircd::m::dbs::desc::event_horizon__cache_comp__size)
ircd::m::dbs::desc::event_horizon__cache_comp__size
{
{
{ "name", "ircd.m.dbs._event_horizon.cache_comp.size" },
{ "default", long(0_MiB) },
}, []
{
const size_t &value{event_horizon__cache_comp__size};
db::capacity(db::cache_compressed(dbs::event_horizon), value);
}
};
const ircd::db::prefix_transform
ircd::m::dbs::desc::event_horizon__pfx
{
"_event_horizon",
[](const string_view &key)
{
return has(key, '\0');
},
[](const string_view &key)
{
assert(size(key) >= sizeof(event::idx));
return split(key, '\0').first;
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::event_horizon
{
// name
"_event_horizon",
// explanation
R"(Unresolved references in the reverse reference graph of events.
event_id | event_idx => --
The first part of the key is an event_id which the server does not have.
The suffix of the key is the index number of an event which the server
does have and it contains a reference to event_id.
We use the information in this column to find all of the events which
have an unresolved reference to this event and complete the holes in the
event_refs graph which could not be completed without this event.
When a new event is written to the database the event_horizon column is
queried seeking the event's ID. Each entry in event_horizon is the index
of an event which we previously wrote to the database without knowing the
index of the event currently being written (an out-of-order write).
)",
// typing (key, value)
{
typeid(string_view), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
event_horizon__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0, //uses conf item
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
0,
// expect queries hit
false,
// block size
size_t(event_horizon__block__size),
// meta_block size
size_t(event_horizon__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
// NOTE: QUERY
void
ircd::m::dbs::_index_event_horizon_resolve(db::txn &txn,
const event &event,
const write_opts &opts)
{
char buf[EVENT_HORIZON_KEY_MAX_SIZE];
assert(opts.appendix.test(appendix::EVENT_HORIZON_RESOLVE));
assert(opts.event_idx != 0);
assert(event.event_id);
const string_view &key
{
event_horizon_key(buf, event.event_id)
};
auto it
{
dbs::event_horizon.begin(key)
};
for(; it; ++it)
{
const auto parts
{
event_horizon_key(it->first)
};
const auto &event_idx
{
std::get<0>(parts)
};
assert(event_idx != 0);
assert(event_idx != opts.event_idx);
const event::fetch _event
{
event_idx, std::nothrow
};
if(!_event.valid)
{
log::dwarning
{
log, "Horizon resolve for %s @%lu not possible @%lu",
string_view{event.event_id},
opts.event_idx,
event_idx,
};
continue;
}
log::debug
{
log, "Horizon resolve for %s @%lu; remisé %s @%lu",
string_view{event.event_id},
opts.event_idx,
string_view{_event.event_id},
event_idx,
};
// Make the references on behalf of the future event
write_opts _opts;
_opts.op = opts.op;
_opts.event_idx = event_idx;
_opts.appendix.reset();
_opts.appendix.set(appendix::EVENT_REFS);
_opts.appendix.set(appendix::ROOM_REDACT);
_opts.event_refs = opts.horizon_resolve;
_opts.interpose = &txn;
write(txn, _event, _opts);
// Delete the event_horizon entry after resolving.
thread_local char buf[EVENT_HORIZON_KEY_MAX_SIZE];
const string_view &key
{
event_horizon_key(buf, event.event_id, event_idx)
};
db::txn::append
{
txn, dbs::event_horizon,
{
opts.op == db::op::SET?
db::op::DELETE:
db::op::SET,
key
}
};
}
}
void
ircd::m::dbs::_index_event_horizon(db::txn &txn,
const event &event,
const write_opts &opts,
const m::event::id &unresolved_id)
{
thread_local char buf[EVENT_HORIZON_KEY_MAX_SIZE];
assert(opts.appendix.test(appendix::EVENT_HORIZON));
assert(opts.event_idx != 0 && unresolved_id);
const string_view &key
{
event_horizon_key(buf, unresolved_id, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_horizon,
{
opts.op, key
}
};
}
//
// key
//
std::tuple<ircd::m::event::idx>
ircd::m::dbs::event_horizon_key(const string_view &amalgam)
{
assert(size(amalgam) == 1 + sizeof(event::idx));
assert(amalgam[0] == '\0');
const byte_view<event::idx> &event_idx
{
amalgam.substr(1)
};
return
{
static_cast<event::idx>(event_idx)
};
}
ircd::string_view
ircd::m::dbs::event_horizon_key(const mutable_buffer &out,
const event::id &event_id)
{
return event_horizon_key(out, event_id, 0UL);
}
ircd::string_view
ircd::m::dbs::event_horizon_key(const mutable_buffer &out,
const event::id &event_id,
const event::idx &event_idx)
{
mutable_buffer buf(out);
consume(buf, copy(buf, event_id));
if(event_idx)
{
consume(buf, copy(buf, "\0"_sv));
consume(buf, copy(buf, byte_view<string_view>(event_idx)));
}
const string_view ret
{
data(out), data(buf)
};
assert(size(ret) == size(event_id) || size(ret) == size(event_id) + sizeof(event::idx) + 1);
return ret;
}

170
matrix/dbs_event_idx.cc Normal file
View file

@ -0,0 +1,170 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::event_idx)
ircd::m::dbs::event_idx;
decltype(ircd::m::dbs::desc::event_idx__block__size)
ircd::m::dbs::desc::event_idx__block__size
{
{ "name", "ircd.m.dbs._event_idx.block.size" },
{ "default", 256L },
};
decltype(ircd::m::dbs::desc::event_idx__meta_block__size)
ircd::m::dbs::desc::event_idx__meta_block__size
{
{ "name", "ircd.m.dbs._event_idx.meta_block.size" },
{ "default", 2048L },
};
decltype(ircd::m::dbs::desc::event_idx__cache__size)
ircd::m::dbs::desc::event_idx__cache__size
{
{
{ "name", "ircd.m.dbs._event_idx.cache.size" },
{ "default", long(64_MiB) },
}, []
{
const size_t &value{event_idx__cache__size};
db::capacity(db::cache(dbs::event_idx), value);
}
};
decltype(ircd::m::dbs::desc::event_idx__cache_comp__size)
ircd::m::dbs::desc::event_idx__cache_comp__size
{
{
{ "name", "ircd.m.dbs._event_idx.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{event_idx__cache_comp__size};
db::capacity(db::cache_compressed(dbs::event_idx), value);
}
};
decltype(ircd::m::dbs::desc::event_idx__bloom__bits)
ircd::m::dbs::desc::event_idx__bloom__bits
{
{ "name", "ircd.m.dbs._event_idx.bloom.bits" },
{ "default", 10L },
};
decltype(ircd::m::dbs::desc::event_idx)
ircd::m::dbs::desc::event_idx
{
// name
"_event_idx",
// explanation
R"(Maps matrix event_id strings into internal index numbers.
event_id => event_idx
The key is an event_id and the value is the index number to be used as the
key to all the event data columns. The index number is referred to as the
event_idx and is a fixed 8 byte unsigned integer. All other columns which
may key on an event_id string instead use this event_idx index number. The
index number was generated sequentially based on the order the event was
written to the database. Index numbers start at 1 because 0 is used as a
sentinel value and is not valid. The index numbers throughout the database
generally do not have gaps and can be iterated, however gaps may exist when
an event is erased from the database (which is rare for the matrix
application).
)",
// typing (key, value)
{
typeid(string_view), typeid(uint64_t)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0, //uses conf item
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(event_idx__bloom__bits),
// expect queries hit
false,
// block size
size_t(event_idx__block__size),
// meta_block size
size_t(event_idx__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
void
ircd::m::dbs::_index_event_id(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_ID));
assert(opts.event_idx);
assert(event.event_id);
db::txn::append
{
txn, dbs::event_idx,
{
opts.op,
string_view{event.event_id},
byte_view<string_view>(opts.event_idx)
}
};
// For a v1 event, the "event_id" property will be saved into the `event_id`
// column by the direct property->column indexer.
if(json::get<"event_id"_>(event))
return;
// For v3+ events, the direct column indexer won't see any "event_id"
// property. In this case we insert the `event.event_id` manually into
// that column here.
db::txn::append
{
txn, event_column.at(json::indexof<m::event, "event_id"_>()),
{
opts.op,
byte_view<string_view>(opts.event_idx),
string_view{event.event_id},
}
};
}

188
matrix/dbs_event_json.cc Normal file
View file

@ -0,0 +1,188 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::event_json)
ircd::m::dbs::event_json;
decltype(ircd::m::dbs::desc::event_json__block__size)
ircd::m::dbs::desc::event_json__block__size
{
{ "name", "ircd.m.dbs._event_json.block.size" },
{ "default", long(1_KiB) },
};
decltype(ircd::m::dbs::desc::event_json__meta_block__size)
ircd::m::dbs::desc::event_json__meta_block__size
{
{ "name", "ircd.m.dbs._event_json.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_json__cache__size)
ircd::m::dbs::desc::event_json__cache__size
{
{
{ "name", "ircd.m.dbs._event_json.cache.size" },
{ "default", long(64_MiB) },
}, []
{
const size_t &value{event_json__cache__size};
db::capacity(db::cache(dbs::event_json), value);
}
};
decltype(ircd::m::dbs::desc::event_json__cache_comp__size)
ircd::m::dbs::desc::event_json__cache_comp__size
{
{
{ "name", "ircd.m.dbs._event_json.cache_comp.size" },
{ "default", long(0_MiB) },
}, []
{
const size_t &value{event_json__cache_comp__size};
db::capacity(db::cache_compressed(dbs::event_json), value);
}
};
decltype(ircd::m::dbs::desc::event_json__bloom__bits)
ircd::m::dbs::desc::event_json__bloom__bits
{
{ "name", "ircd.m.dbs._event_json.bloom.bits" },
{ "default", 9L },
};
const ircd::db::descriptor
ircd::m::dbs::desc::event_json
{
// name
"_event_json",
// explanation
R"(Full JSON object of an event.
event_idx => event_json
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0, //uses conf item
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(event_json__bloom__bits),
// expect queries hit
true,
// block size
size_t(event_json__block__size),
// meta_block size
size_t(event_json__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestLargestSeqFirst"s,
// target_file_size
{
2_GiB, // base
1L, // multiplier
},
// max_bytes_for_level[8]
{
{ 128_MiB, 1L }, // max_bytes_for_level_base
{ 0L, 0L }, // max_bytes_for_level[0]
{ 0L, 1L }, // max_bytes_for_level[1]
{ 0L, 1L }, // max_bytes_for_level[2]
{ 0L, 3L }, // max_bytes_for_level[3]
{ 0L, 7L }, // max_bytes_for_level[4]
{ 0L, 15L }, // max_bytes_for_level[5]
{ 0L, 31L }, // max_bytes_for_level[6]
},
};
//
// indexer
//
void
ircd::m::dbs::_index_event_json(db::txn &txn,
const event &event,
const write_opts &opts)
{
const ctx::critical_assertion ca;
thread_local char buf[m::event::MAX_SIZE];
assert(opts.appendix.test(appendix::EVENT_JSON));
assert(opts.event_idx);
const string_view &key
{
byte_view<string_view>(opts.event_idx)
};
const string_view &val
{
// If an already-strung json::object is carried by the event and
// the opts allow us, we use it directly. This is not the default
// path unless the developer knows the source JSON is good enough
// to store directly.
opts.op == db::op::SET && event.source && opts.json_source?
string_view{event.source}:
// If an already-strung json::object is carried by the event we
// re-stringify it into a temporary buffer. This is the common case
// because the original source might be crap JSON w/ spaces etc.
opts.op == db::op::SET && event.source?
json::stringify(mutable_buffer{buf}, event.source):
// If no source was given with the event we can generate it.
opts.op == db::op::SET?
json::stringify(mutable_buffer{buf}, event):
// Empty value; generally for a non-SET db::op
string_view{}
};
db::txn::append
{
txn, event_json,
{
opts.op, // db::op
key, // key
val, // val
}
};
}

782
matrix/dbs_event_refs.cc Normal file
View file

@ -0,0 +1,782 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::m::dbs
{
static void _index_event_refs_m_room_redaction(db::txn &, const event &, const write_opts &); //query
static void _index_event_refs_m_receipt_m_read(db::txn &, const event &, const write_opts &); //query
static void _index_event_refs_m_relates_m_reply(db::txn &, const event &, const write_opts &); //query
static void _index_event_refs_m_relates(db::txn &, const event &, const write_opts &); //query
static void _index_event_refs_state(db::txn &, const event &, const write_opts &); // query
static void _index_event_refs_auth(db::txn &, const event &, const write_opts &); //query
static void _index_event_refs_prev(db::txn &, const event &, const write_opts &); //query
static bool event_refs__cmp_less(const string_view &a, const string_view &b);
}
decltype(ircd::m::dbs::event_refs)
ircd::m::dbs::event_refs;
decltype(ircd::m::dbs::desc::event_refs__block__size)
ircd::m::dbs::desc::event_refs__block__size
{
{ "name", "ircd.m.dbs._event_refs.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_refs__meta_block__size)
ircd::m::dbs::desc::event_refs__meta_block__size
{
{ "name", "ircd.m.dbs._event_refs.meta_block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_refs__cache__size)
ircd::m::dbs::desc::event_refs__cache__size
{
{
{ "name", "ircd.m.dbs._event_refs.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{event_refs__cache__size};
db::capacity(db::cache(dbs::event_refs), value);
}
};
decltype(ircd::m::dbs::desc::event_refs__cache_comp__size)
ircd::m::dbs::desc::event_refs__cache_comp__size
{
{
{ "name", "ircd.m.dbs._event_refs.cache_comp.size" },
{ "default", long(0_MiB) },
}, []
{
const size_t &value{event_refs__cache_comp__size};
db::capacity(db::cache_compressed(dbs::event_refs), value);
}
};
const ircd::db::prefix_transform
ircd::m::dbs::desc::event_refs__pfx
{
"_event_refs",
[](const string_view &key)
{
return size(key) >= sizeof(event::idx) * 2;
},
[](const string_view &key)
{
assert(size(key) >= sizeof(event::idx));
return string_view
{
data(key), data(key) + sizeof(event::idx)
};
}
};
const ircd::db::comparator
ircd::m::dbs::desc::event_refs__cmp
{
"_event_refs",
event_refs__cmp_less,
std::equal_to<string_view>{},
};
const ircd::db::descriptor
ircd::m::dbs::desc::event_refs
{
// name
"_event_refs",
// explanation
R"(Inverse reference graph of events.
event_idx | ref, event_idx => --
The first part of the key is the event being referenced. The second part
of the key is the event which refers to the first event somewhere in its
prev_events references. The event_idx in the second part of the key also
contains a dbs::ref type in its highest order byte so we can store
different kinds of references.
The prefix transform is in effect; an event may be referenced multiple
times. We can find all the events we have which reference a target, and
why. The database must already contain both events (hence they have
event::idx numbers).
The value is currently unused/empty; we may eventually store metadata with
information about this reference (i.e. is depth adjacent? is the ref
redundant with another in the same event and should not be made? etc).
)",
// typing (key, value)
{
typeid(uint64_t), typeid(string_view)
},
// options
{},
// comparator
event_refs__cmp,
// prefix transform
event_refs__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0, //uses conf item
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
0,
// expect queries hit
true,
// block size
size_t(event_refs__block__size),
// meta_block size
size_t(event_refs__meta_block__size),
// compression
{}, // no compression for this column
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexers
//
void
ircd::m::dbs::_index_event_refs(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
if(opts.event_refs.test(uint(ref::NEXT)))
_index_event_refs_prev(txn, event, opts);
if(opts.event_refs.test(uint(ref::NEXT_AUTH)))
_index_event_refs_auth(txn, event, opts);
if(opts.event_refs.test(uint(ref::NEXT_STATE)) ||
opts.event_refs.test(uint(ref::PREV_STATE)))
_index_event_refs_state(txn, event, opts);
if(opts.event_refs.test(uint(ref::M_RECEIPT__M_READ)))
_index_event_refs_m_receipt_m_read(txn, event, opts);
if(opts.event_refs.test(uint(ref::M_RELATES)))
_index_event_refs_m_relates(txn, event, opts);
if(opts.event_refs.test(uint(ref::M_RELATES)))
_index_event_refs_m_relates_m_reply(txn, event, opts);
if(opts.event_refs.test(uint(ref::M_ROOM_REDACTION)))
_index_event_refs_m_room_redaction(txn, event, opts);
}
// NOTE: QUERY
void
ircd::m::dbs::_index_event_refs_prev(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
assert(opts.event_refs.test(uint(ref::NEXT)));
const event::prev prev{event};
for(size_t i(0); i < prev.prev_events_count(); ++i)
{
const event::id &prev_id
{
prev.prev_event(i)
};
const event::idx &prev_idx
{
find_event_idx(prev_id, opts)
};
if(opts.appendix.test(appendix::EVENT_HORIZON) && !prev_idx)
{
_index_event_horizon(txn, event, opts, prev_id);
continue;
}
else if(!prev_idx)
{
log::dwarning
{
log, "No index found to ref %s PREV of %s",
string_view{prev_id},
string_view{event.event_id},
};
continue;
}
thread_local char buf[EVENT_REFS_KEY_MAX_SIZE];
assert(opts.event_idx != 0 && prev_idx != 0);
assert(opts.event_idx != prev_idx);
const string_view &key
{
event_refs_key(buf, prev_idx, ref::NEXT, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
}
// NOTE: QUERY
void
ircd::m::dbs::_index_event_refs_auth(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
assert(opts.event_refs.test(uint(ref::NEXT_AUTH)));
if(!m::room::auth::is_power_event(event))
return;
const event::prev prev{event};
for(size_t i(0); i < prev.auth_events_count(); ++i)
{
const event::id &auth_id
{
prev.auth_event(i)
};
const event::idx &auth_idx
{
find_event_idx(auth_id, opts)
};
if(unlikely(!auth_idx))
{
if(opts.appendix.test(appendix::EVENT_HORIZON))
_index_event_horizon(txn, event, opts, auth_id);
log::error
{
log, "No index found to ref %s AUTH of %s",
string_view{auth_id},
string_view{event.event_id}
};
continue;
}
thread_local char buf[EVENT_REFS_KEY_MAX_SIZE];
assert(opts.event_idx != 0 && auth_idx != 0);
assert(opts.event_idx != auth_idx);
const string_view &key
{
event_refs_key(buf, auth_idx, ref::NEXT_AUTH, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
}
// NOTE: QUERY
void
ircd::m::dbs::_index_event_refs_state(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
assert(opts.event_refs.test(uint(ref::NEXT_STATE)) ||
opts.event_refs.test(uint(ref::PREV_STATE)));
if(!json::get<"room_id"_>(event))
return;
if(!json::get<"state_key"_>(event))
return;
const m::room room
{
at<"room_id"_>(event) //TODO: ABA ABA ABA ABA
};
const m::room::state state
{
room
};
const event::idx &prev_state_idx
{
opts.allow_queries?
state.get(std::nothrow, at<"type"_>(event), at<"state_key"_>(event)): // query
0UL
};
// No previous state; nothing to do.
if(!prev_state_idx)
return;
// If the previous state's event_idx is greater than the event_idx of the
// event we're transacting this is almost surely a replay/rewrite. Bail
// out for now rather than corrupting the graph.
if(unlikely(prev_state_idx >= opts.event_idx))
return;
thread_local char buf[EVENT_REFS_KEY_MAX_SIZE];
assert(opts.event_idx != 0 && prev_state_idx != 0);
assert(opts.event_idx != prev_state_idx);
if(opts.event_refs.test(uint(ref::NEXT_STATE)))
{
const string_view &key
{
event_refs_key(buf, prev_state_idx, ref::NEXT_STATE, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
if(opts.event_refs.test(uint(ref::PREV_STATE)))
{
const string_view &key
{
event_refs_key(buf, opts.event_idx, ref::PREV_STATE, prev_state_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
}
// NOTE: QUERY
void
ircd::m::dbs::_index_event_refs_m_receipt_m_read(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
assert(opts.event_refs.test(uint(ref::M_RECEIPT__M_READ)));
if(json::get<"type"_>(event) != "ircd.read")
return;
if(!my_host(json::get<"origin"_>(event)))
return;
//TODO: disallow local forge?
const json::string &event_id
{
json::get<"content"_>(event).get("event_id")
};
const event::idx &event_idx
{
find_event_idx(event_id, opts)
};
if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx)
{
_index_event_horizon(txn, event, opts, event_id);
return;
}
else if(!event_idx)
{
log::dwarning
{
log, "No index found to ref %s M_RECEIPT__M_READ of %s",
string_view{event_id},
string_view{event.event_id}
};
return;
}
thread_local char buf[EVENT_REFS_KEY_MAX_SIZE];
assert(opts.event_idx != 0 && event_idx != 0);
assert(opts.event_idx != event_idx);
const string_view &key
{
event_refs_key(buf, event_idx, ref::M_RECEIPT__M_READ, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
// NOTE: QUERY
void
ircd::m::dbs::_index_event_refs_m_relates(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
assert(opts.event_refs.test(uint(ref::M_RELATES)));
if(!json::get<"content"_>(event).has("m.relates_to"))
return;
if(json::type(json::get<"content"_>(event).get("m.relates_to")) != json::OBJECT)
return;
const json::object &m_relates_to
{
json::get<"content"_>(event).get("m.relates_to")
};
const json::string &event_id
{
m_relates_to.get("event_id")
};
if(!event_id)
return;
if(!valid(m::id::EVENT, event_id))
{
log::derror
{
log, "Cannot index m.relates_to in %s; '%s' is not an event_id.",
string_view{event.event_id},
string_view{event_id}
};
return;
}
const event::idx &event_idx
{
find_event_idx(event_id, opts)
};
if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx)
{
// If we don't have the event being related to yet, place a marker in
// the event_horizon indicating need for re-evaluation later.
_index_event_horizon(txn, event, opts, event_id);
return;
}
else if(!event_idx)
{
log::derror
{
log, "Cannot index m.relates_to in %s; referenced %s not found.",
string_view{event.event_id},
string_view{event_id}
};
return;
}
thread_local char buf[EVENT_REFS_KEY_MAX_SIZE];
assert(opts.event_idx != 0 && event_idx != 0);
assert(opts.event_idx != event_idx);
const string_view &key
{
event_refs_key(buf, event_idx, ref::M_RELATES, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
// NOTE: QUERY
void
ircd::m::dbs::_index_event_refs_m_relates_m_reply(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
assert(opts.event_refs.test(uint(ref::M_RELATES)));
if(json::get<"type"_>(event) != "m.room.message")
return;
if(!json::get<"content"_>(event).has("m.relates_to"))
return;
if(json::type(json::get<"content"_>(event).get("m.relates_to")) != json::OBJECT)
return;
const json::object &m_relates_to
{
json::get<"content"_>(event).get("m.relates_to")
};
if(!m_relates_to.has("m.in_reply_to"))
return;
if(json::type(m_relates_to.get("m.in_reply_to")) != json::OBJECT)
{
log::derror
{
log, "Cannot index m.in_reply_to in %s; not an OBJECT.",
string_view{event.event_id}
};
return;
}
const json::object &m_in_reply_to
{
m_relates_to.get("m.in_reply_to")
};
const json::string &event_id
{
m_in_reply_to.get("event_id")
};
if(!valid(m::id::EVENT, event_id))
{
log::derror
{
log, "Cannot index m.in_reply_to in %s; '%s' is not an event_id.",
string_view{event.event_id},
string_view{event_id}
};
return;
}
const event::idx &event_idx
{
find_event_idx(event_id, opts)
};
if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx)
{
_index_event_horizon(txn, event, opts, event_id);
return;
}
else if(!event_idx)
{
log::dwarning
{
log, "Cannot index m.in_reply_to in %s; referenced %s not found.",
string_view{event.event_id},
string_view{event_id}
};
return;
}
thread_local char buf[EVENT_REFS_KEY_MAX_SIZE];
assert(opts.event_idx != 0 && event_idx != 0);
assert(opts.event_idx != event_idx);
const string_view &key
{
event_refs_key(buf, event_idx, ref::M_RELATES, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
// NOTE: QUERY
void
ircd::m::dbs::_index_event_refs_m_room_redaction(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_REFS));
assert(opts.event_refs.test(uint(ref::M_ROOM_REDACTION)));
if(json::get<"type"_>(event) != "m.room.redaction")
return;
if(!valid(m::id::EVENT, json::get<"redacts"_>(event)))
return;
const auto &event_id
{
json::get<"redacts"_>(event)
};
const event::idx &event_idx
{
find_event_idx(event_id, opts)
};
if(opts.appendix.test(appendix::EVENT_HORIZON) && !event_idx)
{
_index_event_horizon(txn, event, opts, event_id);
return;
}
else if(!event_idx)
{
log::dwarning
{
log, "Cannot index m.room.redaction in %s; referenced %s not found.",
string_view{event.event_id},
string_view{event_id}
};
return;
}
thread_local char buf[EVENT_REFS_KEY_MAX_SIZE];
assert(opts.event_idx != 0 && event_idx != 0);
assert(opts.event_idx != event_idx);
const string_view &key
{
event_refs_key(buf, event_idx, ref::M_ROOM_REDACTION, opts.event_idx)
};
db::txn::append
{
txn, dbs::event_refs,
{
opts.op, key
}
};
}
//
// cmp
//
bool
ircd::m::dbs::event_refs__cmp_less(const string_view &a,
const string_view &b)
{
static const size_t half(sizeof(event::idx));
static const size_t full(half * 2);
assert(size(a) >= half);
assert(size(b) >= half);
const event::idx *const key[2]
{
reinterpret_cast<const event::idx *>(data(a)),
reinterpret_cast<const event::idx *>(data(b)),
};
return
key[0][0] < key[1][0]? true:
key[0][0] > key[1][0]? false:
size(a) < size(b)? true:
size(a) > size(b)? false:
size(a) == half? false:
key[0][1] < key[1][1]? true:
false;
}
//
// key
//
std::tuple<ircd::m::dbs::ref, ircd::m::event::idx>
ircd::m::dbs::event_refs_key(const string_view &amalgam)
{
const event::idx key
{
byte_view<event::idx>{amalgam}
};
return
{
ref(key >> ref_shift), key & ~ref_mask
};
}
ircd::string_view
ircd::m::dbs::event_refs_key(const mutable_buffer &out,
const event::idx &tgt,
const ref &type,
const event::idx &src)
{
assert((src & ref_mask) == 0);
assert(size(out) >= sizeof(event::idx) * 2);
event::idx *const &key
{
reinterpret_cast<event::idx *>(data(out))
};
key[0] = tgt;
key[1] = src;
key[1] |= uint64_t(type) << ref_shift;
return string_view
{
data(out), data(out) + sizeof(event::idx) * 2
};
}
//
// util
//
ircd::string_view
ircd::m::dbs::reflect(const ref &type)
{
switch(type)
{
case ref::NEXT: return "NEXT";
case ref::NEXT_AUTH: return "NEXT_AUTH";
case ref::NEXT_STATE: return "NEXT_STATE";
case ref::PREV_STATE: return "PREV_STATE";
case ref::M_RECEIPT__M_READ: return "M_RECEIPT__M_READ";
case ref::M_RELATES: return "M_RELATES";
case ref::M_ROOM_REDACTION: return "M_ROOM_REDACTION";
}
return "????";
}

297
matrix/dbs_event_sender.cc Normal file
View file

@ -0,0 +1,297 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::event_sender)
ircd::m::dbs::event_sender;
decltype(ircd::m::dbs::desc::event_sender__block__size)
ircd::m::dbs::desc::event_sender__block__size
{
{ "name", "ircd.m.dbs._event_sender.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_sender__meta_block__size)
ircd::m::dbs::desc::event_sender__meta_block__size
{
{ "name", "ircd.m.dbs._event_sender.meta_block.size" },
{ "default", 4096L },
};
decltype(ircd::m::dbs::desc::event_sender__cache__size)
ircd::m::dbs::desc::event_sender__cache__size
{
{
{ "name", "ircd.m.dbs._event_sender.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{event_sender__cache__size};
db::capacity(db::cache(dbs::event_sender), value);
}
};
decltype(ircd::m::dbs::desc::event_sender__cache_comp__size)
ircd::m::dbs::desc::event_sender__cache_comp__size
{
{
{ "name", "ircd.m.dbs._event_sender.cache_comp.size" },
{ "default", long(0_MiB) },
}, []
{
const size_t &value{event_sender__cache_comp__size};
db::capacity(db::cache_compressed(dbs::event_sender), value);
}
};
const ircd::db::prefix_transform
ircd::m::dbs::desc::event_sender__pfx
{
"_event_sender",
[](const string_view &key)
{
return startswith(key, '@')?
has(key, '\0'):
has(key, '@');
},
[](const string_view &key)
{
const auto &[prefix, suffix]
{
// Split @localpart:hostpart\0event_idx by '\0'
startswith(key, '@')?
split(key, '\0'):
// Split hostpart@localpart\0event_idx by '@'
split(key, '@')
};
return prefix;
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::event_sender
{
// name
"_event_sender",
// explanation
R"(Index of senders to their events.
mxid | event_idx => --
origin | localpart, event_idx => --
The senders of events are indexed by this column. This allows for all
events from a sender to be iterated. Additionally, all events from a
server and all known servers can be iterated from this column.
key #1:
The first type of key is made from a user mxid and an event_idx concat.
key #2:
The second type of key is made from a user mxid and an event_id, where
the mxid is part-swapped so the origin comes first, and the @localpart
comes after.
Note that the indexers of this column ignores the actual "origin" field
of an event. Only the "sender" data is used here.
)",
// typing (key, value)
{
typeid(string_view), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
event_sender__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0, //uses conf item
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
0,
// expect queries hit
false,
// block size
size_t(event_sender__block__size),
// meta_block size
size_t(event_sender__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
void
ircd::m::dbs::_index_event_sender(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_SENDER));
assert(json::get<"sender"_>(event));
assert(opts.event_idx);
thread_local char buf[2][EVENT_SENDER_KEY_MAX_SIZE];
const string_view &sender_key
{
event_sender_key(buf[0], at<"sender"_>(event), opts.event_idx)
};
const string_view &sender_origin_key
{
event_sender_origin_key(buf[1], at<"sender"_>(event), opts.event_idx)
};
db::txn::append
{
txn, dbs::event_sender,
{
opts.op, sender_key
}
};
db::txn::append
{
txn, dbs::event_sender,
{
opts.op, sender_origin_key
}
};
}
//
// key
//
// sender_key
std::tuple<ircd::m::event::idx>
ircd::m::dbs::event_sender_key(const string_view &amalgam)
{
const auto &parts
{
split(amalgam, '\0')
};
assert(empty(parts.first));
return
{
byte_view<event::idx>(parts.second),
};
}
ircd::string_view
ircd::m::dbs::event_sender_key(const mutable_buffer &out_,
const user::id &user_id,
const event::idx &event_idx)
{
assert(size(out_) >= EVENT_SENDER_KEY_MAX_SIZE);
assert(!event_idx || user_id);
mutable_buffer out{out_};
consume(out, copy(out, user_id));
if(user_id && event_idx)
{
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, byte_view<string_view>(event_idx)));
}
return { data(out_), data(out) };
}
bool
ircd::m::dbs::is_event_sender_key(const string_view &key)
{
return empty(key) || startswith(key, '@');
}
// sender_origin_key
std::tuple<ircd::string_view, ircd::m::event::idx>
ircd::m::dbs::event_sender_origin_key(const string_view &amalgam)
{
const auto &parts
{
split(amalgam, '\0')
};
assert(!empty(parts.first) && !empty(parts.second));
assert(startswith(parts.first, '@'));
return
{
parts.first,
byte_view<event::idx>(parts.second),
};
}
ircd::string_view
ircd::m::dbs::event_sender_origin_key(const mutable_buffer &out,
const user::id &user_id,
const event::idx &event_idx)
{
return event_sender_origin_key(out, user_id.host(), user_id.local(), event_idx);
}
ircd::string_view
ircd::m::dbs::event_sender_origin_key(const mutable_buffer &out_,
const string_view &origin,
const string_view &localpart,
const event::idx &event_idx)
{
assert(size(out_) >= EVENT_SENDER_KEY_MAX_SIZE);
assert(!event_idx || localpart);
assert(!localpart || startswith(localpart, '@'));
mutable_buffer out{out_};
consume(out, copy(out, origin));
consume(out, copy(out, localpart));
if(localpart && event_idx)
{
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, byte_view<string_view>(event_idx)));
}
return { data(out_), data(out) };
}
bool
ircd::m::dbs::is_event_sender_origin_key(const string_view &key)
{
return !startswith(key, '@');
}

278
matrix/dbs_event_state.cc Normal file
View file

@ -0,0 +1,278 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::m::dbs
{
static bool event_state__cmp_lt(const string_view &, const string_view &);
}
decltype(ircd::m::dbs::event_state)
ircd::m::dbs::event_state;
decltype(ircd::m::dbs::desc::event_state__block__size)
ircd::m::dbs::desc::event_state__block__size
{
{ "name", "ircd.m.dbs._event_state.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_state__meta_block__size)
ircd::m::dbs::desc::event_state__meta_block__size
{
{ "name", "ircd.m.dbs._event_state.meta_block.size" },
{ "default", long(2_KiB) },
};
decltype(ircd::m::dbs::desc::event_state__cache__size)
ircd::m::dbs::desc::event_state__cache__size
{
{
{ "name", "ircd.m.dbs._event_state.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{event_state__cache__size};
db::capacity(db::cache(dbs::event_state), value);
}
};
decltype(ircd::m::dbs::desc::event_state__cache_comp__size)
ircd::m::dbs::desc::event_state__cache_comp__size
{
{
{ "name", "ircd.m.dbs._event_state.cache_comp.size" },
{ "default", long(0_MiB) },
}, []
{
const size_t &value{event_state__cache_comp__size};
db::capacity(db::cache_compressed(dbs::event_state), value);
}
};
const ircd::db::comparator
ircd::m::dbs::desc::event_state__cmp
{
"_event_state",
event_state__cmp_lt,
std::equal_to<string_view>{},
};
const ircd::db::descriptor
ircd::m::dbs::desc::event_state
{
// name
"_event_state",
// explanation
R"(Index of states of events.
state_key, type, room_id, depth, event_idx => --
The state transitions of events are indexed by this column,
based on the state_key property.
)",
// typing (key, value)
{
typeid(string_view), typeid(string_view)
},
// options
{},
// comparator
event_state__cmp,
// prefix transform
{},
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0, //uses conf item
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
0,
// expect queries hit
false,
// block size
size_t(event_state__block__size),
// meta_block size
size_t(event_state__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
void
ircd::m::dbs::_index_event_state(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_STATE));
assert(json::get<"type"_>(event));
assert(opts.event_idx);
if(!defined(json::get<"state_key"_>(event)))
return;
thread_local char buf[EVENT_STATE_KEY_MAX_SIZE];
db::txn::append
{
txn, dbs::event_state,
{
opts.op, event_state_key(buf, event_state_tuple
{
at<"state_key"_>(event),
at<"type"_>(event),
at<"room_id"_>(event),
at<"depth"_>(event),
opts.event_idx,
})
}
};
}
//
// cmp
//
bool
ircd::m::dbs::event_state__cmp_lt(const string_view &a,
const string_view &b)
{
const event_state_tuple key[2]
{
event_state_key(a),
event_state_key(b),
};
const auto &[state_key_a, type_a, room_id_a, depth_a, event_idx_a]
{
key[0]
};
const auto &[state_key_b, type_b, room_id_b, depth_b, event_idx_b]
{
key[1]
};
if(state_key_a != state_key_b)
return state_key_a < state_key_b;
if(type_a != type_b)
return type_a < type_b;
if(room_id_a != room_id_b)
return room_id_a < room_id_b;
if(depth_a != depth_b)
return depth_a > depth_b;
if(event_idx_a != event_idx_b)
return event_idx_a > event_idx_b;
return false;
}
//
// key
//
ircd::m::dbs::event_state_tuple
ircd::m::dbs::event_state_key(const string_view &amalgam)
{
assert(!startswith(amalgam, '\0'));
const auto &[state_key, r0]
{
split(amalgam, "\0"_sv)
};
const auto &[type, r1]
{
split(r0, "\0"_sv)
};
const auto &[room_id, r2]
{
split(r1, "\0"_sv)
};
assert(!room_id || m::valid(m::id::ROOM, room_id));
return event_state_tuple
{
state_key,
type,
room_id,
r2.size() >= 8?
int64_t(byte_view<int64_t>(r2.substr(0, 8))):
-1L,
r2.size() >= 16?
event::idx(byte_view<uint64_t>(r2.substr(8))):
0UL,
};
}
ircd::string_view
ircd::m::dbs::event_state_key(const mutable_buffer &out_,
const event_state_tuple &tuple)
{
assert(size(out_) >= EVENT_STATE_KEY_MAX_SIZE);
const auto &[state_key, type, room_id, depth, event_idx]
{
tuple
};
if(!state_key)
return {};
mutable_buffer out{out_};
consume(out, copy(out, state_key));
if(!type)
return {data(out_), data(out)};
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, type));
if(!room_id)
return {data(out_), data(out)};
assert(m::valid(m::id::ROOM, room_id));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, room_id));
if(depth < 0)
return {data(out_), data(out)};
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, byte_view<string_view>(depth)));
if(!event_idx)
return {data(out_), data(out)};
consume(out, copy(out, byte_view<string_view>(event_idx)));
return {data(out_), data(out)};
}

191
matrix/dbs_event_type.cc Normal file
View file

@ -0,0 +1,191 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::event_type)
ircd::m::dbs::event_type;
decltype(ircd::m::dbs::desc::event_type__block__size)
ircd::m::dbs::desc::event_type__block__size
{
{ "name", "ircd.m.dbs._event_type.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::event_type__meta_block__size)
ircd::m::dbs::desc::event_type__meta_block__size
{
{ "name", "ircd.m.dbs._event_type.meta_block.size" },
{ "default", long(4_KiB) },
};
decltype(ircd::m::dbs::desc::event_type__cache__size)
ircd::m::dbs::desc::event_type__cache__size
{
{
{ "name", "ircd.m.dbs._event_type.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{event_type__cache__size};
db::capacity(db::cache(dbs::event_type), value);
}
};
decltype(ircd::m::dbs::desc::event_type__cache_comp__size)
ircd::m::dbs::desc::event_type__cache_comp__size
{
{
{ "name", "ircd.m.dbs._event_type.cache_comp.size" },
{ "default", long(0_MiB) },
}, []
{
const size_t &value{event_type__cache_comp__size};
db::capacity(db::cache_compressed(dbs::event_type), value);
}
};
const ircd::db::prefix_transform
ircd::m::dbs::desc::event_type__pfx
{
"_event_type",
[](const string_view &key)
{
return has(key, '\0');
},
[](const string_view &key)
{
return split(key, '\0').first;
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::event_type
{
// name
"_event_type",
// explanation
R"(Index of types of events.
type | event_idx => --
The types of events are indexed by this column. All events of a specific type can be
iterated efficiently. The type string forms the prefix domain.
)",
// typing (key, value)
{
typeid(string_view), typeid(string_view)
},
// options
{},
// comparator
{},
// prefix transform
event_type__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0, //uses conf item
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
0,
// expect queries hit
false,
// block size
size_t(event_type__block__size),
// meta_block size
size_t(event_type__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
void
ircd::m::dbs::_index_event_type(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::EVENT_TYPE));
assert(json::get<"type"_>(event));
assert(opts.event_idx);
thread_local char buf[EVENT_TYPE_KEY_MAX_SIZE];
const string_view &key
{
event_type_key(buf, at<"type"_>(event), opts.event_idx)
};
db::txn::append
{
txn, dbs::event_type,
{
opts.op, key
}
};
}
//
// key
//
std::tuple<ircd::m::event::idx>
ircd::m::dbs::event_type_key(const string_view &amalgam)
{
assert(size(amalgam) == sizeof(event::idx) + 1);
const auto &key
{
amalgam.substr(1)
};
assert(size(key) == sizeof(event::idx));
return
{
byte_view<event::idx>(key)
};
}
ircd::string_view
ircd::m::dbs::event_type_key(const mutable_buffer &out_,
const string_view &type,
const event::idx &event_idx)
{
assert(size(out_) >= EVENT_TYPE_KEY_MAX_SIZE);
mutable_buffer out{out_};
consume(out, copy(out, type));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, byte_view<string_view>(event_idx)));
return { data(out_), data(out) };
}

312
matrix/dbs_room_events.cc Normal file
View file

@ -0,0 +1,312 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::m::dbs
{
static bool room_events__cmp_lt(const string_view &, const string_view &);
}
/// Linkage for a reference to the room_events column
decltype(ircd::m::dbs::room_events)
ircd::m::dbs::room_events;
decltype(ircd::m::dbs::desc::room_events__block__size)
ircd::m::dbs::desc::room_events__block__size
{
{ "name", "ircd.m.dbs._room_events.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::room_events__meta_block__size)
ircd::m::dbs::desc::room_events__meta_block__size
{
{ "name", "ircd.m.dbs._room_events.meta_block.size" },
{ "default", long(16_KiB) },
};
decltype(ircd::m::dbs::desc::room_events__cache__size)
ircd::m::dbs::desc::room_events__cache__size
{
{
{ "name", "ircd.m.dbs._room_events.cache.size" },
{ "default", long(32_MiB) },
}, []
{
const size_t &value{room_events__cache__size};
db::capacity(db::cache(dbs::room_events), value);
}
};
decltype(ircd::m::dbs::desc::room_events__cache_comp__size)
ircd::m::dbs::desc::room_events__cache_comp__size
{
{
{ "name", "ircd.m.dbs._room_events.cache_comp.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{room_events__cache_comp__size};
db::capacity(db::cache_compressed(dbs::room_events), value);
}
};
/// Prefix transform for the room_events. The prefix here is a room_id
/// and the suffix is the depth+event_id concatenation.
/// for efficient sequences
///
const ircd::db::prefix_transform
ircd::m::dbs::desc::room_events__pfx
{
"_room_events",
[](const string_view &key)
{
return has(key, "\0"_sv);
},
[](const string_view &key)
{
return split(key, "\0"_sv).first;
}
};
/// Comparator for the room_events. The goal here is to sort the
/// events within a room by their depth from highest to lowest, so the
/// highest depth is hit first when a room is sought from this column.
///
const ircd::db::comparator
ircd::m::dbs::desc::room_events__cmp
{
"_room_events",
room_events__cmp_lt,
std::equal_to<string_view>{},
};
/// This column stores events in sequence in a room. Consider the following:
///
/// [room_id | depth + event_idx]
///
/// The key is composed from three parts:
///
/// - `room_id` is the official prefix, bounding the sequence. That means we
/// make a blind query with just a room_id and get to the beginning of the
/// sequence, then iterate until we stop before the next room_id (upper bound).
///
/// - `depth` is the ordering. Within the sequence, all elements are ordered by
/// depth from HIGHEST TO LOWEST. The sequence will start at the highest depth.
/// NOTE: Depth is a fixed 8 byte binary integer.
///
/// - `event_idx` is the key suffix. This column serves to sequence all events
/// within a room ordered by depth. There may be duplicate room_id|depth
/// prefixing but the event_idx suffix gives the key total uniqueness.
/// NOTE: event_idx is a fixed 8 byte binary integer.
///
const ircd::db::descriptor
ircd::m::dbs::desc::room_events
{
// name
"_room_events",
// explanation
R"(Indexes events in timeline sequence for a room
[room_id | depth + event_idx]
)",
// typing (key, value)
{
typeid(string_view), typeid(string_view)
},
// options
{},
// comparator
room_events__cmp,
// prefix transform
room_events__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
0, // no bloom filter because of possible comparator issues
// expect queries hit
true,
// block size
size_t(room_events__block__size),
// meta_block size
size_t(room_events__meta_block__size),
};
//
// indexer
//
/// Adds the entry for the room_events column into the txn.
void
ircd::m::dbs::_index_room_events(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::ROOM_EVENTS));
thread_local char buf[ROOM_EVENTS_KEY_MAX_SIZE];
const ctx::critical_assertion ca;
const string_view &key
{
room_events_key(buf, at<"room_id"_>(event), at<"depth"_>(event), opts.event_idx)
};
db::txn::append
{
txn, room_events,
{
opts.op, // db::op
key, // key,
}
};
}
//
// cmp
//
bool
ircd::m::dbs::room_events__cmp_lt(const string_view &a,
const string_view &b)
{
static const auto &pt
{
desc::room_events__pfx
};
// Extract the prefix from the keys
const string_view pre[2]
{
pt.get(a),
pt.get(b),
};
if(size(pre[0]) != size(pre[1]))
return size(pre[0]) < size(pre[1]);
if(pre[0] != pre[1])
return pre[0] < pre[1];
// After the prefix is the depth + event_idx
const string_view post[2]
{
a.substr(size(pre[0])),
b.substr(size(pre[1])),
};
// These conditions are matched on some queries when the user only
// supplies a room id.
if(empty(post[0]))
return true;
if(empty(post[1]))
return false;
// Distill out the depth and event_idx integers
const std::tuple<uint64_t, event::idx> pair[2]
{
room_events_key(post[0]),
room_events_key(post[1])
};
// When two events are at the same depth sort by index (the sequence
// number given as they were admitted into the system) otherwise
// sort by depth. Note this is a reverse order comparison.
return std::get<0>(pair[1]) != std::get<0>(pair[0])?
std::get<0>(pair[1]) < std::get<0>(pair[0]):
std::get<1>(pair[1]) < std::get<1>(pair[0]);
}
//
// key
//
std::tuple<uint64_t, ircd::m::event::idx>
ircd::m::dbs::room_events_key(const string_view &amalgam)
{
assert(size(amalgam) >= 1 + 8 + 8 || size(amalgam) == 1 + 8);
assert(amalgam.front() == '\0');
const uint64_t &depth
{
*reinterpret_cast<const uint64_t *>(data(amalgam) + 1)
};
const event::idx &event_idx
{
size(amalgam) >= 1 + 8 + 8?
*reinterpret_cast<const uint64_t *>(data(amalgam) + 1 + 8):
std::numeric_limits<uint64_t>::max()
};
// Make sure these are copied rather than ever returning references in
// a tuple because the chance the integers will be aligned is low.
return { depth, event_idx };
}
ircd::string_view
ircd::m::dbs::room_events_key(const mutable_buffer &out_,
const id::room &room_id,
const uint64_t &depth)
{
const const_buffer depth_cb
{
reinterpret_cast<const char *>(&depth), sizeof(depth)
};
mutable_buffer out{out_};
consume(out, copy(out, room_id));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, depth_cb));
return { data(out_), data(out) };
}
ircd::string_view
ircd::m::dbs::room_events_key(const mutable_buffer &out_,
const id::room &room_id,
const uint64_t &depth,
const event::idx &event_idx)
{
const const_buffer depth_cb
{
reinterpret_cast<const char *>(&depth), sizeof(depth)
};
const const_buffer event_idx_cb
{
reinterpret_cast<const char *>(&event_idx), sizeof(event_idx)
};
mutable_buffer out{out_};
consume(out, copy(out, room_id));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, depth_cb));
consume(out, copy(out, event_idx_cb));
return { data(out_), data(out) };
}

229
matrix/dbs_room_head.cc Normal file
View file

@ -0,0 +1,229 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::room_head)
ircd::m::dbs::room_head;
decltype(ircd::m::dbs::desc::room_head__block__size)
ircd::m::dbs::desc::room_head__block__size
{
{ "name", "ircd.m.dbs._room_head.block.size" },
{ "default", long(4_KiB) },
};
decltype(ircd::m::dbs::desc::room_head__meta_block__size)
ircd::m::dbs::desc::room_head__meta_block__size
{
{ "name", "ircd.m.dbs._room_head.meta_block.size" },
{ "default", long(4_KiB) },
};
decltype(ircd::m::dbs::desc::room_head__cache__size)
ircd::m::dbs::desc::room_head__cache__size
{
{
{ "name", "ircd.m.dbs._room_head.cache.size" },
{ "default", long(8_MiB) },
}, []
{
const size_t &value{room_head__cache__size};
db::capacity(db::cache(dbs::room_head), value);
}
};
/// prefix transform for room_id,event_id in room_id
///
const ircd::db::prefix_transform
ircd::m::dbs::desc::room_head__pfx
{
"_room_head",
[](const string_view &key)
{
return has(key, "\0"_sv);
},
[](const string_view &key)
{
return split(key, "\0"_sv).first;
}
};
/// This column stores unreferenced (head) events for a room.
///
const ircd::db::descriptor
ircd::m::dbs::desc::room_head
{
// name
"_room_head",
// explanation
R"(Unreferenced events in a room.
[room_id | event_id => event_idx]
The key is a room_id and event_id concatenation. The value is an event_idx
of the event_id in the key. The key amalgam was specifically selected to
allow for DELETES sent to the WAL "in the blind" for all prev_events when
any new event is saved to the database, without making any read IO's to
look up anything about the prev reference to remove.
This is a fast-moving column where unreferenced events are inserted and
then deleted the first time another event is seen which references it so
it collects a lot of DELETE commands in the WAL and has to be compacted
often to reduce them out.
)",
// typing (key, value)
{
typeid(string_view), typeid(uint64_t)
},
// options
{},
// comparator
{},
// prefix transform
room_head__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
0, //no compresed cache
// bloom filter bits
0, //table too ephemeral for bloom generation/usefulness
// expect queries hit
false,
// block size
size_t(room_head__block__size),
// meta_block size
size_t(room_head__meta_block__size),
// compression
{}, // no compression for this column
// compactor
{},
// compaction priority algorithm
"kByCompensatedSize"s,
};
//
// indexer
//
void
ircd::m::dbs::_index_room_head(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::ROOM_HEAD));
assert(opts.event_idx);
assert(event.event_id);
const ctx::critical_assertion ca;
thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE];
const string_view &key
{
room_head_key(buf, at<"room_id"_>(event), event.event_id)
};
db::txn::append
{
txn, room_head,
{
opts.op,
key,
byte_view<string_view>{opts.event_idx}
}
};
}
void
ircd::m::dbs::_index_room_head_resolve(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::ROOM_HEAD_RESOLVE));
//TODO: If op is DELETE and we are deleting this event and thereby
//TODO: potentially creating a gap in the reference graph (just for us
//TODO: though) can we *re-add* the prev_events to the head?
if(opts.op != db::op::SET)
return;
const event::prev prev{event};
for(size_t i(0); i < prev.prev_events_count(); ++i)
{
const auto &event_id
{
prev.prev_event(i)
};
thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE];
const ctx::critical_assertion ca;
const string_view &key
{
room_head_key(buf, at<"room_id"_>(event), event_id)
};
db::txn::append
{
txn, room_head,
{
db::op::DELETE,
key,
}
};
}
}
//
// key
//
ircd::string_view
ircd::m::dbs::room_head_key(const string_view &amalgam)
{
const auto &key
{
lstrip(amalgam, "\0"_sv)
};
return
{
key
};
}
ircd::string_view
ircd::m::dbs::room_head_key(const mutable_buffer &out_,
const id::room &room_id,
const id::event &event_id)
{
mutable_buffer out{out_};
consume(out, copy(out, room_id));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, event_id));
return { data(out_), data(out) };
}

244
matrix/dbs_room_joined.cc Normal file
View file

@ -0,0 +1,244 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::room_joined)
ircd::m::dbs::room_joined;
decltype(ircd::m::dbs::desc::room_joined__block__size)
ircd::m::dbs::desc::room_joined__block__size
{
{ "name", "ircd.m.dbs._room_joined.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::room_joined__meta_block__size)
ircd::m::dbs::desc::room_joined__meta_block__size
{
{ "name", "ircd.m.dbs._room_joined.meta_block.size" },
{ "default", long(8_KiB) },
};
decltype(ircd::m::dbs::desc::room_joined__cache__size)
ircd::m::dbs::desc::room_joined__cache__size
{
{
{ "name", "ircd.m.dbs._room_joined.cache.size" },
{ "default", long(8_MiB) },
}, []
{
const size_t &value{room_joined__cache__size};
db::capacity(db::cache(dbs::room_joined), value);
}
};
decltype(ircd::m::dbs::desc::room_joined__cache_comp__size)
ircd::m::dbs::desc::room_joined__cache_comp__size
{
{
{ "name", "ircd.m.dbs._room_joined.cache_comp.size" },
{ "default", long(8_MiB) },
}, []
{
const size_t &value{room_joined__cache_comp__size};
db::capacity(db::cache_compressed(dbs::room_joined), value);
}
};
decltype(ircd::m::dbs::desc::room_joined__bloom__bits)
ircd::m::dbs::desc::room_joined__bloom__bits
{
{ "name", "ircd.m.dbs._room_joined.bloom.bits" },
{ "default", 6L },
};
/// Prefix transform for the room_joined
///
const ircd::db::prefix_transform
ircd::m::dbs::desc::room_joined__pfx
{
"_room_joined",
[](const string_view &key)
{
return has(key, "\0"_sv);
},
[](const string_view &key)
{
return split(key, "\0"_sv).first;
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::room_joined
{
// name
"_room_joined",
// explanation
R"(Specifically indexes joined members of a room for fast iteration.
[room_id | origin + mxid] => event_idx
)",
// typing (key, value)
{
typeid(string_view), typeid(uint64_t)
},
// options
{},
// comparator
{},
// prefix transform
room_joined__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(room_joined__bloom__bits),
// expect queries hit
false,
// block size
size_t(room_joined__block__size),
// meta_block size
size_t(room_joined__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
/// Adds the entry for the room_joined column into the txn.
void
ircd::m::dbs::_index_room_joined(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::ROOM_JOINED));
assert(at<"type"_>(event) == "m.room.member");
thread_local char buf[ROOM_JOINED_KEY_MAX_SIZE];
const ctx::critical_assertion ca;
const string_view &key
{
room_joined_key(buf, at<"room_id"_>(event), at<"origin"_>(event), at<"state_key"_>(event))
};
const string_view &membership
{
m::membership(event)
};
assert(!empty(membership));
db::op op;
if(opts.op == db::op::SET) switch(hash(membership))
{
case hash("join"):
op = db::op::SET;
break;
case hash("ban"):
case hash("leave"):
op = db::op::DELETE;
break;
default:
return;
}
else if(opts.op == db::op::DELETE)
op = opts.op;
else
return;
db::txn::append
{
txn, room_joined,
{
op,
key,
}
};
}
//
// key
//
std::tuple<ircd::string_view, ircd::string_view>
ircd::m::dbs::room_joined_key(const string_view &amalgam)
{
const auto &key
{
lstrip(amalgam, "\0"_sv)
};
const auto &s
{
split(key, "@"_sv)
};
return
{
{ s.first },
!empty(s.second)?
string_view{begin(s.second) - 1, end(s.second)}:
string_view{}
};
}
ircd::string_view
ircd::m::dbs::room_joined_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &origin)
{
mutable_buffer out{out_};
consume(out, copy(out, room_id));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, origin));
return { data(out_), data(out) };
}
ircd::string_view
ircd::m::dbs::room_joined_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &origin,
const id::user &member)
{
mutable_buffer out{out_};
consume(out, copy(out, room_id));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, origin));
consume(out, copy(out, member));
return { data(out_), data(out) };
}

234
matrix/dbs_room_state.cc Normal file
View file

@ -0,0 +1,234 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
decltype(ircd::m::dbs::room_state)
ircd::m::dbs::room_state;
decltype(ircd::m::dbs::desc::room_state__block__size)
ircd::m::dbs::desc::room_state__block__size
{
{ "name", "ircd.m.dbs._room_state.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::room_state__meta_block__size)
ircd::m::dbs::desc::room_state__meta_block__size
{
{ "name", "ircd.m.dbs._room_state.meta_block.size" },
{ "default", 8192L },
};
decltype(ircd::m::dbs::desc::room_state__cache__size)
ircd::m::dbs::desc::room_state__cache__size
{
{
{ "name", "ircd.m.dbs._room_state.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{room_state__cache__size};
db::capacity(db::cache(dbs::room_state), value);
}
};
decltype(ircd::m::dbs::desc::room_state__cache_comp__size)
ircd::m::dbs::desc::room_state__cache_comp__size
{
{
{ "name", "ircd.m.dbs._room_state.cache_comp.size" },
{ "default", long(8_MiB) },
}, []
{
const size_t &value{room_state__cache_comp__size};
db::capacity(db::cache_compressed(dbs::room_state), value);
}
};
decltype(ircd::m::dbs::desc::room_state__bloom__bits)
ircd::m::dbs::desc::room_state__bloom__bits
{
{ "name", "ircd.m.dbs._room_state.bloom.bits" },
{ "default", 10L },
};
/// prefix transform for type,state_key in room_id
///
/// This transform is special for concatenating room_id with type and state_key
/// in that order with prefix being the room_id (this may change to room_id+
/// type
///
const ircd::db::prefix_transform
ircd::m::dbs::desc::room_state__pfx
{
"_room_state",
[](const string_view &key)
{
return has(key, "\0"_sv);
},
[](const string_view &key)
{
return split(key, "\0"_sv).first;
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::room_state
{
// name
"_room_state",
// explanation
R"(The present state of the room.
[room_id | type + state_key] => event_idx
This column is also known as the "present state table." It contains the
very important present state of the room for this server. The key contains
plaintext room_id, type and state_key elements for direct point-lookup as
well as iteration. The value is the index of the apropos state event.
)",
// typing (key, value)
{
typeid(string_view), typeid(uint64_t)
},
// options
{},
// comparator
{},
// prefix transform
room_state__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(room_state__bloom__bits),
// expect queries hit
false,
// block size
size_t(room_state__block__size),
// meta_block size
size_t(room_state__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
void
ircd::m::dbs::_index_room_state(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::ROOM_STATE));
const ctx::critical_assertion ca;
thread_local char buf[ROOM_STATE_KEY_MAX_SIZE];
const string_view &key
{
room_state_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"state_key"_>(event))
};
const string_view val
{
byte_view<string_view>(opts.event_idx)
};
db::txn::append
{
txn, room_state,
{
opts.op,
key,
value_required(opts.op)? val : string_view{},
}
};
}
//
// key
//
ircd::string_view
ircd::m::dbs::room_state_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &type)
{
return room_state_key(out_, room_id, type, string_view{});
}
ircd::string_view
ircd::m::dbs::room_state_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &type,
const string_view &state_key)
{
mutable_buffer out{out_};
consume(out, copy(out, room_id));
assert(room_id);
if(likely(defined(type)))
{
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, type));
}
if(defined(state_key))
{
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, state_key));
}
return { data(out_), data(out) };
}
std::tuple<ircd::string_view, ircd::string_view>
ircd::m::dbs::room_state_key(const string_view &amalgam)
{
const auto &key
{
lstrip(amalgam, "\0"_sv)
};
const auto &s
{
split(key, "\0"_sv)
};
return
{
s.first, s.second
};
}

331
matrix/dbs_room_state_space.cc Normal file
View file

@ -0,0 +1,331 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::m::dbs
{
static bool room_state_space__cmp_lt(const string_view &, const string_view &);
}
decltype(ircd::m::dbs::room_state_space)
ircd::m::dbs::room_state_space;
decltype(ircd::m::dbs::desc::room_state_space__block__size)
ircd::m::dbs::desc::room_state_space__block__size
{
{ "name", "ircd.m.dbs._room_state_space.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::room_state_space__meta_block__size)
ircd::m::dbs::desc::room_state_space__meta_block__size
{
{ "name", "ircd.m.dbs._room_state_space.meta_block.size" },
{ "default", long(8_KiB) },
};
decltype(ircd::m::dbs::desc::room_state_space__cache__size)
ircd::m::dbs::desc::room_state_space__cache__size
{
{
{ "name", "ircd.m.dbs._room_state_space.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{room_state_space__cache__size};
db::capacity(db::cache(dbs::room_state_space), value);
}
};
decltype(ircd::m::dbs::desc::room_state_space__cache_comp__size)
ircd::m::dbs::desc::room_state_space__cache_comp__size
{
{
{ "name", "ircd.m.dbs._room_state_space.cache_comp.size" },
{ "default", long(8_MiB) },
}, []
{
const size_t &value{room_state_space__cache_comp__size};
db::capacity(db::cache_compressed(dbs::room_state_space), value);
}
};
decltype(ircd::m::dbs::desc::room_state_space__bloom__bits)
ircd::m::dbs::desc::room_state_space__bloom__bits
{
{ "name", "ircd.m.dbs._room_state_space.bloom.bits" },
{ "default", 10L },
};
const ircd::db::comparator
ircd::m::dbs::desc::room_state_space__cmp
{
"_room_state_space",
room_state_space__cmp_lt,
std::equal_to<string_view>{},
};
const ircd::db::prefix_transform
ircd::m::dbs::desc::room_state_space__pfx
{
"_room_state_space",
[](const string_view &key)
{
return has(key, "\0"_sv);
},
[](const string_view &key)
{
return split(key, "\0"_sv).first;
}
};
const ircd::db::descriptor
ircd::m::dbs::desc::room_state_space
{
// name
"_room_state_space",
// explanation
R"(All states of the room.
)",
// typing (key, value)
{
typeid(string_view), typeid(uint64_t)
},
// options
{},
// comparator
room_state_space__cmp,
// prefix transform
room_state_space__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
size_t(room_state_space__bloom__bits),
// expect queries hit
false,
// block size
size_t(room_state_space__block__size),
// meta_block size
size_t(room_state_space__meta_block__size),
// compression
"kLZ4Compression;kSnappyCompression"s,
// compactor
{},
// compaction priority algorithm
"kOldestSmallestSeqFirst"s,
};
//
// indexer
//
void
ircd::m::dbs::_index_room_state_space(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::ROOM_STATE_SPACE));
const ctx::critical_assertion ca;
thread_local char buf[ROOM_STATE_SPACE_KEY_MAX_SIZE];
const string_view &key
{
room_state_space_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"state_key"_>(event), at<"depth"_>(event), opts.event_idx)
};
db::txn::append
{
txn, room_state_space,
{
opts.op,
key,
}
};
}
//
// cmp
//
bool
ircd::m::dbs::room_state_space__cmp_lt(const string_view &a,
const string_view &b)
{
static const auto &pt
{
desc::room_state_space__pfx
};
const string_view pre[2]
{
pt.get(a),
pt.get(b),
};
if(size(pre[0]) != size(pre[1]))
return size(pre[0]) < size(pre[1]);
if(pre[0] != pre[1])
return pre[0] < pre[1];
const string_view post[2]
{
a.substr(size(pre[0])),
b.substr(size(pre[1])),
};
// These conditions are matched on some queries when the user only
// supplies a room_id.
if(empty(post[0]))
return true;
if(empty(post[1]))
return false;
// Decompose the postfix of the key for granular sorting
const room_state_space_key_parts k[2]
{
room_state_space_key(post[0]),
room_state_space_key(post[1])
};
// type
if(std::get<0>(k[0]) < std::get<0>(k[1]))
return true;
else if(std::get<0>(k[0]) > std::get<0>(k[1]))
return false;
// state_key
if(std::get<1>(k[0]) < std::get<1>(k[1]))
return true;
else if(std::get<1>(k[0]) > std::get<1>(k[1]))
return false;
// depth (ORDER IS DESCENDING!)
if(uint64_t(std::get<2>(k[0])) > uint64_t(std::get<2>(k[1])))
return true;
else if(uint64_t(std::get<2>(k[0])) < uint64_t(std::get<2>(k[1])))
return false;
// event_idx (ORDER IS DESCENDING!)
if(std::get<3>(k[0]) > std::get<3>(k[1]))
return true;
else if(std::get<3>(k[0]) < std::get<3>(k[1]))
return false;
return false;
}
//
// key
//
ircd::m::dbs::room_state_space_key_parts
ircd::m::dbs::room_state_space_key(const string_view &amalgam)
{
const auto &key
{
lstrip(amalgam, "\0"_sv)
};
const auto &[type, after_type]
{
split(key, "\0"_sv)
};
const auto &[state_key, after_state_key]
{
split(after_type, "\0"_sv)
};
const int64_t &depth
{
size(after_state_key) >= 8?
int64_t(byte_view<int64_t>(after_state_key.substr(0, 8))):
-1L
};
const event::idx &event_idx
{
size(after_state_key) >= 16?
event::idx(byte_view<event::idx>(after_state_key.substr(8, 8))):
0UL
};
return
{
type, state_key, depth, event_idx
};
}
ircd::string_view
ircd::m::dbs::room_state_space_key(const mutable_buffer &out_,
const id::room &room_id)
{
return room_state_space_key(out_, room_id, string_view{}, string_view{}, -1L, 0L);
}
ircd::string_view
ircd::m::dbs::room_state_space_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &type)
{
return room_state_space_key(out_, room_id, type, string_view{}, -1L, 0L);
}
ircd::string_view
ircd::m::dbs::room_state_space_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &type,
const string_view &state_key)
{
return room_state_space_key(out_, room_id, type, state_key, -1L, 0L);
}
ircd::string_view
ircd::m::dbs::room_state_space_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &type,
const string_view &state_key,
const int64_t &depth,
const event::idx &event_idx)
{
mutable_buffer out{out_};
consume(out, copy(out, room_id));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, type));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, state_key));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, byte_view<string_view>(depth)));
consume(out, copy(out, byte_view<string_view>(event_idx)));
return { data(out_), data(out) };
}

314
matrix/dbs_room_type.cc Normal file
View file

@ -0,0 +1,314 @@
// The Construct
//
// Copyright (C) The Construct Developers, Authors & Contributors
// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::m::dbs
{
static bool room_type__cmp_lt(const string_view &, const string_view &);
}
decltype(ircd::m::dbs::room_type)
ircd::m::dbs::room_type;
decltype(ircd::m::dbs::desc::room_type__block__size)
ircd::m::dbs::desc::room_type__block__size
{
{ "name", "ircd.m.dbs._room_type.block.size" },
{ "default", 512L },
};
decltype(ircd::m::dbs::desc::room_type__meta_block__size)
ircd::m::dbs::desc::room_type__meta_block__size
{
{ "name", "ircd.m.dbs._room_type.meta_block.size" },
{ "default", 8192L },
};
decltype(ircd::m::dbs::desc::room_type__cache__size)
ircd::m::dbs::desc::room_type__cache__size
{
{
{ "name", "ircd.m.dbs._room_type.cache.size" },
{ "default", long(16_MiB) },
}, []
{
const size_t &value{room_type__cache__size};
db::capacity(db::cache(dbs::room_type), value);
}
};
decltype(ircd::m::dbs::desc::room_type__cache_comp__size)
ircd::m::dbs::desc::room_type__cache_comp__size
{
{
{ "name", "ircd.m.dbs._room_type.cache_comp.size" },
{ "default", long(8_MiB) },
}, []
{
const size_t &value{room_type__cache_comp__size};
db::capacity(db::cache_compressed(dbs::room_type), value);
}
};
/// Prefix transform for the room_type. The prefix here is a room_id
/// and the suffix is the type+depth+event_id concatenation.
/// for efficient sequences
///
const ircd::db::prefix_transform
ircd::m::dbs::desc::room_type__pfx
{
"_room_type",
[](const string_view &key)
{
return has(key, "\0"_sv);
},
[](const string_view &key)
{
return split(key, "\0"_sv).first;
}
};
/// Comparator for the room_type. The goal here is to sort the
/// events within a room by their depth from highest to lowest, so the
/// highest depth is hit first when a room is sought from this column.
///
const ircd::db::comparator
ircd::m::dbs::desc::room_type__cmp
{
"_room_type",
room_type__cmp_lt,
std::equal_to<string_view>{},
};
/// This column stores events by type in sequence in a room. Consider the
/// following:
///
/// [room_id | type, depth, event_idx]
///
const ircd::db::descriptor
ircd::m::dbs::desc::room_type
{
// name
"_room_type",
// explanation
R"(Indexes events per type in timeline sequence for a room
[room_id | type, depth, event_idx]
)",
// typing (key, value)
{
typeid(string_view), typeid(string_view)
},
// options
{},
// comparator
room_type__cmp,
// prefix transform
room_type__pfx,
// drop column
false,
// cache size
bool(cache_enable)? -1 : 0,
// cache size for compressed assets
bool(cache_comp_enable)? -1 : 0,
// bloom filter bits
0, // no bloom filter because of possible comparator issues
// expect queries hit
true,
// block size
size_t(room_type__block__size),
// meta_block size
size_t(room_type__meta_block__size),
};
//
// indexer
//
/// Adds the entry for the room_type column into the txn.
void
ircd::m::dbs::_index_room_type(db::txn &txn,
const event &event,
const write_opts &opts)
{
assert(opts.appendix.test(appendix::ROOM_TYPE));
thread_local char buf[ROOM_TYPE_KEY_MAX_SIZE];
const ctx::critical_assertion ca;
const string_view &key
{
room_type_key(buf, at<"room_id"_>(event), at<"type"_>(event), at<"depth"_>(event), opts.event_idx)
};
db::txn::append
{
txn, room_type,
{
opts.op, // db::op
key, // key,
}
};
}
//
// cmp
//
bool
ircd::m::dbs::room_type__cmp_lt(const string_view &a,
const string_view &b)
{
static const auto &pt
{
desc::room_type__pfx
};
// Extract the prefix from the keys
const string_view pre[2]
{
pt.get(a),
pt.get(b),
};
// Prefix size comparison has highest priority for rocksdb
if(size(pre[0]) < size(pre[1]))
return true;
// Prefix size comparison has highest priority for rocksdb
if(size(pre[0]) > size(pre[1]))
return false;
// Prefix lexical comparison sorts prefixes of the same size
if(pre[0] < pre[1])
return true;
// Prefix lexical comparison sorts prefixes of the same size
if(pre[0] > pre[1])
return false;
// After the prefix is the \0,type,\0,depth,event_idx
const string_view post[2]
{
a.substr(size(pre[0])),
b.substr(size(pre[1])),
};
// These conditions are matched on some queries when the user only
// supplies a room id.
if(empty(post[0]))
return true;
if(empty(post[1]))
return false;
const auto &[type_a, depth_a, event_idx_a]
{
room_type_key(post[0])
};
const auto &[type_b, depth_b, event_idx_b]
{
room_type_key(post[1])
};
if(type_a < type_b)
return true;
if(type_a > type_b)
return false;
// reverse depth to start from highest first like room_events
if(depth_a < depth_b)
return false;
// reverse depth to start from highest first like room_events
if(depth_a > depth_b)
return true;
// reverse event_idx to start from highest first like room_events)
if(event_idx_a < event_idx_b)
return false;
if(event_idx_a > event_idx_b)
return true;
// equal is not less; so false
return false;
}
//
// key
//
ircd::m::dbs::room_type_tuple
ircd::m::dbs::room_type_key(const string_view &amalgam_)
{
assert(size(amalgam_) >= 1 + 1 + 8 + 8);
assert(amalgam_.front() == '\0');
const string_view &amalgam
{
amalgam_.substr(1)
};
assert(amalgam.size() >= 1 + 8 + 1);
const auto &[type, trail]
{
split(amalgam, '\0')
};
assert(trail.size() >= 8 + 8);
return room_type_tuple
{
type,
likely(trail.size() >= 8)?
uint64_t(byte_view<uint64_t>(trail.substr(0, 8))):
-1UL,
likely(trail.size() >= 16)?
event::idx(byte_view<uint64_t>(trail.substr(8))):
0UL,
};
}
ircd::string_view
ircd::m::dbs::room_type_key(const mutable_buffer &out_,
const id::room &room_id,
const string_view &type,
const uint64_t &depth,
const event::idx &event_idx)
{
assert(room_id);
mutable_buffer out{out_};
consume(out, copy(out, room_id));
if(!type)
return { data(out_), data(out) };
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, type));
consume(out, copy(out, "\0"_sv));
consume(out, copy(out, byte_view<string_view>(depth)));
consume(out, copy(out, byte_view<string_view>(event_idx)));
return { data(out_), data(out) };
}

6
matrix/events.cc
View file

@ -486,7 +486,7 @@ ircd::m::events::type::for_each(const string_view &prefix,
const auto &prefixer
{
dbs::desc::events__event_type__pfx
dbs::desc::event_type__pfx
};
string_view last;
@ -570,7 +570,7 @@ ircd::m::events::origin::for_each(const string_view &prefix,
const auto &prefixer
{
dbs::desc::events__event_sender__pfx
dbs::desc::event_sender__pfx
};
if(unlikely(startswith(prefix, '@')))
@ -654,7 +654,7 @@ ircd::m::events::sender::for_each(const string_view &prefix_,
const auto &prefixer
{
dbs::desc::events__event_sender__pfx
dbs::desc::event_sender__pfx
};
// We MUST query the column with a key starting with '@' here. For a more