diff --git a/include/ircd/m/dbs.h b/include/ircd/m/dbs.h index 7dcfb3c91..213d155a3 100644 --- a/include/ircd/m/dbs.h +++ b/include/ircd/m/dbs.h @@ -13,27 +13,12 @@ namespace ircd::m::dbs { - struct init; - - extern std::map modules; - extern std::map> databases; - bool exists(const event::id &); void append_indexes(const event &, db::txn &); void write(const event &, db::txn &); } -class ircd::m::dbs::init -{ - void _modules(); - void _databases(); - - public: - init(); - ~init() noexcept; -}; - namespace ircd::m::dbs { using closure = std::function; diff --git a/include/ircd/m/event.h b/include/ircd/m/event.h index 595a56d73..3bf381a3f 100644 --- a/include/ircd/m/event.h +++ b/include/ircd/m/event.h @@ -10,8 +10,6 @@ #pragma once #define HAVE_IRCD_M_EVENT_H -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsubobject-linkage" namespace ircd::m { @@ -30,6 +28,8 @@ namespace ircd::m id::event event_id(const event &); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsubobject-linkage" /// The _Main Event_. Most fundamental primitive of the Matrix protocol. /// /// This json::tuple provides at least all of the legal members of the matrix @@ -65,6 +65,7 @@ struct ircd::m::event json::property > { + struct init; struct fetch; struct sync; struct prev; @@ -74,7 +75,9 @@ struct ircd::m::event using closure = std::function; using closure_bool = std::function; - static database *events; + static const db::database::description description; + static std::shared_ptr events; + static std::array column; using super_type::tuple; using super_type::operator=; @@ -82,6 +85,7 @@ struct ircd::m::event event(const id &, const mutable_buffer &buf); event() = default; }; +#pragma GCC diagnostic pop namespace ircd::m { @@ -93,6 +97,8 @@ namespace ircd::m std::string pretty_oneline(const event::prev &); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsubobject-linkage" struct ircd::m::event::prev :json::tuple < @@ -106,6 +112,13 @@ struct ircd::m::event::prev using super_type::tuple; using super_type::operator=; }; +#pragma GCC diagnostic pop + +struct ircd::m::event::init +{ + init(); + ~init() noexcept; +}; inline bool ircd::m::my(const event &event) @@ -118,5 +131,3 @@ ircd::m::my(const id::event &event_id) { return self::host(event_id.host()); } - -#pragma GCC diagnostic pop diff --git a/include/ircd/m/m.h b/include/ircd/m/m.h index fc35db179..b76ca9ac2 100644 --- a/include/ircd/m/m.h +++ b/include/ircd/m/m.h @@ -68,7 +68,7 @@ struct ircd::m::init void listeners(); void modules(); - dbs::init _dbs; + event::init _event; state::init _state; keys::init _keys; diff --git a/ircd/m/dbs.cc b/ircd/m/dbs.cc index 9e076f21e..0cc287204 100644 --- a/ircd/m/dbs.cc +++ b/ircd/m/dbs.cc @@ -40,63 +40,6 @@ struct ircd::m::indexer virtual ~indexer() noexcept = default; }; -decltype(ircd::m::dbs::databases) -ircd::m::dbs::databases -{}; - -decltype(ircd::m::dbs::modules) -ircd::m::dbs::modules -{}; - -// -// init -// - -ircd::m::dbs::init::init() -{ - _modules(); - _databases(); - - ircd::m::event::events = databases.at("events").get(); -} - -ircd::m::dbs::init::~init() -noexcept -{ - ircd::m::event::events = nullptr; - - databases.clear(); - modules.clear(); -} - -void -ircd::m::dbs::init::_databases() -{ - for(const auto &pair : modules) - { - const auto &name(pair.first); - const auto dbname(mods::unpostfixed(name)); - const std::string shortname(lstrip(dbname, "db_")); - const std::string symname(shortname + "_database"s); - databases.emplace(shortname, import_shared - { - dbname, symname - }); - } -} - -void -ircd::m::dbs::init::_modules() -{ - for(const auto &name : mods::available()) - if(startswith(name, "db_")) - modules.emplace(name, name); -} - -// -// -// - void ircd::m::dbs::write(const event &event, db::txn &txn) diff --git a/ircd/m/event.cc b/ircd/m/event.cc index 1d7b99298..0798d489a 100644 --- a/ircd/m/event.cc +++ b/ircd/m/event.cc @@ -10,10 +10,48 @@ #include -ircd::database * +decltype(ircd::m::event::column) +ircd::m::event::column +{}; + +decltype(ircd::m::event::events) ircd::m::event::events {}; +// +// init +// + +ircd::m::event::init::init() +{ + // Open the events database + event::events = std::make_shared("events"s, ""s, event::description); + + // Cache the columns for the event tuple in order for constant time lookup + std::array keys; //TODO: why did this happen? + _key_transform(event{}, begin(keys), end(keys)); //TODO: how did this happen? + for(size_t i(0); i < keys.size(); ++i) + event::column[i] = db::column + { + *event::events, keys[i] + }; +} + +ircd::m::event::init::~init() +noexcept +{ + // Columns have to be closed before DB closes + for(auto &column : event::column) + column = {}; + + // Close DB (if no other ref) + event::events = {}; +} + +// +// misc +// + ircd::m::id::event ircd::m::event_id(const event &event, id::event::buf &buf) @@ -319,12 +357,849 @@ ircd::m::pretty_oneline(const event &event) ircd::m::event::event(const id &id, const mutable_buffer &buf) { -/* - fetch tab + assert(events); + + db::gopts opts; + opts.snapshot = database::snapshot{*events}; + for(size_t i(0); i < column.size(); ++i) { - id, buf + const db::cell cell + { + column[i], id, opts + }; + + db::assign(*this, cell, id); + } + + const json::object obj + { + string_view{data(buf), json::print(buf, *this)} }; - new (this) event{tab}; -*/ + new (this) m::event(obj); } + +ircd::m::event::event(fetch &tab) +{ + io::acquire(tab); +} + +// +// Database descriptors +// + +const ircd::database::descriptor +events_event_id_descriptor +{ + // name + "event_id", + + // explanation + R"(### protocol note: + + 10.1 + The id of event. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_id. This is redundant data but we have to have it for now. + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_type_descriptor +{ + // name + "type", + + // explanation + R"(### protocol note: + + 10.1 + The type of event. This SHOULD be namespaced similar to Java package naming conventions + e.g. 'com.example.subdomain.event.type'. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_id + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_content_descriptor +{ + // name + "content", + + // explanation + R"(### protocol note: + + 10.1 + The fields in this object will vary depending on the type of event. When interacting + with the REST API, this is the HTTP body. + + ### developer note: + Since events must not exceed 65 KB the maximum size for the content is the remaining + space after all the other fields for the event are rendered. + + key is event_id + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_room_id_descriptor +{ + // name + "room_id", + + // explanation + R"(### protocol note: + + 10.2 (apropos room events) + Required. The ID of the room associated with this event. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_id + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_sender_descriptor +{ + // name + "sender", + + // explanation + R"(### protocol note: + + 10.2 (apropos room events) + Required. Contains the fully-qualified ID of the user who sent this event. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_id + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_state_key_descriptor +{ + // name + "state_key", + + // explanation + R"(### protocol note: + + 10.3 (apropos room state events) + A unique key which defines the overwriting semantics for this piece of room state. + This value is often a zero-length string. The presence of this key makes this event a + State Event. The key MUST NOT start with '_'. + + 10.4 + MUST NOT exceed 255 bytes. + + ### developer note: + key is event_id + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_origin_descriptor +{ + // name + "origin", + + // explanation + R"(### protocol note: + + FEDERATION 4.1 + DNS name of homeserver that created this PDU + + ### developer note: + key is event_id + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_origin_server_ts_descriptor +{ + // name + "origin_server_ts", + + // explanation + R"(### protocol note: + + FEDERATION 4.1 + Timestamp in milliseconds on origin homeserver when this PDU was created. + + ### developer note: + key is event_id + value is a machine integer (binary) + + TODO: consider unsigned rather than time_t because of millisecond precision + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(time_t) + } +}; + +const ircd::database::descriptor +events_unsigned_descriptor +{ + // name + "unsigned", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_signatures_descriptor +{ + // name + "signatures", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_auth_events_descriptor +{ + // name + "auth_events", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id. + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_depth_descriptor +{ + // name + "depth", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id value is long integer + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(int64_t) + } +}; + +const ircd::database::descriptor +events_hashes_descriptor +{ + // name + "hashes", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id. + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_membership_descriptor +{ + // name + "membership", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id. + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_prev_events_descriptor +{ + // name + "prev_events", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id. + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +const ircd::database::descriptor +events_prev_state_descriptor +{ + // name + "prev_state", + + // explanation + R"(### protocol note: + + ### developer note: + key is event_id. + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + } +}; + +/// prefix transform for event_id suffixes +/// +/// This transform expects a concatenation ending with an event_id which means +/// the prefix can be the same for multiple event_id's; therefor we can find +/// or iterate "event_id in X" where X is some key like a room_id +/// +const ircd::db::prefix_transform +event_id_in +{ + "event_id in", + [](const ircd::string_view &key) + { + return key.find('$') != key.npos; + }, + [](const ircd::string_view &key) + { + return rsplit(key, '$').first; + } +}; + +const ircd::database::descriptor +event_id_in_sender +{ + // name + "event_id in sender", + + // explanation + R"(### developer note: + + key is "@sender$event_id" + the prefix transform is in effect. this column indexes events by + sender offering an iterable bound of the index prefixed by sender + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + event_id_in, +}; + +const ircd::database::descriptor +state_head_for_event_id_in_room_id +{ + // name + "state_head for event_id in room_id", + + // explanation + R"(### developer note: + + key is "!room_id$event_id" + the prefix transform is in effect. this column indexes events by + room_id offering an iterable bound of the index prefixed by room_id + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator - sorts from highest to lowest + {}, //ircd::db::reverse_cmp_ircd::string_view{}, + + // prefix transform + event_id_in, +}; + +/// prefix transform for origin in +/// +/// This transform expects a concatenation ending with an origin which means +/// the prefix can be the same for multiple origins; therefor we can find +/// or iterate "origin in X" where X is some repeated prefix +/// +/// TODO: strings will have character conflicts. must address +const ircd::db::prefix_transform +origin_in +{ + "origin in", + [](const ircd::string_view &key) + { + return has(key, ":::"); + //return key.find(':') != key.npos; + }, + [](const ircd::string_view &key) + { + return split(key, ":::").first; + //return rsplit(key, ':').first; + } +}; + +const ircd::database::descriptor +origin_in_room_id +{ + // name + "origin in room_id", + + // explanation + R"(### developer note: + + key is "!room_id:origin" + the prefix transform is in effect. this column indexes origins in a + room_id offering an iterable bound of the index prefixed by room_id + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator - sorts from highest to lowest + {}, //ircd::db::reverse_cmp_string_view{}, + + // prefix transform + origin_in, +}; + +const ircd::database::descriptor +origin_joined_in_room_id +{ + // name + "origin_joined in room_id", + + // explanation + R"(### developer note: + + key is "!room_id:origin" + the prefix transform is in effect. this column indexes origins in a + room_id offering an iterable bound of the index prefixed by room_id + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator - sorts from highest to lowest + {}, //ircd::db::reverse_cmp_string_view{}, + + // prefix transform + origin_in, +}; + +/// prefix transform for room_id +/// +/// This transform expects a concatenation ending with a room_id which means +/// the prefix can be the same for multiple room_id's; therefor we can find +/// or iterate "room_id in X" where X is some repeated prefix +/// +const ircd::db::prefix_transform room_id_in +{ + "room_id in", + [](const ircd::string_view &key) + { + return key.find('!') != key.npos; + }, + [](const ircd::string_view &key) + { + return rsplit(key, '!').first; + } +}; + +/// prefix transform for type,state_key in room_id +/// +/// This transform is special for concatenating room_id with type and state_key +/// in that order with prefix being the room_id (this may change to room_id+ +/// type +/// +/// TODO: arbitrary type strings will have character conflicts. must address +/// TODO: with grammars. +const ircd::db::prefix_transform type_state_key_in_room_id +{ + "type,state_key in room_id", + [](const ircd::string_view &key) + { + return key.find("..") != key.npos; + }, + [](const ircd::string_view &key) + { + return split(key, "..").first; + } +}; + +const ircd::database::descriptor +event_id_for_type_state_key_in_room_id +{ + // name + "event_id for type,state_key in room_id", + + // explanation + R"(### developer note: + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + type_state_key_in_room_id +}; + +const ircd::database::descriptor +prev_event_id_for_event_id_in_room_id +{ + // name + "prev_event_id for event_id in room_id", + + // explanation + R"(### developer note: + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + event_id_in +}; + +/// prefix transform for event_id in room_id,type,state_key +/// +/// This transform is special for concatenating room_id with type and state_key +/// and event_id in that order with prefix being the room_id,type,state_key. This +/// will index multiple event_ids with the same type,state_key in a room which +/// allows for a temporal depth to the database; event_id for type,state_key only +/// resolves to a single latest event and overwrites itself as per the room state +/// algorithm whereas this can map all of them and then allows for tracing. +/// +/// TODO: arbitrary type strings will have character conflicts. must address +/// TODO: with grammars. +const ircd::db::prefix_transform +event_id_in_room_id_type_state_key +{ + "event_id in room_id,type_state_key", + [](const ircd::string_view &key) + { + return has(key, '$'); + }, + [](const ircd::string_view &key) + { + return split(key, '$').first; + } +}; + +const ircd::database::descriptor +prev_event_id_for_type_state_key_event_id_in_room_id +{ + // name + "prev_event_id for type,state_key,event_id in room_id", + + // explanation + R"(### developer note: + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + event_id_in_room_id_type_state_key +}; + +const ircd::database::descriptor +state_head +{ + // name + "state_head", + + // explanation + R"(### developer note: + + key is "!room_id" + value is the key of a state_node + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, +}; + +const ircd::database::descriptor +state_node +{ + // name + "state_node", + + // explanation + R"(### developer note: + + )", + + // typing (key, value) + { + typeid(ircd::string_view), typeid(ircd::string_view) + }, + + // options + {}, + + // comparator + {}, + + // prefix transform + {}, +}; + +decltype(ircd::m::event::description) +ircd::m::event::description +{ + { "default" }, + + //////// + // + // These columns directly represent event fields indexed by event_id and + // the value is the actual event values. Some values may be JSON, like + // content. + // + events_event_id_descriptor, + events_type_descriptor, + events_content_descriptor, + events_room_id_descriptor, + events_sender_descriptor, + events_state_key_descriptor, + events_origin_descriptor, + events_origin_server_ts_descriptor, + events_unsigned_descriptor, + events_signatures_descriptor, + events_auth_events_descriptor, + events_depth_descriptor, + events_hashes_descriptor, + events_membership_descriptor, + events_prev_events_descriptor, + events_prev_state_descriptor, + + //////// + // + // These columns are metadata composed from the event data. Specifically, + // they are designed for fast sequential iterations. + // + + // (sender, event_id) => () + // Sequence of all events in all rooms for a sender, EVER + // * broad but useful in cases + event_id_in_sender, + + // (room_id, event_id) => (state_head) + // Sequence of all events for a room, EVER + // * broad but useful in cases + // ? eliminate for prev_event? + // ? eliminate/combine with state tree related? + state_head_for_event_id_in_room_id, + + // (room_id, origin) => () + // Sequence of all origins for a room, EVER + //TODO: value should have [JOIN, LEAVE, ...) counts/data + //TODO: remove? + origin_in_room_id, + + // (room_id, origin) => () + // Sequence of all origins with joined member for a room, AT PRESENT + // * Intended to be a fast sequential iteration for sending out messages. + origin_joined_in_room_id, + + // (room_id, type, state_key) => (event_id) + // Sequence of events of type+state_key in a room, AT PRESENT + // * Fast for current room state iteration, but only works for the present. + event_id_for_type_state_key_in_room_id, + + //////// + // + // These columns are metadata composed from the event data. They are + // linked forward lists where the value is used to lookup the next key + // TODO: these might be better as sequences; if not removed altogether. + // + + // (room_id, event_id) => (prev event_id) + // List of events in a room resolving to the previous event in a room + // in our subjective euclidean tape TOTAL order. + // * This is where any branches in the DAG are linearized based on how we + // feel the state machine should execute them one by one. + // * This is not a sequence; each value is the key for another lookup. + prev_event_id_for_event_id_in_room_id, + + // (room_id, type, state_key, event_id) => (prev event_id) + // Events of a (type, state_key) in a room resolving to the previous event + // of (type, state_key) in a room in our subjective euclidean tape order. + // * Similar to the above but focuses only on state events for various + // "state chains" + prev_event_id_for_type_state_key_event_id_in_room_id, + + //////// + // + // These columns are metadata composed from the event data. They are + // used to create structures that can represent the state of a room + // at any given event. + // + + // (room_id) => (state_head) + state_head, + + // (state tree node id) => (state tree node) + // + state_node, +}; diff --git a/modules/Makefile.am b/modules/Makefile.am index 435689145..532ed4180 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -41,14 +41,6 @@ module_LTLIBRARIES = \ root.la \ ### -# This puts the source in db/ but the installed -# library is db_X.so in the main modules dir. -db_moduledir = @moduledir@ -db_db_events_la_SOURCES = db/events.cc -db_module_LTLIBRARIES = \ - db/db_events.la \ - ### - # This puts the source in client/ but the installed # library is client_X.so in the main modules dir. client_moduledir = @moduledir@ diff --git a/modules/db/events.cc b/modules/db/events.cc deleted file mode 100644 index f8bcdd026..000000000 --- a/modules/db/events.cc +++ /dev/null @@ -1,808 +0,0 @@ -// Matrix Construct -// -// Copyright (C) Matrix Construct Developers, Authors & Contributors -// Copyright (C) 2016-2018 Jason Volk -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice is present in all copies. The -// full license for this software is available in the LICENSE file. - -using namespace ircd; - -const database::descriptor events_event_id_descriptor -{ - // name - "event_id", - - // explanation - R"(### protocol note: - - 10.1 - The id of event. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_id. This is redundant data but we have to have it for now. - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_type_descriptor -{ - // name - "type", - - // explanation - R"(### protocol note: - - 10.1 - The type of event. This SHOULD be namespaced similar to Java package naming conventions - e.g. 'com.example.subdomain.event.type'. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_id - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_content_descriptor -{ - // name - "content", - - // explanation - R"(### protocol note: - - 10.1 - The fields in this object will vary depending on the type of event. When interacting - with the REST API, this is the HTTP body. - - ### developer note: - Since events must not exceed 65 KB the maximum size for the content is the remaining - space after all the other fields for the event are rendered. - - key is event_id - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_room_id_descriptor -{ - // name - "room_id", - - // explanation - R"(### protocol note: - - 10.2 (apropos room events) - Required. The ID of the room associated with this event. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_id - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_sender_descriptor -{ - // name - "sender", - - // explanation - R"(### protocol note: - - 10.2 (apropos room events) - Required. Contains the fully-qualified ID of the user who sent this event. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_id - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_state_key_descriptor -{ - // name - "state_key", - - // explanation - R"(### protocol note: - - 10.3 (apropos room state events) - A unique key which defines the overwriting semantics for this piece of room state. - This value is often a zero-length string. The presence of this key makes this event a - State Event. The key MUST NOT start with '_'. - - 10.4 - MUST NOT exceed 255 bytes. - - ### developer note: - key is event_id - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_origin_descriptor -{ - // name - "origin", - - // explanation - R"(### protocol note: - - FEDERATION 4.1 - DNS name of homeserver that created this PDU - - ### developer note: - key is event_id - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_origin_server_ts_descriptor -{ - // name - "origin_server_ts", - - // explanation - R"(### protocol note: - - FEDERATION 4.1 - Timestamp in milliseconds on origin homeserver when this PDU was created. - - ### developer note: - key is event_id - value is a machine integer (binary) - - TODO: consider unsigned rather than time_t because of millisecond precision - - )", - - // typing (key, value) - { - typeid(string_view), typeid(time_t) - } -}; - -const database::descriptor events_unsigned_descriptor -{ - // name - "unsigned", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_signatures_descriptor -{ - // name - "signatures", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_auth_events_descriptor -{ - // name - "auth_events", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id. - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_depth_descriptor -{ - // name - "depth", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id value is long integer - )", - - // typing (key, value) - { - typeid(string_view), typeid(int64_t) - } -}; - -const database::descriptor events_hashes_descriptor -{ - // name - "hashes", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id. - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_membership_descriptor -{ - // name - "membership", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id. - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_prev_events_descriptor -{ - // name - "prev_events", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id. - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -const database::descriptor events_prev_state_descriptor -{ - // name - "prev_state", - - // explanation - R"(### protocol note: - - ### developer note: - key is event_id. - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - } -}; - -/// prefix transform for event_id suffixes -/// -/// This transform expects a concatenation ending with an event_id which means -/// the prefix can be the same for multiple event_id's; therefor we can find -/// or iterate "event_id in X" where X is some key like a room_id -/// -const ircd::db::prefix_transform event_id_in -{ - "event_id in", - [](const string_view &key) - { - return key.find('$') != key.npos; - }, - [](const string_view &key) - { - return rsplit(key, '$').first; - } -}; - -const database::descriptor event_id_in_sender -{ - // name - "event_id in sender", - - // explanation - R"(### developer note: - - key is "@sender$event_id" - the prefix transform is in effect. this column indexes events by - sender offering an iterable bound of the index prefixed by sender - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - event_id_in, -}; - -const database::descriptor state_head_for_event_id_in_room_id -{ - // name - "state_head for event_id in room_id", - - // explanation - R"(### developer note: - - key is "!room_id$event_id" - the prefix transform is in effect. this column indexes events by - room_id offering an iterable bound of the index prefixed by room_id - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - sorts from highest to lowest - {}, //ircd::db::reverse_cmp_string_view{}, - - // prefix transform - event_id_in, -}; - -/// prefix transform for origin in -/// -/// This transform expects a concatenation ending with an origin which means -/// the prefix can be the same for multiple origins; therefor we can find -/// or iterate "origin in X" where X is some repeated prefix -/// -/// TODO: strings will have character conflicts. must address -const ircd::db::prefix_transform origin_in -{ - "origin in", - [](const string_view &key) - { - return has(key, ":::"); - //return key.find(':') != key.npos; - }, - [](const string_view &key) - { - return split(key, ":::").first; - //return rsplit(key, ':').first; - } -}; - -const database::descriptor origin_in_room_id -{ - // name - "origin in room_id", - - // explanation - R"(### developer note: - - key is "!room_id:origin" - the prefix transform is in effect. this column indexes origins in a - room_id offering an iterable bound of the index prefixed by room_id - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - sorts from highest to lowest - {}, //ircd::db::reverse_cmp_string_view{}, - - // prefix transform - origin_in, -}; - -const database::descriptor origin_joined_in_room_id -{ - // name - "origin_joined in room_id", - - // explanation - R"(### developer note: - - key is "!room_id:origin" - the prefix transform is in effect. this column indexes origins in a - room_id offering an iterable bound of the index prefixed by room_id - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - sorts from highest to lowest - {}, //ircd::db::reverse_cmp_string_view{}, - - // prefix transform - origin_in, -}; - -/// prefix transform for room_id -/// -/// This transform expects a concatenation ending with a room_id which means -/// the prefix can be the same for multiple room_id's; therefor we can find -/// or iterate "room_id in X" where X is some repeated prefix -/// -const ircd::db::prefix_transform room_id_in -{ - "room_id in", - [](const string_view &key) - { - return key.find('!') != key.npos; - }, - [](const string_view &key) - { - return rsplit(key, '!').first; - } -}; - -/// prefix transform for type,state_key in room_id -/// -/// This transform is special for concatenating room_id with type and state_key -/// in that order with prefix being the room_id (this may change to room_id+ -/// type -/// -/// TODO: arbitrary type strings will have character conflicts. must address -/// TODO: with grammars. -const ircd::db::prefix_transform type_state_key_in_room_id -{ - "type,state_key in room_id", - [](const string_view &key) - { - return key.find("..") != key.npos; - }, - [](const string_view &key) - { - return split(key, "..").first; - } -}; - -const database::descriptor event_id_for_type_state_key_in_room_id -{ - // name - "event_id for type,state_key in room_id", - - // explanation - R"(### developer note: - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - type_state_key_in_room_id -}; - -const database::descriptor prev_event_id_for_event_id_in_room_id -{ - // name - "prev_event_id for event_id in room_id", - - // explanation - R"(### developer note: - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - event_id_in -}; - -/// prefix transform for event_id in room_id,type,state_key -/// -/// This transform is special for concatenating room_id with type and state_key -/// and event_id in that order with prefix being the room_id,type,state_key. This -/// will index multiple event_ids with the same type,state_key in a room which -/// allows for a temporal depth to the database; event_id for type,state_key only -/// resolves to a single latest event and overwrites itself as per the room state -/// algorithm whereas this can map all of them and then allows for tracing. -/// -/// TODO: arbitrary type strings will have character conflicts. must address -/// TODO: with grammars. -const ircd::db::prefix_transform event_id_in_room_id_type_state_key -{ - "event_id in room_id,type_state_key", - [](const string_view &key) - { - return has(key, '$'); - }, - [](const string_view &key) - { - return split(key, '$').first; - } -}; - -const database::descriptor prev_event_id_for_type_state_key_event_id_in_room_id -{ - // name - "prev_event_id for type,state_key,event_id in room_id", - - // explanation - R"(### developer note: - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - event_id_in_room_id_type_state_key -}; - -const database::descriptor state_head -{ - // name - "state_head", - - // explanation - R"(### developer note: - - key is "!room_id" - value is the key of a state_node - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, -}; - -const database::descriptor state_node -{ - // name - "state_node", - - // explanation - R"(### developer note: - - )", - - // typing (key, value) - { - typeid(string_view), typeid(string_view) - }, - - // options - {}, - - // comparator - {}, - - // prefix transform - {}, -}; - -const database::description events_description -{ - { "default" }, - - //////// - // - // These columns directly represent event fields indexed by event_id and - // the value is the actual event values. Some values may be JSON, like - // content. - // - events_event_id_descriptor, - events_type_descriptor, - events_content_descriptor, - events_room_id_descriptor, - events_sender_descriptor, - events_state_key_descriptor, - events_origin_descriptor, - events_origin_server_ts_descriptor, - events_unsigned_descriptor, - events_signatures_descriptor, - events_auth_events_descriptor, - events_depth_descriptor, - events_hashes_descriptor, - events_membership_descriptor, - events_prev_events_descriptor, - events_prev_state_descriptor, - - //////// - // - // These columns are metadata composed from the event data. Specifically, - // they are designed for fast sequential iterations. - // - - // (sender, event_id) => () - // Sequence of all events in all rooms for a sender, EVER - // * broad but useful in cases - event_id_in_sender, - - // (room_id, event_id) => (state_head) - // Sequence of all events for a room, EVER - // * broad but useful in cases - // ? eliminate for prev_event? - // ? eliminate/combine with state tree related? - state_head_for_event_id_in_room_id, - - // (room_id, origin) => () - // Sequence of all origins for a room, EVER - //TODO: value should have [JOIN, LEAVE, ...) counts/data - //TODO: remove? - origin_in_room_id, - - // (room_id, origin) => () - // Sequence of all origins with joined member for a room, AT PRESENT - // * Intended to be a fast sequential iteration for sending out messages. - origin_joined_in_room_id, - - // (room_id, type, state_key) => (event_id) - // Sequence of events of type+state_key in a room, AT PRESENT - // * Fast for current room state iteration, but only works for the present. - event_id_for_type_state_key_in_room_id, - - //////// - // - // These columns are metadata composed from the event data. They are - // linked forward lists where the value is used to lookup the next key - // TODO: these might be better as sequences; if not removed altogether. - // - - // (room_id, event_id) => (prev event_id) - // List of events in a room resolving to the previous event in a room - // in our subjective euclidean tape TOTAL order. - // * This is where any branches in the DAG are linearized based on how we - // feel the state machine should execute them one by one. - // * This is not a sequence; each value is the key for another lookup. - prev_event_id_for_event_id_in_room_id, - - // (room_id, type, state_key, event_id) => (prev event_id) - // Events of a (type, state_key) in a room resolving to the previous event - // of (type, state_key) in a room in our subjective euclidean tape order. - // * Similar to the above but focuses only on state events for various - // "state chains" - prev_event_id_for_type_state_key_event_id_in_room_id, - - //////// - // - // These columns are metadata composed from the event data. They are - // used to create structures that can represent the state of a room - // at any given event. - // - - // (room_id) => (state_head) - state_head, - - // (state tree node id) => (state tree node) - // - state_node, -}; - -std::shared_ptr events_database -{ - std::make_shared("events"s, ""s, events_description) -}; - -mapi::header IRCD_MODULE -{ - "Hosts the 'events' database" -};