ircd:Ⓜ️:dbs: Reorg the indexer conditions.

include/ircd/m/dbs.h

@@ -94,10 +94,16 @@ namespace ircd::m::dbs
 	void blacklist(db::txn &, const event::id &, const write_opts &);
 }
 
+namespace ircd::m::dbs::appendix
+{
+	enum index :uint8_t;
+}
+
 /// Options that affect the dbs::write() of an event to the transaction.
 struct ircd::m::dbs::write_opts
 {
 	static const std::bitset<256> event_refs_all;
+	static const std::bitset<64> appendix_all;
 
 	/// Operation code
 	db::op op {db::op::SET};
@@ -105,8 +111,8 @@ struct ircd::m::dbs::write_opts
 	/// Principal's index number. Most codepaths do not permit zero; must set.
 	uint64_t event_idx {0};
 
-	/// Whether the event_id should be indexed in event_idx (you want yes).
+	/// Fuse panel to toggle transaction elements.
-	bool event_id {true};
+	std::bitset<64> appendix {appendix_all};
 
 	/// Whether the event.source can be used directly for event_json. Defaults
 	/// to false unless the caller wants to avoid a redundant re-stringify.
@@ -137,21 +143,71 @@ struct ircd::m::dbs::write_opts
 	/// After the update is performed, the new state btree root is returned
 	/// into this buffer.
 	mutable_buffer root_out;
+};
+
+enum ircd::m::dbs::appendix::index
+:std::underlying_type<ircd::m::dbs::appendix::index>::type
+{
+	/// This bit offers coarse control over all the EVENT_ appendices.
+	EVENT,
+
+	/// Involves the event_idx column; translates an event_id to our internal
+	/// index number. This bit can be dark during re-indexing operations.
+	EVENT_ID,
+
+	/// Involves the event_json column; writes a full JSON serialization
+	/// of the event. See the `json_source` write_opts option. This bit can be
+	/// dark during re-indexing operations to avoid rewriting the same data.
+	EVENT_JSON,
+
+	/// Involves any direct event columns; such columns are forward-indexed
+	/// values from the original event data but split into columns for each
+	/// property. Can be dark during re-indexing similar to EVENT_JSON.
+	EVENT_COLS,
+
+	/// Take branch to handle event reference graphing. A separate bitset is
+	/// offered in write_opts for fine-grained control over which reference
+	/// types are involved.
+	EVENT_REFS,
+
+	/// Involves the event_sender column (reverse index on the event sender).
+	EVENT_SENDER,
+
+	/// Involves the event_type column (reverse index on the event type).
+	EVENT_TYPE,
+
+	/// Take branch to handle events with a room_id
+	ROOM,
+
+	/// Take branch to handle room state events.
+	STATE,
+
+	/// Perform state btree manip for room history.
+	HISTORY,
+
+	/// Take branch to handle room redaction events.
+	REDACT,
+
+	/// Take branch to handle other types of events.
+	OTHER,
 
 	/// Whether the event should be added to the room_head, indicating that
 	/// it has not yet been referenced at the time of this write. Defaults
 	/// to true, but if this is an older event this opt should be rethought.
-	bool room_head {true};
+	ROOM_HEAD,
 
 	/// Whether the event removes the prev_events it references from the
 	/// room_head. This defaults to true and should almost always be true.
-	bool room_refs {true};
+	ROOM_HEAD_REFS,
 
-	/// Whether to update the event_sender index.
+	/// Involves room_events table.
-	bool event_sender {true};
+	ROOM_EVENTS,
 
-	/// Whether to update the event_type index.
+	/// Involves room_joined table.
-	bool event_type {true};
+	ROOM_JOINED,
+
+	/// Involves room_state (present state) table.
+	ROOM_STATE,
 };
 
 /// Types of references indexed by event_refs. This is a single byte integer,
@@ -332,6 +388,7 @@ namespace ircd::m::dbs
 	void _index__room_state(db::txn &,  const event &, const write_opts &);
 	void _index__room_events(db::txn &,  const event &, const write_opts &, const string_view &);
 	void _index__room_joined(db::txn &, const event &, const write_opts &);
+	void _index__room_head_refs(db::txn &, const event &, const write_opts &);
 	void _index__room_head(db::txn &, const event &, const write_opts &);
 	string_view _index_state(db::txn &, const event &, const write_opts &);
 	string_view _index_redact(db::txn &, const event &, const write_opts &);
@@ -346,10 +403,10 @@ namespace ircd::m::dbs
 	void _index_event_refs_auth(db::txn &, const event &, const write_opts &);
 	void _index_event_refs_prev(db::txn &, const event &, const write_opts &);
 	void _index_event_refs(db::txn &, const event &, const write_opts &);
+	void _index_event_json(db::txn &, const event &, const write_opts &);
+	void _index_event_cols(db::txn &, const event &, const write_opts &);
 	void _index_event_id(db::txn &, const event &, const write_opts &);
 	void _index_event(db::txn &, const event &, const write_opts &);
-	void _append_json(db::txn &, const event &, const write_opts &);
-	void _append_cols(db::txn &, const event &, const write_opts &);
 }
 
 struct ircd::m::dbs::init

ircd/m_dbs.cc

@@ -183,6 +183,17 @@ ircd::m::dbs::write_opts::event_refs_all{[]
 	};
 }()};
 
+decltype(ircd::m::dbs::write_opts::appendix_all)
+ircd::m::dbs::write_opts::appendix_all{[]
+{
+	char full[appendix_all.size()];
+	memset(full, '1', sizeof(full));
+	return decltype(appendix_all)
+	{
+		full, sizeof(full)
+	};
+}()};
+
 //
 // Basic write suite
 //
@@ -226,14 +237,10 @@ ircd::m::dbs::write(db::txn &txn,
 			"Cannot write to database: no index specified for event."
 		};
 
+	if(opts.appendix.test(appendix::EVENT))
 		_index_event(txn, event, opts);
 
-	// direct columns
+	if(opts.appendix.test(appendix::ROOM))
-	_append_cols(txn, event, opts);
-
-	// full json
-	_append_json(txn, event, opts);
-
 		if(json::get<"room_id"_>(event))
 			return _index_room(txn, event, opts);
 
@@ -245,10 +252,55 @@ ircd::m::dbs::write(db::txn &txn,
 //
 
 void
-ircd::m::dbs::_append_cols(db::txn &txn,
+ircd::m::dbs::_index_event(db::txn &txn,
                            const event &event,
                            const write_opts &opts)
 {
+	if(opts.appendix.test(appendix::EVENT_ID))
+		_index_event_id(txn, event, opts);
+
+	if(opts.appendix.test(appendix::EVENT_COLS))
+		_index_event_cols(txn, event, opts);
+
+	if(opts.appendix.test(appendix::EVENT_JSON))
+		_index_event_json(txn, event, opts);
+
+	if(opts.appendix.test(appendix::EVENT_REFS) && opts.event_refs.any())
+		_index_event_refs(txn, event, opts);
+
+	if(opts.appendix.test(appendix::EVENT_SENDER))
+		_index_event_sender(txn, event, opts);
+
+	if(opts.appendix.test(appendix::EVENT_TYPE))
+		_index_event_type(txn, event, opts);
+}
+
+void
+ircd::m::dbs::_index_event_id(db::txn &txn,
+                              const event &event,
+                              const write_opts &opts)
+{
+	assert(opts.appendix.test(appendix::EVENT_ID));
+	assert(opts.event_idx);
+
+	db::txn::append
+	{
+		txn, dbs::event_idx,
+		{
+			opts.op,
+			at<"event_id"_>(event),
+			byte_view<string_view>(opts.event_idx)
+		}
+	};
+}
+
+void
+ircd::m::dbs::_index_event_cols(db::txn &txn,
+                                const event &event,
+                                const write_opts &opts)
+{
+	assert(opts.appendix.test(appendix::EVENT_COLS));
+	assert(opts.event_idx);
 	const byte_view<string_view> key
 	{
 		opts.event_idx
@@ -284,12 +336,14 @@ ircd::m::dbs::_append_cols(db::txn &txn,
 }
 
 void
-ircd::m::dbs::_append_json(db::txn &txn,
+ircd::m::dbs::_index_event_json(db::txn &txn,
                                 const event &event,
                                 const write_opts &opts)
 {
 	const ctx::critical_assertion ca;
 	thread_local char buf[m::event::MAX_SIZE];
+	assert(opts.appendix.test(appendix::EVENT_JSON));
+	assert(opts.event_idx);
 
 	const string_view &key
 	{
@@ -330,46 +384,13 @@ ircd::m::dbs::_append_json(db::txn &txn,
 	};
 }
 
-void
-ircd::m::dbs::_index_event(db::txn &txn,
-                           const event &event,
-                           const write_opts &opts)
-{
-	// event_id
-	if(opts.event_id)
-		_index_event_id(txn, event, opts);
-
-	if(opts.event_refs.any())
-		_index_event_refs(txn, event, opts);
-
-	if(opts.event_sender)
-		_index_event_sender(txn, event, opts);
-
-	if(opts.event_type)
-		_index_event_type(txn, event, opts);
-}
-
-void
-ircd::m::dbs::_index_event_id(db::txn &txn,
-                              const event &event,
-                              const write_opts &opts)
-{
-	db::txn::append
-	{
-		txn, dbs::event_idx,
-		{
-			opts.op,
-			at<"event_id"_>(event),
-			byte_view<string_view>(opts.event_idx)
-		}
-	};
-}
-
 void
 ircd::m::dbs::_index_event_refs(db::txn &txn,
                                 const event &event,
                                 const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::EVENT_REFS));
+
 	if(opts.event_refs.test(uint(ref::PREV)))
 		_index_event_refs_prev(txn, event, opts);
 
@@ -395,6 +416,7 @@ ircd::m::dbs::_index_event_refs_prev(db::txn &txn,
                                      const event &event,
                                      const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::EVENT_REFS));
 	assert(opts.event_refs.test(uint(ref::PREV)));
 
 	const event::prev &prev{event};
@@ -445,6 +467,7 @@ ircd::m::dbs::_index_event_refs_auth(db::txn &txn,
                                      const event &event,
                                      const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::EVENT_REFS));
 	assert(opts.event_refs.test(uint(ref::AUTH)));
 	if(!event::auth::is_power_event(event))
 		return;
@@ -497,6 +520,7 @@ ircd::m::dbs::_index_event_refs_state(db::txn &txn,
                                       const event &event,
                                       const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::EVENT_REFS));
 	assert(opts.event_refs.test(uint(ref::STATE)) ||
 	       opts.event_refs.test(uint(ref::PREV_STATE)));
 
@@ -569,6 +593,7 @@ ircd::m::dbs::_index_event_refs_m_receipt_m_read(db::txn &txn,
                                                  const event &event,
                                                  const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::EVENT_REFS));
 	assert(opts.event_refs.test(uint(ref::M_RECEIPT__M_READ)));
 
 	if(json::get<"type"_>(event) != "ircd.read")
@@ -622,6 +647,7 @@ ircd::m::dbs::_index_event_refs_m_relates_m_reply(db::txn &txn,
                                                   const event &event,
                                                   const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::EVENT_REFS));
 	assert(opts.event_refs.test(uint(ref::M_RELATES__M_REPLY)));
 
 	if(json::get<"type"_>(event) != "m.room.message")
@@ -713,6 +739,7 @@ ircd::m::dbs::_index_event_refs_m_room_redaction(db::txn &txn,
                                                  const event &event,
                                                  const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::EVENT_REFS));
 	assert(opts.event_refs.test(uint(ref::M_ROOM_REDACTION)));
 
 	if(json::get<"type"_>(event) != "m.room.redaction")
@@ -756,9 +783,9 @@ ircd::m::dbs::_index_event_sender(db::txn &txn,
                                   const event &event,
                                   const write_opts &opts)
 {
-	assert(opts.event_sender);
+	assert(opts.appendix.test(appendix::EVENT_SENDER));
-	assert(opts.event_idx);
 	assert(json::get<"sender"_>(event));
+	assert(opts.event_idx);
 
 	thread_local char buf[EVENT_SENDER_KEY_MAX_SIZE];
 	const string_view &key
@@ -780,9 +807,9 @@ ircd::m::dbs::_index_event_type(db::txn &txn,
                                 const event &event,
                                 const write_opts &opts)
 {
-	assert(opts.event_type);
+	assert(opts.appendix.test(appendix::EVENT_TYPE));
-	assert(opts.event_idx);
 	assert(json::get<"type"_>(event));
+	assert(opts.event_idx);
 
 	thread_local char buf[EVENT_TYPE_KEY_MAX_SIZE];
 	const string_view &key
@@ -804,16 +831,24 @@ ircd::m::dbs::_index_room(db::txn &txn,
                           const event &event,
                           const write_opts &opts)
 {
-	if(opts.room_head || opts.room_refs)
+	assert(opts.appendix.test(appendix::ROOM));
+
+	if(opts.appendix.test(appendix::ROOM_HEAD))
 		_index__room_head(txn, event, opts);
 
-	if(defined(json::get<"state_key"_>(event)))
+	if(opts.appendix.test(appendix::ROOM_HEAD_REFS))
+		_index__room_head_refs(txn, event, opts);
+
+	if(opts.appendix.test(appendix::STATE) && defined(json::get<"state_key"_>(event)))
 		return _index_state(txn, event, opts);
 
-	if(at<"type"_>(event) == "m.room.redaction")
+	if(opts.appendix.test(appendix::REDACT) && json::get<"type"_>(event) == "m.room.redaction")
 		return _index_redact(txn, event, opts);
 
+	if(opts.appendix.test(appendix::OTHER))
 		return _index_other(txn, event, opts);
+
+	return {};
 }
 
 ircd::string_view
@@ -822,6 +857,8 @@ ircd::m::dbs::_index_state(db::txn &txn,
                            const write_opts &opts)
 try
 {
+	assert(opts.appendix.test(appendix::STATE));
+
 	const auto &type
 	{
 		at<"type"_>(event)
@@ -834,14 +871,20 @@ try
 
 	const string_view &new_root
 	{
-		opts.op == db::op::SET && opts.history?
+		opts.op == db::op::SET && opts.appendix.test(appendix::HISTORY)?
 			state::insert(txn, opts.root_out, opts.root_in, event):
 			strlcpy(opts.root_out, opts.root_in)
 	};
 
+	if(opts.appendix.test(appendix::ROOM_EVENTS))
 		_index__room_events(txn, event, opts, new_root);
+
+	if(opts.appendix.test(appendix::ROOM_JOINED))
 		_index__room_joined(txn, event, opts);
+
+	if(opts.appendix.test(appendix::ROOM_STATE))
 		_index__room_state(txn, event, opts);
+
 	return new_root;
 }
 catch(const std::exception &e)
@@ -860,6 +903,8 @@ ircd::m::dbs::_index_redact(db::txn &txn,
                             const write_opts &opts)
 try
 {
+	assert(opts.appendix.test(appendix::REDACT));
+
 	const auto &target_id
 	{
 		at<"redacts"_>(event)
@@ -885,13 +930,18 @@ try
 
 	const string_view new_root
 	{
-		target.valid && defined(json::get<"state_key"_>(target))?
+		target.valid &&
+		defined(json::get<"state_key"_>(target)) &&
+		opts.appendix.test(appendix::HISTORY)?
 			//state::remove(txn, state_root_out, state_root_in, target):
 			strlcpy(opts.root_out, opts.root_in):
 			strlcpy(opts.root_out, opts.root_in)
 	};
 
+	if(opts.appendix.test(appendix::ROOM_EVENTS))
 		_index__room_events(txn, event, opts, opts.root_in);
+
+	if(opts.appendix.test(appendix::ROOM_STATE))
 		if(target.valid && defined(json::get<"state_key"_>(target)))
 		{
 			auto _opts(opts);
@@ -916,7 +966,11 @@ ircd::m::dbs::_index_other(db::txn &txn,
                            const event &event,
                            const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::OTHER));
+
+	if(opts.appendix.test(appendix::ROOM_EVENTS))
 		_index__room_events(txn, event, opts, opts.root_in);
+
 	return strlcpy(opts.root_out, opts.root_in);
 }
 
@@ -925,11 +979,11 @@ ircd::m::dbs::_index__room_head(db::txn &txn,
                                 const event &event,
                                 const write_opts &opts)
 {
-	thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE];
 	const ctx::critical_assertion ca;
+	thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE];
+	assert(opts.appendix.test(appendix::ROOM_HEAD));
+	assert(opts.event_idx);
 
-	if(opts.room_head)
-	{
 	const string_view &key
 	{
 		room_head_key(buf, at<"room_id"_>(event), at<"event_id"_>(event))
@@ -944,14 +998,22 @@ ircd::m::dbs::_index__room_head(db::txn &txn,
 			byte_view<string_view>{opts.event_idx}
 		}
 	};
-	}
+}
+
+void
+ircd::m::dbs::_index__room_head_refs(db::txn &txn,
+                                     const event &event,
+                                     const write_opts &opts)
+{
+	assert(opts.appendix.test(appendix::ROOM_HEAD_REFS));
 
 	//TODO: If op is DELETE and we are deleting this event and thereby
 	//TODO: potentially creating a gap in the reference graph (just for us
 	//TODO: though) can we *re-add* the prev_events to the head?
 
-	if(opts.room_refs && opts.op == db::op::SET)
+	if(opts.op != db::op::SET)
-	{
+		return;
+
 	const m::event::prev &prev{event};
 	for(const json::array &p : json::get<"prev_events"_>(prev))
 	{
@@ -960,6 +1022,8 @@ ircd::m::dbs::_index__room_head(db::txn &txn,
 			unquote(p.at(0))
 		};
 
+		thread_local char buf[ROOM_HEAD_KEY_MAX_SIZE];
+		const ctx::critical_assertion ca;
 		const string_view &key
 		{
 			room_head_key(buf, at<"room_id"_>(event), event_id)
@@ -974,7 +1038,6 @@ ircd::m::dbs::_index__room_head(db::txn &txn,
 			}
 		};
 	}
-	}
 }
 
 /// Adds the entry for the room_events column into the txn.
@@ -985,8 +1048,10 @@ ircd::m::dbs::_index__room_events(db::txn &txn,
                                   const write_opts &opts,
                                   const string_view &new_root)
 {
-	const ctx::critical_assertion ca;
+	assert(opts.appendix.test(appendix::ROOM_EVENTS));
+
 	thread_local char buf[ROOM_EVENTS_KEY_MAX_SIZE];
+	const ctx::critical_assertion ca;
 	const string_view &key
 	{
 		room_events_key(buf, at<"room_id"_>(event), at<"depth"_>(event), opts.event_idx)
@@ -1004,20 +1069,21 @@ ircd::m::dbs::_index__room_events(db::txn &txn,
 }
 
 /// Adds the entry for the room_joined column into the txn.
-/// This only is affected if opts.present=true
 void
 ircd::m::dbs::_index__room_joined(db::txn &txn,
                                    const event &event,
                                    const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::ROOM_JOINED));
+
 	if(!opts.present)
 		return;
 
 	if(at<"type"_>(event) != "m.room.member")
 		return;
 
-	const ctx::critical_assertion ca;
 	thread_local char buf[ROOM_JOINED_KEY_MAX_SIZE];
+	const ctx::critical_assertion ca;
 	const string_view &key
 	{
 		room_joined_key(buf, at<"room_id"_>(event), at<"origin"_>(event), at<"state_key"_>(event))
@@ -1067,6 +1133,8 @@ ircd::m::dbs::_index__room_state(db::txn &txn,
                                  const event &event,
                                  const write_opts &opts)
 {
+	assert(opts.appendix.test(appendix::ROOM_STATE));
+
 	if(!opts.present)
 		return;