ircd::db: Add automated column drop support; update existing descriptors.

include/ircd/db/descriptor.h

@@ -44,6 +44,13 @@ struct ircd::db::descriptor
 	/// User given prefix extractor.
 	db::prefix_transform prefix {};
 
+	/// Indicates if this column should be marked for deletion. Users who
+	/// upgrade to the new schema will still require a legacy descriptor
+	/// with most of the essential fields preceding this value to open db.
+	///
+	/// !!! Setting this to true deletes all data for this column !!!
+	bool drop { false };
+
 	/// Size of the LRU cache for uncompressed blocks
 	ssize_t cache_size { -1 };
 

ircd/db.cc

@@ -806,18 +806,21 @@ try
 }
 ,column_names{[this]
 {
-	// Existing columns at path. If any are left the descriptor set did not
-	// describe all of the columns found in the database at path.
 	const auto opts
 	{
 		make_dbopts(this->optstr)
 	};
 
+	// Existing columns at path. If any are left the descriptor set did not
+	// describe all of the columns found in the database at path.
 	const auto required
 	{
 		db::column_names(path, opts)
 	};
 
+	// As we find descriptors for all of the columns on the disk we'll
+	// remove their names from this set. Anything remaining is undescribed
+	// and that's a fatal error.
 	std::set<string_view> existing
 	{
 		begin(required), end(required)
@@ -827,6 +830,13 @@ try
 	decltype(this->column_names) ret;
 	for(auto &descriptor : descriptors)
 	{
+		// Deprecated columns which have already been dropped won't appear
+		// in the existing (required) list. We don't need to construct those.
+		if(!existing.count(descriptor.name) && descriptor.drop)
+			continue;
+
+		// Construct the column instance and indicate that we have a description
+		// for it by removing it from existing.
 		ret.emplace(descriptor.name, std::make_shared<column>(*this, descriptor));
 		existing.erase(descriptor.name);
 	}
@@ -1064,6 +1074,15 @@ try
 	return checkpointer;
 }()}
 {
+	// Conduct drops from schema changes. The database must be fully opened
+	// as if they were not dropped first, then we conduct the drop operation
+	// here. The drop operation has no effects until the database is next
+	// closed; the dropped columns will still work during this instance.
+	for(const auto &colptr : columns)
+		if(describe(*colptr).drop)
+			db::drop(*colptr);
+
+	// Database integrity check branch.
 	if(ircd::checkdb)
 	{
 		log::notice

ircd/m/dbs.cc

@@ -662,6 +662,9 @@ ircd::m::dbs::desc::events__event_idx
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0, //uses conf item
 
@@ -795,6 +798,9 @@ ircd::m::dbs::desc::events__room_head
 	// prefix transform
 	events__room_head__pfx,
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1070,6 +1076,9 @@ ircd::m::dbs::desc::events__room_events
 	// prefix transform
 	events__room_events__pfx,
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1233,6 +1242,9 @@ ircd::m::dbs::desc::events__room_joined
 	// prefix transform
 	events__room_joined__pfx,
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1407,6 +1419,9 @@ ircd::m::dbs::desc::events__room_state
 	// prefix transform
 	events__room_state__pfx,
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1509,6 +1524,9 @@ ircd::m::dbs::desc::events__state_node
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1619,6 +1637,9 @@ ircd::m::dbs::desc::events_event_id
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1711,6 +1732,9 @@ ircd::m::dbs::desc::events_type
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1803,6 +1827,9 @@ ircd::m::dbs::desc::events_content
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1889,6 +1916,9 @@ ircd::m::dbs::desc::events_redacts
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -1980,6 +2010,9 @@ ircd::m::dbs::desc::events_room_id
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2071,6 +2104,9 @@ ircd::m::dbs::desc::events_sender
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2164,6 +2200,9 @@ ircd::m::dbs::desc::events_state_key
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2252,6 +2291,9 @@ ircd::m::dbs::desc::events_origin
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2351,6 +2393,9 @@ ircd::m::dbs::desc::events_origin_server_ts
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2437,6 +2482,9 @@ ircd::m::dbs::desc::events_signatures
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2522,6 +2570,9 @@ ircd::m::dbs::desc::events_auth_events
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2614,6 +2665,9 @@ ircd::m::dbs::desc::events_depth
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2699,6 +2753,9 @@ ircd::m::dbs::desc::events_hashes
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2784,6 +2841,9 @@ ircd::m::dbs::desc::events_membership
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2869,6 +2929,9 @@ ircd::m::dbs::desc::events_prev_events
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -2954,6 +3017,9 @@ ircd::m::dbs::desc::events_prev_state
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	bool(events_cache_enable)? -1 : 0,
 
@@ -3003,6 +3069,9 @@ ircd::m::dbs::desc::events__default
 	// prefix transform
 	{},
 
+	// drop column
+	false,
+
 	// cache size
 	0_MiB,
 
@@ -3016,6 +3085,10 @@ ircd::m::dbs::desc::events__default
 	false,
 };
 
+//
+// Description vector
+//
+
 const ircd::db::description
 ircd::m::dbs::desc::events
 {

modules/media/media.cc

@@ -77,6 +77,7 @@ media_blocks_descriptor
 	{},      // options
 	{},      // comparaor
 	{},      // prefix transform
+	false,   // drop column
 
 	bool(media_blocks_cache_enable)? -1 : 0,