From 3f6ae85f8f7d75eb69e2a4031cc7924d14bba948 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 30 Nov 2018 14:43:51 -0800 Subject: [PATCH] ircd::db: Add automated column drop support; update existing descriptors. --- include/ircd/db/descriptor.h | 7 ++++ ircd/db.cc | 23 +++++++++++- ircd/m/dbs.cc | 73 ++++++++++++++++++++++++++++++++++++ modules/media/media.cc | 1 + 4 files changed, 102 insertions(+), 2 deletions(-) diff --git a/include/ircd/db/descriptor.h b/include/ircd/db/descriptor.h index b54f2ea9d..13de155dc 100644 --- a/include/ircd/db/descriptor.h +++ b/include/ircd/db/descriptor.h @@ -44,6 +44,13 @@ struct ircd::db::descriptor /// User given prefix extractor. db::prefix_transform prefix {}; + /// Indicates if this column should be marked for deletion. Users who + /// upgrade to the new schema will still require a legacy descriptor + /// with most of the essential fields preceding this value to open db. + /// + /// !!! Setting this to true deletes all data for this column !!! + bool drop { false }; + /// Size of the LRU cache for uncompressed blocks ssize_t cache_size { -1 }; diff --git a/ircd/db.cc b/ircd/db.cc index 97140bca7..44fc2f924 100644 --- a/ircd/db.cc +++ b/ircd/db.cc @@ -806,18 +806,21 @@ try } ,column_names{[this] { - // Existing columns at path. If any are left the descriptor set did not - // describe all of the columns found in the database at path. const auto opts { make_dbopts(this->optstr) }; + // Existing columns at path. If any are left the descriptor set did not + // describe all of the columns found in the database at path. const auto required { db::column_names(path, opts) }; + // As we find descriptors for all of the columns on the disk we'll + // remove their names from this set. Anything remaining is undescribed + // and that's a fatal error. std::set existing { begin(required), end(required) @@ -827,6 +830,13 @@ try decltype(this->column_names) ret; for(auto &descriptor : descriptors) { + // Deprecated columns which have already been dropped won't appear + // in the existing (required) list. We don't need to construct those. + if(!existing.count(descriptor.name) && descriptor.drop) + continue; + + // Construct the column instance and indicate that we have a description + // for it by removing it from existing. ret.emplace(descriptor.name, std::make_shared(*this, descriptor)); existing.erase(descriptor.name); } @@ -1064,6 +1074,15 @@ try return checkpointer; }()} { + // Conduct drops from schema changes. The database must be fully opened + // as if they were not dropped first, then we conduct the drop operation + // here. The drop operation has no effects until the database is next + // closed; the dropped columns will still work during this instance. + for(const auto &colptr : columns) + if(describe(*colptr).drop) + db::drop(*colptr); + + // Database integrity check branch. if(ircd::checkdb) { log::notice diff --git a/ircd/m/dbs.cc b/ircd/m/dbs.cc index 3130165c7..9308babd7 100644 --- a/ircd/m/dbs.cc +++ b/ircd/m/dbs.cc @@ -662,6 +662,9 @@ ircd::m::dbs::desc::events__event_idx // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, //uses conf item @@ -795,6 +798,9 @@ ircd::m::dbs::desc::events__room_head // prefix transform events__room_head__pfx, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1070,6 +1076,9 @@ ircd::m::dbs::desc::events__room_events // prefix transform events__room_events__pfx, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1233,6 +1242,9 @@ ircd::m::dbs::desc::events__room_joined // prefix transform events__room_joined__pfx, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1407,6 +1419,9 @@ ircd::m::dbs::desc::events__room_state // prefix transform events__room_state__pfx, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1509,6 +1524,9 @@ ircd::m::dbs::desc::events__state_node // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1619,6 +1637,9 @@ ircd::m::dbs::desc::events_event_id // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1711,6 +1732,9 @@ ircd::m::dbs::desc::events_type // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1803,6 +1827,9 @@ ircd::m::dbs::desc::events_content // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1889,6 +1916,9 @@ ircd::m::dbs::desc::events_redacts // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -1980,6 +2010,9 @@ ircd::m::dbs::desc::events_room_id // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2071,6 +2104,9 @@ ircd::m::dbs::desc::events_sender // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2164,6 +2200,9 @@ ircd::m::dbs::desc::events_state_key // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2252,6 +2291,9 @@ ircd::m::dbs::desc::events_origin // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2351,6 +2393,9 @@ ircd::m::dbs::desc::events_origin_server_ts // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2437,6 +2482,9 @@ ircd::m::dbs::desc::events_signatures // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2522,6 +2570,9 @@ ircd::m::dbs::desc::events_auth_events // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2614,6 +2665,9 @@ ircd::m::dbs::desc::events_depth // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2699,6 +2753,9 @@ ircd::m::dbs::desc::events_hashes // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2784,6 +2841,9 @@ ircd::m::dbs::desc::events_membership // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2869,6 +2929,9 @@ ircd::m::dbs::desc::events_prev_events // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -2954,6 +3017,9 @@ ircd::m::dbs::desc::events_prev_state // prefix transform {}, + // drop column + false, + // cache size bool(events_cache_enable)? -1 : 0, @@ -3003,6 +3069,9 @@ ircd::m::dbs::desc::events__default // prefix transform {}, + // drop column + false, + // cache size 0_MiB, @@ -3016,6 +3085,10 @@ ircd::m::dbs::desc::events__default false, }; +// +// Description vector +// + const ircd::db::description ircd::m::dbs::desc::events { diff --git a/modules/media/media.cc b/modules/media/media.cc index 2d9ff9892..8caee371e 100644 --- a/modules/media/media.cc +++ b/modules/media/media.cc @@ -77,6 +77,7 @@ media_blocks_descriptor {}, // options {}, // comparaor {}, // prefix transform + false, // drop column bool(media_blocks_cache_enable)? -1 : 0,