From b95a588bc40720896113b8d76f253c35866d665f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 17 Apr 2018 15:28:08 -0700 Subject: [PATCH] ircd::db: Comment/improve descriptor/column options. --- include/ircd/db/database/descriptor.h | 27 +++++++++++++++ ircd/db.cc | 50 ++++++++++++++++++++------- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/include/ircd/db/database/descriptor.h b/include/ircd/db/database/descriptor.h index ffb73e4bc..49409376f 100644 --- a/include/ircd/db/database/descriptor.h +++ b/include/ircd/db/database/descriptor.h @@ -17,13 +17,40 @@ struct ircd::db::database::descriptor { using typing = std::pair; + /// User given name for this column. Must be consistent. std::string name; + + /// User given description of this column; not used by RocksDB std::string explain; + + /// Indicate key and value type. typing type { typeid(string_view), typeid(string_view) }; + + /// RocksDB ColumnFamilyOptions string; can be used for items not + /// otherwise specified here. std::string options {}; + + /// User given comparator. We can automatically set this value for + /// some types given for the type.first typeid; otherwise it must be + /// set for exotic/unsupported keys. db::comparator cmp {}; + + /// User given prefix extractor. db::prefix_transform prefix {}; + + /// Size of the LRU cache for uncompressed blocks size_t cache_size { 16_MiB }; + + /// Size of the LRU cache for compressed blocks size_t cache_size_comp { 8_MiB }; + + /// Bloom filter bits. Filter is still useful even if queries are expected + /// to always hit on this column; see `expect_queries_hit` option. size_t bloom_bits { 10 }; + + /// Set this option to true if queries to this column are expected to + /// find keys that exist. This is useful for columns with keys that + /// were first found from values in another column, where if the first + /// column missed there'd be no reason to query this column. + bool expect_queries_hit { false }; }; diff --git a/ircd/db.cc b/ircd/db.cc index e79a59069..98ed75bb8 100644 --- a/ircd/db.cc +++ b/ircd/db.cc @@ -930,16 +930,22 @@ ircd::db::database::column::column(database *const &d, } } { + // If possible, deduce comparator based on type given in descriptor if(!this->descriptor.cmp.less) { if(key_type == typeid(string_view)) this->cmp.user = cmp_string_view{}; else if(key_type == typeid(int64_t)) this->cmp.user = cmp_int64_t{}; + else if(key_type == typeid(uint64_t)) + this->cmp.user = cmp_uint64_t{}; else - throw error("column '%s' key type[%s] requires user supplied comparator", - this->name, - key_type.name()); + throw error + { + "column '%s' key type[%s] requires user supplied comparator", + this->name, + key_type.name() + }; } // Set the key comparator @@ -952,26 +958,46 @@ ircd::db::database::column::column(database *const &d, &this->prefix, [](const rocksdb::SliceTransform *) {} }; - //if(d->mergeop->merger) - // this->options.merge_operator = d->mergeop; + // + // Table options + // + // Setup the cache for assets. const auto &cache_size(this->descriptor.cache_size); - table_opts.block_cache = rocksdb::NewLRUCache(cache_size); + if(cache_size) + table_opts.block_cache = rocksdb::NewLRUCache(cache_size); + // Setup the cache for compressed assets. const auto &cache_size_comp(this->descriptor.cache_size_comp); - table_opts.block_cache_compressed = rocksdb::NewLRUCache(cache_size_comp); - - // Tickers::READ_AMP_TOTAL_READ_BYTES / Tickers::READ_AMP_ESTIMATE_USEFUL_BYTES - //table_opts.read_amp_bytes_per_bit = 8; + if(cache_size_comp) + table_opts.block_cache_compressed = rocksdb::NewLRUCache(cache_size_comp); + // Setup the bloom filter. const auto &bloom_bits(this->descriptor.bloom_bits); if(bloom_bits) table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(bloom_bits, false)); + // Tickers::READ_AMP_TOTAL_READ_BYTES / Tickers::READ_AMP_ESTIMATE_USEFUL_BYTES + //table_opts.read_amp_bytes_per_bit = 8; + this->options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_opts)); - //log.debug("'%s': Creating new column '%s'", d->name, this->name); - //throw_on_error(d->d->CreateColumnFamily(this->options, this->name, &this->handle)); + // + // Misc options + // + + // Set the compaction style; we don't override this in the descriptor yet. + this->options.compaction_style = rocksdb::kCompactionStyleLevel; + + // Set the compaction priority; this should probably be in the descriptor + // but this is currently selected for the general matrix workload. + this->options.compaction_pri = rocksdb::CompactionPri::kOldestLargestSeqFirst; + + // Set filter reductions for this column. This means we expect a key to exist. + this->options.optimize_filters_for_hits = this->descriptor.expect_queries_hit; + + // Compression + this->options.compression = rocksdb::kSnappyCompression; log.debug("schema '%s' declares column [%s => %s] cmp[%s] pfx[%s] lru:%zu:%zu bloom:%zu %s", db::name(*d),