From 5df4bf6da247de58c7277d059f91dd1130eefb83 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 23 Sep 2016 21:01:57 -0700 Subject: [PATCH] ircd::db: Add database interface. Support RocksDB. --- .gitmodules | 3 + .travis.yml | 15 ++- charybdis/Makefile.am | 2 + configure.ac | 50 +++++++ include/ircd/db.h | 88 ++++++++++++ include/ircd/db_meta.h | 56 ++++++++ include/ircd/stdinc.h | 1 + include/rb/requires.h | 4 + ircd/Makefile.am | 6 +- ircd/db.cc | 294 +++++++++++++++++++++++++++++++++++++++++ ircd/ircd.cc | 1 + rocksdb | 1 + tools/buildrocks.sh | 51 +++++++ 13 files changed, 569 insertions(+), 3 deletions(-) create mode 100644 include/ircd/db.h create mode 100644 include/ircd/db_meta.h create mode 100644 ircd/db.cc create mode 160000 rocksdb create mode 100755 tools/buildrocks.sh diff --git a/.gitmodules b/.gitmodules index 2a3f21535..1f0b6727f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "boost"] path = boost url = https://github.com/boostorg/boost.git +[submodule "rocksdb"] + path = rocksdb + url = https://github.com/facebook/rocksdb.git diff --git a/.travis.yml b/.travis.yml index ec61f9778..853267932 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,8 @@ matrix: env: - CCOMPILER=gcc-4.9 - CXXCOMPILER=g++-4.9 + - WITH_INCLUDED_BOOST=--with-included-boost + - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared - os: linux dist: precise @@ -48,6 +50,8 @@ matrix: env: - CCOMPILER=gcc-5 - CXXCOMPILER=g++-5 + - WITH_INCLUDED_BOOST=--with-included-boost + - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared - os: linux dist: trusty @@ -65,6 +69,8 @@ matrix: env: - CCOMPILER=gcc-6 - CXXCOMPILER=g++-6 + - WITH_INCLUDED_BOOST=--with-included-boost + - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared - os: linux dist: precise @@ -82,6 +88,8 @@ matrix: env: - CCOMPILER=clang-3.6 - CXXCOMPILER=clang++-3.6 + - WITH_INCLUDED_BOOST=--with-included-boost + - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared - os: linux dist: trusty @@ -101,6 +109,8 @@ matrix: env: - CCOMPILER=clang-3.8 - CXXCOMPILER=clang++-3.8 + - WITH_INCLUDED_BOOST=--with-included-boost + - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared - os: osx compiler: clang @@ -108,7 +118,8 @@ matrix: - CCOMPILER=clang - CXXCOMPILER=clang++ - LIBTOOLIZE=glibtoolize - + - WITH_INCLUDED_BOOST=--with-included-boost + - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared osx_image: xcode7.3 @@ -126,6 +137,6 @@ script: - $CC --version - $CXX --version - time bash autogen.sh -- time ./configure --with-shared-sqlite --with-included-boost CC=$CC CXX=$CXX +- time ./configure --with-shared-sqlite $WITH_INCLUDED_BOOST $WITH_INCLUDED_ROCKSDB CC=$CC CXX=$CXX - time make -j4 - time make -j4 install diff --git a/charybdis/Makefile.am b/charybdis/Makefile.am index b3c52bc16..9c5ce9447 100644 --- a/charybdis/Makefile.am +++ b/charybdis/Makefile.am @@ -10,6 +10,7 @@ AM_LDFLAGS = \ AM_LDFLAGS += \ -L$(top_srcdir)/ircd \ -L$(top_srcdir)/rb \ + @ROCKSDB_LDFLAGS@ \ @BOOST_LDFLAGS@ @@ -22,4 +23,5 @@ charybdis_SOURCES = \ charybdis_LDADD = \ -lircd \ -lrb \ + @ROCKSDB_LIBS@ \ @BOOST_LIBS@ diff --git a/configure.ac b/configure.ac index 558f5a0a4..aca9733a9 100644 --- a/configure.ac +++ b/configure.ac @@ -338,6 +338,10 @@ RB_CHK_SYSHEADER([iomanip], [IOMANIP]) RB_CHK_SYSHEADER([cstdio], [CSTDIO]) RB_CHK_SYSHEADER([chrono], [CHRONO]) RB_CHK_SYSHEADER([ctime], [CTIME]) +RB_CHK_SYSHEADER([atomic], [ATOMIC]) +RB_CHK_SYSHEADER([thread], [THREAD]) +RB_CHK_SYSHEADER([mutex], [MUTEX]) +RB_CHK_SYSHEADER([condition_variable], [CONDITION_VARIABLE]) dnl experimental RB_CHK_SYSHEADER([string_view], [STRING_VIEW]) @@ -866,6 +870,51 @@ RB_DEFINE_UNQUOTED([INC_BOOST_SPIRIT_KARMA_HPP], [boost/spirit/include/karma.hpp +dnl +dnl RocksDB support +dnl + +AC_MSG_CHECKING([whether you asked to use the RocksDB included here]) +AC_ARG_WITH(included-rocksdb, +AC_HELP_STRING([--with-included-rocksdb[[[=shared]]]], [Use the RocksDB sources from included submodule]), +[ + AC_MSG_RESULT([yes]) + with_included_rocksdb="yes" + + AC_SUBST(ROCKSDB_CPPFLAGS, ["-I $PWD/rocksdb/include"]) + AC_SUBST(ROCKSDB_LDFLAGS, ["-L$PWD/rocksdb/"]) + + AC_MSG_CHECKING([whether to use shared RocksDB]) + if [[ $withval = "shared" ]]; then + AC_MSG_RESULT([yes]) + rocksdb_linkage="shared_lib" + AC_MSG_NOTICE([Shared RocksDB linkage requires running charybdis with an intact build directory]) + ROCKSDB_LDFLAGS+=" -Wl,-rpath -Wl,$PWD/rocksdb/" + AC_SUBST(ROCKSDB_LIBS, ["-lrocksdb"]) + else + AC_MSG_RESULT([no]) + rocksdb_linkage="static_lib" + AC_MSG_NOTICE([static RocksDB linkage requires multiple dependencies]) + AC_MSG_NOTICE([| You will need: bzip2, zlib, snappy]) + AC_SUBST(ROCKSDB_LIBS, ["$PWD/rocksdb/librocksdb.a"]) + fi + + bash tools/buildrocks.sh $rocksdb_linkage +],[ + AC_MSG_RESULT([no]) + with_included_rocksdb="no" + + AC_CHECK_LIB(rocksdb, rocksdb_open, [], [ + AC_MSG_ERROR([Unable to find required RocksDB package. Try apt-get install librocksdb-dev]) + ]) + + AC_SUBST(ROCKSDB_CPPFLAGS, []) + AC_SUBST(ROCKSDB_LDFLAGS, []) + AC_SUBST(ROCKSDB_LIBS, ["-lrocksdb"]) +]) + + + dnl dnl OpenSSL support dnl @@ -1283,6 +1332,7 @@ echo "Configuration time ................ $RB_DATESTR" echo "Compiler .......................... $CXX" echo "Compiler flags (CXXFLAGS) ......... $CXXFLAGS" echo "Building boost .................... $with_included_boost" +echo "Building RocksDB................... $with_included_rocksdb" echo "Precompiled headers ............... $build_pch" echo "Developer debug ................... $debug" echo "IPv6 support ...................... $ipv6" diff --git a/include/ircd/db.h b/include/ircd/db.h new file mode 100644 index 000000000..eafcf92e3 --- /dev/null +++ b/include/ircd/db.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2016 Charybdis Development Team + * Copyright (C) 2016 Jason Volk + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice is present in all copies. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#pragma once +#define HAVE_IRCD_DB_H + +namespace rocksdb +{ + struct DB; +} + +namespace ircd { +namespace db { + +IRCD_EXCEPTION(ircd::error, error) +IRCD_EXCEPTION(error, not_found) +IRCD_EXCEPTION(error, corruption) +IRCD_EXCEPTION(error, not_supported) +IRCD_EXCEPTION(error, invalid_argument) +IRCD_EXCEPTION(error, io_error) +IRCD_EXCEPTION(error, merge_in_progress) +IRCD_EXCEPTION(error, incomplete) +IRCD_EXCEPTION(error, shutdown_in_progress) +IRCD_EXCEPTION(error, timed_out) +IRCD_EXCEPTION(error, aborted) +IRCD_EXCEPTION(error, busy) +IRCD_EXCEPTION(error, expired) +IRCD_EXCEPTION(error, try_again) + +std::string path(const std::string &name); + +struct opts +{ + bool create_if_missing = true; +}; + +struct read_opts +{ +}; + +struct write_opts +{ +}; + +class handle +{ + std::unique_ptr meta; + std::unique_ptr d; + + public: + using char_closure = std::function; + using string_closure = std::function; + + bool has(const std::string &key, const read_opts & = {}); + void get(const std::string &key, const char_closure &, const read_opts & = {}); + void set(const std::string &key, const std::string &value, const write_opts & = {}); + + handle(const std::string &name, const opts &opts = {}); + ~handle() noexcept; +}; + +struct init +{ + init(); + ~init() noexcept; +}; + +} // namespace db +} // namespace ircd diff --git a/include/ircd/db_meta.h b/include/ircd/db_meta.h new file mode 100644 index 000000000..83155e323 --- /dev/null +++ b/include/ircd/db_meta.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2016 Charybdis Development Team + * Copyright (C) 2016 Jason Volk + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice is present in all copies. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#pragma once +#define HAVE_IRCD_DB_META_H + +#include + +namespace ircd { +namespace db { + +struct meta +{ + std::string name; + std::string path; + rocksdb::Options opts; + + meta(const std::string &name, const std::string &path, const struct opts &opts); +}; + +inline +meta::meta(const std::string &name, + const std::string &path, + const struct opts &opts) +:name{name} +,path{path} +,opts{[&opts] +{ + rocksdb::Options ret; + ret.create_if_missing = opts.create_if_missing; + return ret; +}()} +{ +} + +} // namespace db +} // namespace ircd diff --git a/include/ircd/stdinc.h b/include/ircd/stdinc.h index 9f70d277f..3399057da 100644 --- a/include/ircd/stdinc.h +++ b/include/ircd/stdinc.h @@ -83,6 +83,7 @@ namespace ircd #include "fmt.h" #include "err.h" #include "fs.h" +#include "db.h" #include "s_assert.h" #include "match.h" #include "mode_table.h" diff --git a/include/rb/requires.h b/include/rb/requires.h index 98e88931c..811847a49 100644 --- a/include/rb/requires.h +++ b/include/rb/requires.h @@ -83,6 +83,10 @@ extern "C" { #include + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice is present in all copies. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +namespace ircd { +namespace db { + +using rocksdb::DB; + +namespace work +{ + using closure = std::function; + + std::mutex mutex; + std::condition_variable cond; + std::deque queue; + bool interruption; + std::thread *thread; + + closure pop(); + void worker() noexcept; + void push(closure &&); + + void fini(); + void init(); +} + +void throw_on_error(const rocksdb::Status &); +void query(std::function); + +} // namespace db +} // namespace ircd + +using namespace ircd; + +db::init::init() +{ + work::init(); +} + +db::init::~init() +noexcept +{ + work::fini(); +} + +db::handle::handle(const std::string &name, + const opts &opts) +try +:meta{std::make_unique(name, path(name), opts)} +,d{[this, &name] +{ + DB *ptr; + throw_on_error(DB::Open(meta->opts, path(name), &ptr)); + std::unique_ptr ret{ptr}; + return ret; +}()} +{ +} +catch(const std::exception &e) +{ + throw error("Failed to open db '%s': %s", + name.c_str(), + e.what()); +} + +db::handle::~handle() +noexcept +{ +} + +void +db::handle::set(const std::string &key, + const std::string &value, + const write_opts &opts) +{ + using rocksdb::WriteOptions; + using rocksdb::Slice; + + const Slice k(key.data(), key.size()); + const Slice v(value.data(), value.size()); + throw_on_error(d->Put(WriteOptions(), k, v)); +} + +void +db::handle::get(const std::string &key, + const char_closure &func, + const read_opts &opts) +{ + using rocksdb::ReadOptions; + using rocksdb::Iterator; + using rocksdb::Slice; + + ReadOptions ropts; + const Slice sk(key.data(), key.size()); + query([this, &sk, &func, &ropts] + { + const std::unique_ptr it(d->NewIterator(ropts)); + + it->Seek(sk); + throw_on_error(it->status()); + + const auto &v(it->value()); + func(v.data(), v.size()); + }); +} + +bool +db::handle::has(const std::string &key, + const read_opts &opts) +{ + using rocksdb::ReadOptions; + using rocksdb::Iterator; + using rocksdb::Slice; + + bool ret; + ReadOptions ropts; + const Slice k(key.data(), key.size()); + query([this, &k, &ret, &ropts] + { + if(!d->KeyMayExist(ropts, k, nullptr, nullptr)) + { + ret = false; + return; + } + + const std::unique_ptr it(d->NewIterator(ropts)); + + it->Seek(k); + switch(it->status().code()) + { + using rocksdb::Status; + + case Status::kOk: ret = true; return; + case Status::kNotFound: ret = false; return; + default: + throw_on_error(it->status()); + } + }); + + return ret; +} + +void +db::query(std::function func) +{ + std::exception_ptr eptr; + auto &context(ctx::cur()); + std::atomic done{false}; + auto closure([func(std::move(func)), &eptr, &context, &done] + () noexcept + { + try + { + func(); + } + catch(...) + { + eptr = std::current_exception(); + } + + done.store(true, std::memory_order_release); + notify(context); + }); + + work::push(std::move(closure)); do + { + ctx::wait(); + } + while(!done.load(std::memory_order_consume)); + + if(eptr) + std::rethrow_exception(eptr); +} + +void +db::work::init() +{ + assert(!thread); + interruption = false; + thread = new std::thread(&worker); +} + +void +db::work::fini() +{ + if(!thread) + return; + + mutex.lock(); + interruption = true; + cond.notify_one(); + mutex.unlock(); + thread->join(); + delete thread; + thread = nullptr; +} + +void +db::work::push(closure &&func) +{ + const std::lock_guard lock(mutex); + queue.emplace_back(std::move(func)); + cond.notify_one(); +} + +void +db::work::worker() +noexcept try +{ + while(1) + { + const auto func(pop()); + func(); + } +} +catch(const ctx::interrupted &) +{ + return; +} + +db::work::closure +db::work::pop() +{ + std::unique_lock lock(mutex); + cond.wait(lock, [] + { + if(!queue.empty()) + return true; + + if(unlikely(interruption)) + throw ctx::interrupted(); + + return false; + }); + + auto c(std::move(queue.front())); + queue.pop_front(); + return std::move(c); +} + +std::string +db::path(const std::string &name) +{ + const auto prefix(path::get(path::DB)); + return path::build({prefix, name}); +} + +void +db::throw_on_error(const rocksdb::Status &s) +{ + using rocksdb::Status; + + switch(s.code()) + { + case Status::kOk: return; + case Status::kNotFound: throw not_found(); + case Status::kCorruption: throw corruption(); + case Status::kNotSupported: throw not_supported(); + case Status::kInvalidArgument: throw invalid_argument(); + case Status::kIOError: throw io_error(); + case Status::kMergeInProgress: throw merge_in_progress(); + case Status::kIncomplete: throw incomplete(); + case Status::kShutdownInProgress: throw shutdown_in_progress(); + case Status::kTimedOut: throw timed_out(); + case Status::kAborted: throw aborted(); + case Status::kBusy: throw busy(); + case Status::kExpired: throw expired(); + case Status::kTryAgain: throw try_again(); + default: + throw error("Unknown error"); + } +} diff --git a/ircd/ircd.cc b/ircd/ircd.cc index c1dad2d4a..517f0e729 100644 --- a/ircd/ircd.cc +++ b/ircd/ircd.cc @@ -92,6 +92,7 @@ noexcept try // to the main context. Initialization can also occur in ircd::init() if static initialization // and destruction is not possible, but there is no complementary destruction up there. mods::init _mods_; + db::init _db_; // Create IRCd's agency ircd::me = add_client(); diff --git a/rocksdb b/rocksdb new file mode 160000 index 000000000..0a1bd9c50 --- /dev/null +++ b/rocksdb @@ -0,0 +1 @@ +Subproject commit 0a1bd9c509786b9ab6365e263b867c1bbdca6cc7 diff --git a/tools/buildrocks.sh b/tools/buildrocks.sh new file mode 100755 index 000000000..d935fff52 --- /dev/null +++ b/tools/buildrocks.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +LINKAGE=$1 +if [ -z $LINKAGE ]; then + LINKAGE="shared_lib" +fi + +JOBS=$2 +if [ -z $JOBS ]; then + JOBS=4 +fi + +run () +{ + COMMAND=$1 + # check for empty commands + if test -z "$COMMAND" ; then + echo -e "\033[1;5;31mERROR\033[0m No command specified!" + return 1 + fi + + shift; + OPTIONS="$@" + # print a message + if test -n "$OPTIONS" ; then + echo -ne "\033[1m$COMMAND $OPTIONS\033[0m ... " + else + echo -ne "\033[1m$COMMAND\033[0m ... " + fi + + # run or die + $COMMAND $OPTIONS ; RESULT=$? + if test $RESULT -ne 0 ; then + echo -e "\033[1;5;31mERROR\033[0m $COMMAND failed. (exit code = $RESULT)" + exit 1 + fi + + echo -e "\033[0;32myes\033[0m" + return 0 +} + + +echo "*** Building RocksDB... " + +USERDIR=$PWD # Save current dir and return to it later + +run git submodule update --init rocksdb + +run cd rocksdb +CFLAGS=-fPIC run make -j $JOBS $LINKAGE +run cd $USERDIR # Return to user's original directory