ircd::db: Add database interface. Support RocksDB.

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "boost"]
 	path = boost
 	url = https://github.com/boostorg/boost.git
+[submodule "rocksdb"]
+	path = rocksdb
+	url = https://github.com/facebook/rocksdb.git

.travis.yml

@@ -32,6 +32,8 @@ matrix:
     env:
     - CCOMPILER=gcc-4.9
     - CXXCOMPILER=g++-4.9
+    - WITH_INCLUDED_BOOST=--with-included-boost
+    - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared
 
   - os: linux
     dist: precise
@@ -48,6 +50,8 @@ matrix:
     env:
     - CCOMPILER=gcc-5
     - CXXCOMPILER=g++-5
+    - WITH_INCLUDED_BOOST=--with-included-boost
+    - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared
 
   - os: linux
     dist: trusty
@@ -65,6 +69,8 @@ matrix:
     env:
      - CCOMPILER=gcc-6
      - CXXCOMPILER=g++-6
+     - WITH_INCLUDED_BOOST=--with-included-boost
+     - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared
 
   - os: linux
     dist: precise
@@ -82,6 +88,8 @@ matrix:
     env:
     - CCOMPILER=clang-3.6
     - CXXCOMPILER=clang++-3.6
+    - WITH_INCLUDED_BOOST=--with-included-boost
+    - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared
 
   - os: linux
     dist: trusty
@@ -101,6 +109,8 @@ matrix:
     env:
     - CCOMPILER=clang-3.8
     - CXXCOMPILER=clang++-3.8
+    - WITH_INCLUDED_BOOST=--with-included-boost
+    - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared
 
   - os: osx
     compiler: clang
@@ -108,7 +118,8 @@ matrix:
     - CCOMPILER=clang
     - CXXCOMPILER=clang++
     - LIBTOOLIZE=glibtoolize
-
+    - WITH_INCLUDED_BOOST=--with-included-boost
+    - WITH_INCLUDED_ROCKSDB=--with-included-rocksdb=shared
 
 osx_image: xcode7.3
 
@@ -126,6 +137,6 @@ script:
 - $CC --version
 - $CXX --version
 - time bash autogen.sh
-- time ./configure --with-shared-sqlite --with-included-boost CC=$CC CXX=$CXX
+- time ./configure --with-shared-sqlite $WITH_INCLUDED_BOOST $WITH_INCLUDED_ROCKSDB CC=$CC CXX=$CXX
 - time make -j4
 - time make -j4 install

charybdis/Makefile.am

@@ -10,6 +10,7 @@ AM_LDFLAGS = \
 AM_LDFLAGS += \
 	-L$(top_srcdir)/ircd \
 	-L$(top_srcdir)/rb \
+	@ROCKSDB_LDFLAGS@ \
 	@BOOST_LDFLAGS@
 
 
@@ -22,4 +23,5 @@ charybdis_SOURCES = \
 charybdis_LDADD = \
 	-lircd \
 	-lrb \
+	@ROCKSDB_LIBS@ \
 	@BOOST_LIBS@

configure.ac

@@ -338,6 +338,10 @@ RB_CHK_SYSHEADER([iomanip],        [IOMANIP])
 RB_CHK_SYSHEADER([cstdio],         [CSTDIO])
 RB_CHK_SYSHEADER([chrono],         [CHRONO])
 RB_CHK_SYSHEADER([ctime],          [CTIME])
+RB_CHK_SYSHEADER([atomic],         [ATOMIC])
+RB_CHK_SYSHEADER([thread],         [THREAD])
+RB_CHK_SYSHEADER([mutex],          [MUTEX])
+RB_CHK_SYSHEADER([condition_variable], [CONDITION_VARIABLE])
 
 dnl experimental
 RB_CHK_SYSHEADER([string_view],                 [STRING_VIEW])
@@ -866,6 +870,51 @@ RB_DEFINE_UNQUOTED([INC_BOOST_SPIRIT_KARMA_HPP], [boost/spirit/include/karma.hpp
 
 
 
+dnl
+dnl RocksDB support
+dnl
+
+AC_MSG_CHECKING([whether you asked to use the RocksDB included here])
+AC_ARG_WITH(included-rocksdb,
+AC_HELP_STRING([--with-included-rocksdb[[[=shared]]]], [Use the RocksDB sources from included submodule]),
+[
+	AC_MSG_RESULT([yes])
+	with_included_rocksdb="yes"
+
+	AC_SUBST(ROCKSDB_CPPFLAGS, ["-I $PWD/rocksdb/include"])
+	AC_SUBST(ROCKSDB_LDFLAGS, ["-L$PWD/rocksdb/"])
+
+	AC_MSG_CHECKING([whether to use shared RocksDB])
+	if [[ $withval = "shared" ]]; then
+		AC_MSG_RESULT([yes])
+		rocksdb_linkage="shared_lib"
+		AC_MSG_NOTICE([Shared RocksDB linkage requires running charybdis with an intact build directory])
+		ROCKSDB_LDFLAGS+=" -Wl,-rpath -Wl,$PWD/rocksdb/"
+		AC_SUBST(ROCKSDB_LIBS, ["-lrocksdb"])
+	else
+		AC_MSG_RESULT([no])
+		rocksdb_linkage="static_lib"
+		AC_MSG_NOTICE([static RocksDB linkage requires multiple dependencies])
+		AC_MSG_NOTICE([| You will need: bzip2, zlib, snappy])
+		AC_SUBST(ROCKSDB_LIBS, ["$PWD/rocksdb/librocksdb.a"])
+	fi
+
+	bash tools/buildrocks.sh $rocksdb_linkage
+],[
+	AC_MSG_RESULT([no])
+	with_included_rocksdb="no"
+
+	AC_CHECK_LIB(rocksdb, rocksdb_open, [], [
+		AC_MSG_ERROR([Unable to find required RocksDB package. Try apt-get install librocksdb-dev])
+	])
+
+	AC_SUBST(ROCKSDB_CPPFLAGS, [])
+	AC_SUBST(ROCKSDB_LDFLAGS, [])
+	AC_SUBST(ROCKSDB_LIBS, ["-lrocksdb"])
+])
+
+
+
 dnl
 dnl OpenSSL support
 dnl
@@ -1283,6 +1332,7 @@ echo "Configuration time ................ $RB_DATESTR"
 echo "Compiler .......................... $CXX"
 echo "Compiler flags (CXXFLAGS) ......... $CXXFLAGS"
 echo "Building boost .................... $with_included_boost"
+echo "Building RocksDB................... $with_included_rocksdb"
 echo "Precompiled headers ............... $build_pch"
 echo "Developer debug ................... $debug"
 echo "IPv6 support ...................... $ipv6"

include/ircd/db.h

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2016 Charybdis Development Team
+ * Copyright (C) 2016 Jason Volk <jason@zemos.net>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice is present in all copies.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#pragma once
+#define HAVE_IRCD_DB_H
+
+namespace rocksdb
+{
+	struct DB;
+}
+
+namespace ircd {
+namespace db   {
+
+IRCD_EXCEPTION(ircd::error, error)
+IRCD_EXCEPTION(error, not_found)
+IRCD_EXCEPTION(error, corruption)
+IRCD_EXCEPTION(error, not_supported)
+IRCD_EXCEPTION(error, invalid_argument)
+IRCD_EXCEPTION(error, io_error)
+IRCD_EXCEPTION(error, merge_in_progress)
+IRCD_EXCEPTION(error, incomplete)
+IRCD_EXCEPTION(error, shutdown_in_progress)
+IRCD_EXCEPTION(error, timed_out)
+IRCD_EXCEPTION(error, aborted)
+IRCD_EXCEPTION(error, busy)
+IRCD_EXCEPTION(error, expired)
+IRCD_EXCEPTION(error, try_again)
+
+std::string path(const std::string &name);
+
+struct opts
+{
+	bool create_if_missing = true;
+};
+
+struct read_opts
+{
+};
+
+struct write_opts
+{
+};
+
+class handle
+{
+	std::unique_ptr<struct meta> meta;
+	std::unique_ptr<rocksdb::DB> d;
+
+  public:
+	using char_closure = std::function<void (const char *, size_t)>;
+	using string_closure = std::function<void (const std::string &)>;
+
+	bool has(const std::string &key, const read_opts & = {});
+	void get(const std::string &key, const char_closure &, const read_opts & = {});
+	void set(const std::string &key, const std::string &value, const write_opts & = {});
+
+	handle(const std::string &name, const opts &opts = {});
+	~handle() noexcept;
+};
+
+struct init
+{
+	init();
+	~init() noexcept;
+};
+
+} // namespace db
+} // namespace ircd

include/ircd/db_meta.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2016 Charybdis Development Team
+ * Copyright (C) 2016 Jason Volk <jason@zemos.net>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice is present in all copies.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#pragma once
+#define HAVE_IRCD_DB_META_H
+
+#include <rocksdb/options.h>
+
+namespace ircd {
+namespace db   {
+
+struct meta
+{
+	std::string name;
+	std::string path;
+	rocksdb::Options opts;
+
+	meta(const std::string &name, const std::string &path, const struct opts &opts);
+};
+
+inline
+meta::meta(const std::string &name,
+           const std::string &path,
+           const struct opts &opts)
+:name{name}
+,path{path}
+,opts{[&opts]
+{
+	rocksdb::Options ret;
+	ret.create_if_missing = opts.create_if_missing;
+	return ret;
+}()}
+{
+}
+
+} // namespace db
+} // namespace ircd

include/ircd/stdinc.h

@@ -83,6 +83,7 @@ namespace ircd
 #include "fmt.h"
 #include "err.h"
 #include "fs.h"
+#include "db.h"
 #include "s_assert.h"
 #include "match.h"
 #include "mode_table.h"

include/rb/requires.h

@@ -83,6 +83,10 @@ extern "C" {
 #include <RB_INC_CSTDIO
 #include <RB_INC_CHRONO
 #include <RB_INC_CTIME
+#include <RB_INC_ATOMIC
+#include <RB_INC_THREAD
+#include <RB_INC_MUTEX
+#include <RB_INC_CONDITION_VARIABLE
 
 //#include <RB_INC_BOOST_LEXICAL_CAST_HPP
 #endif

ircd/Makefile.am

@@ -4,6 +4,7 @@ libircddir	= @libdir@
 
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/include \
+	@ROCKSDB_CPPFLAGS@ \
 	@BOOST_CPPFLAGS@ \
 	-include ircd/ircd.h
 
@@ -19,12 +20,14 @@ AM_LDFLAGS = \
 
 AM_LDFLAGS += \
 	-L$(top_srcdir)/rb \
+	@ROCKSDB_LDFLAGS@ \
 	@BOOST_LDFLAGS@
 
 libircd_LTLIBRARIES = libircd.la
 
 libircd_la_LIBADD = \
 	-lrb \
+	@ROCKSDB_LIBS@ \
 	@BOOST_LIBS@
 
 libircd_la_SOURCES =   \
@@ -45,7 +48,8 @@ libircd_la_SOURCES =   \
 	client.cc          \
 	vm.cc              \
 	hook.cc            \
-	fmt.cc
+	fmt.cc             \
+	db.cc
 
 #authproc.cc			\
 #bandbi.cc                      \

ircd/db.cc

@@ -0,0 +1,294 @@
+/*
+ * Copyright (C) 2016 Charybdis Development Team
+ * Copyright (C) 2016 Jason Volk <jason@zemos.net>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice is present in all copies.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <rocksdb/db.h>
+#include <ircd/db_meta.h>
+
+namespace ircd {
+namespace db   {
+
+using rocksdb::DB;
+
+namespace work
+{
+	using closure = std::function<void () noexcept>;
+
+	std::mutex mutex;
+	std::condition_variable cond;
+	std::deque<closure> queue;
+	bool interruption;
+	std::thread *thread;
+
+	closure pop();
+	void worker() noexcept;
+	void push(closure &&);
+
+	void fini();
+	void init();
+}
+
+void throw_on_error(const rocksdb::Status &);
+void query(std::function<void ()>);
+
+} // namespace db
+} // namespace ircd
+
+using namespace ircd;
+
+db::init::init()
+{
+	work::init();
+}
+
+db::init::~init()
+noexcept
+{
+	work::fini();
+}
+
+db::handle::handle(const std::string &name,
+                   const opts &opts)
+try
+:meta{std::make_unique<struct meta>(name, path(name), opts)}
+,d{[this, &name]
+{
+	DB *ptr;
+	throw_on_error(DB::Open(meta->opts, path(name), &ptr));
+	std::unique_ptr<DB> ret{ptr};
+	return ret;
+}()}
+{
+}
+catch(const std::exception &e)
+{
+	throw error("Failed to open db '%s': %s",
+	            name.c_str(),
+	            e.what());
+}
+
+db::handle::~handle()
+noexcept
+{
+}
+
+void
+db::handle::set(const std::string &key,
+                const std::string &value,
+                const write_opts &opts)
+{
+	using rocksdb::WriteOptions;
+	using rocksdb::Slice;
+
+	const Slice k(key.data(), key.size());
+	const Slice v(value.data(), value.size());
+	throw_on_error(d->Put(WriteOptions(), k, v));
+}
+
+void
+db::handle::get(const std::string &key,
+                const char_closure &func,
+                const read_opts &opts)
+{
+	using rocksdb::ReadOptions;
+	using rocksdb::Iterator;
+	using rocksdb::Slice;
+
+	ReadOptions ropts;
+	const Slice sk(key.data(), key.size());
+	query([this, &sk, &func, &ropts]
+	{
+		const std::unique_ptr<Iterator> it(d->NewIterator(ropts));
+
+		it->Seek(sk);
+		throw_on_error(it->status());
+
+		const auto &v(it->value());
+		func(v.data(), v.size());
+	});
+}
+
+bool
+db::handle::has(const std::string &key,
+                const read_opts &opts)
+{
+	using rocksdb::ReadOptions;
+	using rocksdb::Iterator;
+	using rocksdb::Slice;
+
+	bool ret;
+	ReadOptions ropts;
+	const Slice k(key.data(), key.size());
+	query([this, &k, &ret, &ropts]
+	{
+		if(!d->KeyMayExist(ropts, k, nullptr, nullptr))
+		{
+			ret = false;
+			return;
+		}
+
+		const std::unique_ptr<Iterator> it(d->NewIterator(ropts));
+
+		it->Seek(k);
+		switch(it->status().code())
+		{
+			using rocksdb::Status;
+
+			case Status::kOk:         ret = true;   return;
+			case Status::kNotFound:   ret = false;  return;
+			default:
+				throw_on_error(it->status());
+		}
+	});
+
+	return ret;
+}
+
+void
+db::query(std::function<void ()> func)
+{
+	std::exception_ptr eptr;
+	auto &context(ctx::cur());
+	std::atomic<bool> done{false};
+	auto closure([func(std::move(func)), &eptr, &context, &done]
+	() noexcept
+	{
+		try
+		{
+			func();
+		}
+		catch(...)
+		{
+			eptr = std::current_exception();
+		}
+
+		done.store(true, std::memory_order_release);
+		notify(context);
+	});
+
+	work::push(std::move(closure)); do
+	{
+		ctx::wait();
+	}
+	while(!done.load(std::memory_order_consume));
+
+	if(eptr)
+		std::rethrow_exception(eptr);
+}
+
+void
+db::work::init()
+{
+	assert(!thread);
+	interruption = false;
+	thread = new std::thread(&worker);
+}
+
+void
+db::work::fini()
+{
+	if(!thread)
+		return;
+
+	mutex.lock();
+	interruption = true;
+	cond.notify_one();
+	mutex.unlock();
+	thread->join();
+	delete thread;
+	thread = nullptr;
+}
+
+void
+db::work::push(closure &&func)
+{
+	const std::lock_guard<decltype(mutex)> lock(mutex);
+	queue.emplace_back(std::move(func));
+	cond.notify_one();
+}
+
+void
+db::work::worker()
+noexcept try
+{
+	while(1)
+	{
+		const auto func(pop());
+		func();
+	}
+}
+catch(const ctx::interrupted &)
+{
+	return;
+}
+
+db::work::closure
+db::work::pop()
+{
+	std::unique_lock<decltype(mutex)> lock(mutex);
+	cond.wait(lock, []
+	{
+		if(!queue.empty())
+			return true;
+
+		if(unlikely(interruption))
+			throw ctx::interrupted();
+
+		return false;
+	});
+
+	auto c(std::move(queue.front()));
+	queue.pop_front();
+	return std::move(c);
+}
+
+std::string
+db::path(const std::string &name)
+{
+	const auto prefix(path::get(path::DB));
+	return path::build({prefix, name});
+}
+
+void
+db::throw_on_error(const rocksdb::Status &s)
+{
+	using rocksdb::Status;
+
+	switch(s.code())
+	{
+		case Status::kOk:                   return;
+		case Status::kNotFound:             throw not_found();
+		case Status::kCorruption:           throw corruption();
+		case Status::kNotSupported:         throw not_supported();
+		case Status::kInvalidArgument:      throw invalid_argument();
+		case Status::kIOError:              throw io_error();
+		case Status::kMergeInProgress:      throw merge_in_progress();
+		case Status::kIncomplete:           throw incomplete();
+		case Status::kShutdownInProgress:   throw shutdown_in_progress();
+		case Status::kTimedOut:             throw timed_out();
+		case Status::kAborted:              throw aborted();
+		case Status::kBusy:                 throw busy();
+		case Status::kExpired:              throw expired();
+		case Status::kTryAgain:             throw try_again();
+		default:
+			throw error("Unknown error");
+	}
+}

ircd/ircd.cc

@@ -92,6 +92,7 @@ noexcept try
 	// to the main context. Initialization can also occur in ircd::init() if static initialization
 	// and destruction is not possible, but there is no complementary destruction up there.
 	mods::init _mods_;
+	db::init _db_;
 
 	// Create IRCd's agency
 	ircd::me = add_client();

rocksdb

@@ -0,0 +1 @@
+Subproject commit 0a1bd9c509786b9ab6365e263b867c1bbdca6cc7

tools/buildrocks.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+LINKAGE=$1
+if [ -z $LINKAGE ]; then
+	LINKAGE="shared_lib"
+fi
+
+JOBS=$2
+if [ -z $JOBS ]; then
+	JOBS=4
+fi
+
+run ()
+{
+	COMMAND=$1
+	# check for empty commands
+	if test -z "$COMMAND" ; then
+		echo -e "\033[1;5;31mERROR\033[0m No command specified!"
+		return 1
+	fi
+
+	shift;
+	OPTIONS="$@"
+	# print a message
+	if test -n "$OPTIONS" ; then
+		echo -ne "\033[1m$COMMAND $OPTIONS\033[0m ... "
+	else
+		echo -ne "\033[1m$COMMAND\033[0m ... "
+	fi
+
+	# run or die
+	$COMMAND $OPTIONS ; RESULT=$?
+	if test $RESULT -ne 0 ; then
+		echo -e "\033[1;5;31mERROR\033[0m $COMMAND failed. (exit code = $RESULT)"
+		exit 1
+	fi
+
+	echo -e "\033[0;32myes\033[0m"
+	return 0
+}
+
+
+echo "*** Building RocksDB... "
+
+USERDIR=$PWD            # Save current dir and return to it later
+
+run git submodule update --init rocksdb
+
+run cd rocksdb
+CFLAGS=-fPIC run make -j $JOBS $LINKAGE
+run cd $USERDIR         # Return to user's original directory