From cd74a39330c6b741ca2dae616d8d90174cac69f7 Mon Sep 17 00:00:00 2001
From: Jason Volk <jason@zemos.net>
Date: Sun, 26 Aug 2018 20:24:38 -0700
Subject: [PATCH] ircd::db: Move the write mutex to a global and lock for
 compaction points too.

---
 ircd/db.cc | 23 +++++++++++++++++------
 ircd/db.h  |  1 +
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/ircd/db.cc b/ircd/db.cc
index c4bff6c35..52421d9af 100644
--- a/ircd/db.cc
+++ b/ircd/db.cc
@@ -109,6 +109,9 @@ ircd::db::request
 	0, // don't prespawn because this is static
 };
 
+decltype(ircd::db::write_mutex)
+ircd::db::write_mutex;
+
 ///////////////////////////////////////////////////////////////////////////////
 //
 // init
@@ -6624,9 +6627,6 @@ ircd::db::compact(column &column,
 	d.d->GetColumnFamilyMetaData(c, &cfmd);
 	for(const auto &level : cfmd.levels)
 	{
-		ctx::interruption_point();
-		ctx::uninterruptible::nothrow ui;
-
 		if(level_ != -1 && level.level != level_)
 			continue;
 
@@ -6640,6 +6640,16 @@ ircd::db::compact(column &column,
 			return std::move(metadata.name);
 		});
 
+		// Locking the write_mutex here prevents writes during a column's
+		// compaction. This is needed because if contention occurs inside
+		// rocksdb we will hit some std::mutex's which do not use the
+		// rocksdb::port wrapper and deadlock the process. (It is an error
+		// on the part of rocksdb to directly use std::mutex rather than their
+		// port wrapper).
+		ctx::interruption_point();
+		const std::lock_guard<decltype(write_mutex)> lock{write_mutex};
+		const ctx::uninterruptible::nothrow ui;
+
 		log::debug
 		{
 			log, "'%s':'%s' COMPACT level:%d files:%zu size:%zu",
@@ -7295,9 +7305,10 @@ ircd::db::commit(database &d,
 
 	// This lock is necessary to serialize entry into rocksdb's write impl
 	// otherwise there's a risk of a deadlock if their internal pthread
-	// mutexes are contended.
-	thread_local ctx::mutex mutex;
-	const std::lock_guard<decltype(mutex)> lock{mutex};
+	// mutexes are contended. This is because a few parts of rocksdb are
+	// using std::mutex directly when they ought to be using their
+	// rocksdb::port wrapper.
+	const std::lock_guard<decltype(write_mutex)> lock{write_mutex};
 	const ctx::uninterruptible ui;
 
 	throw_on_error
diff --git a/ircd/db.h b/ircd/db.h
index 7786608d3..c5f8aeae5 100644
--- a/ircd/db.h
+++ b/ircd/db.h
@@ -44,6 +44,7 @@ namespace ircd::db
 	extern conf::item<size_t> request_pool_size;
 	extern conf::item<size_t> request_pool_stack_size;
 	extern ctx::pool request;
+	extern ctx::mutex write_mutex;
 
 	// reflections
 	string_view reflect(const rocksdb::Env::Priority &p);