MirrorHub/construct
0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-11-25 08:12:37 +01:00
Code Issues Releases Wiki Activity

ircd::db: Consolidate rocksdb interpositions into db_fixes unit; minor cleanup.

Browse source
This commit is contained in:
Jason Volk 2020-04-29 00:31:52 -07:00
parent a8870908a5
commit 9d7c1096e8
3 changed files with 26 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
ircd/Makefile.am
View file

@ -155,8 +155,7 @@ libircd_la_SOURCES += mods.cc
if LINUX
libircd_la_SOURCES += mods_ldso.cc
endif
libircd_la_SOURCES += db_write_thread.cc
libircd_la_SOURCES += db_crc32.cc
libircd_la_SOURCES += db_fixes.cc
libircd_la_SOURCES += db_port.cc
libircd_la_SOURCES += db_env.cc
libircd_la_SOURCES += db.cc
@ -189,12 +188,9 @@ ctx_eh.lo: AM_CPPFLAGS := ${ASIO_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
db.lo: AM_CPPFLAGS := ${ROCKSDB_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
db_env.lo: AM_CPPFLAGS := ${ROCKSDB_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
db_port.lo: AM_CPPFLAGS := ${ROCKSDB_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
db_write_thread.lo: AM_CPPFLAGS := ${ROCKSDB_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
db_write_thread.lo: AM_CPPFLAGS += -I$(top_srcdir)/deps/rocksdb/include
db_write_thread.lo: AM_CPPFLAGS += -I$(top_srcdir)/deps/rocksdb
db_crc32.lo: AM_CPPFLAGS := ${ROCKSDB_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
db_crc32.lo: AM_CPPFLAGS += -I$(top_srcdir)/deps/rocksdb/include
db_crc32.lo: AM_CPPFLAGS += -I$(top_srcdir)/deps/rocksdb
db_fixes.lo: AM_CPPFLAGS := ${ROCKSDB_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
db_fixes.lo: AM_CPPFLAGS += -I$(top_srcdir)/deps/rocksdb/include
db_fixes.lo: AM_CPPFLAGS += -I$(top_srcdir)/deps/rocksdb
exception.lo: AM_CPPFLAGS := ${ASIO_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
fmt.lo: AM_CPPFLAGS := ${SPIRIT_UNIT_CPPFLAGS} ${AM_CPPFLAGS}
fmt.lo: AM_CXXFLAGS := ${SPIRIT_UNIT_CXXFLAGS} ${AM_CXXFLAGS}

26
ircd/db_crc32.cc
View file

@ -1,26 +0,0 @@
// Matrix Construct
//
// Copyright (C) Matrix Construct Developers, Authors & Contributors
// Copyright (C) 2016-2019 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
#if __has_include("table/block_fetcher.h")
#define ROCKSDB_PLATFORM_POSIX
#define ZSTD_VERSION_NUMBER 0
#include "table/block_fetcher.h"
//#define IRCD_DB_BYPASS_CHECKSUM
#else
#warning "The RocksDB source code is not available. Cannot interpose bugfixes for db/.h."
#endif
#if defined(IRCD_DB_BYPASS_CHECKSUM)
void
rocksdb::BlockFetcher::CheckBlockChecksum()
{
//assert(0);
}
#endif

53
ircd/db_write_thread.cc → ircd/db_fixes.cc
View file

@ -8,19 +8,22 @@
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
#if __has_include("db/write_thread.h")
#define ROCKSDB_PLATFORM_POSIX
#include "db/write_thread.h"
#define ROCKSDB_PLATFORM_POSIX
#if __has_include("table/block_fetcher.h")
//#define IRCD_DB_BYPASS_CHECKSUM
#define ZSTD_VERSION_NUMBER 0
#include "table/block_fetcher.h"
#endif
#ifndef STORAGE_ROCKSDB_INCLUDE_DB_H_
#warning "The RocksDB source code is not available. Cannot interpose bugfixes for db/write_thread.h."
#if __has_include("db/write_thread.h")
#include "db/write_thread.h"
#endif
///////////////////////////////////////////////////////////////////////////////
//
// This unit exists to mitigate an instance of a bug in RocksDB documented by
// https://github.com/facebook/rocksdb/issues/4654. In summary, some RocksDB
// This section exists to mitigate an instance of a bug in RocksDB documented
// by https://github.com/facebook/rocksdb/issues/4654. In summary, some RocksDB
// code makes direct use of std::mutex and std::condition_variable unlike the
// rest of RocksDB code which uses the rocksdb::port and rocksdb::Env wrapper
// interfaces. We have adapted the latter to work with ircd::ctx userspace
@ -40,29 +43,7 @@
// which tells the kernel to stop the thread until satisfied. Since we are not
// using kernel-driven threads, this is a deadlock.
//
// Our solution is not ideal, but operates effectively and is rarely invoked
// given our prior mitigations to eliminate write contention inside RocksDB.
// This code may be hit during simultaneous flush and compaction jobs, which is
// a rare event with our configuration. When hit, our yield loop tends to have
// one iteration. Our solution using a coarse yield, though, is not ideal: the
// waiting ircd::ctx is not actually being notified of a state mask change
// because the write-leader is notifying an std::condition_variable which leads
// to nowhere. Our coarse yield simply puts the waiting ircd::ctx at the back
// of the ircd::ios queue (similar to a sched_yield(2)) and hopes that the
// write-leader has finished its task and updated the state mask before the
// waiter reaches the front of the queue to check. In the worst-case, the
// waiter will requeue itself to check the state mask too often. We can make
// further improvements here by also overriding the notification mechanism
// as well, but we do not feel it's required at this time, as noted.
//
// Unfortunately, we are overriding a C++ class member function to achieve this.
// Class member functions require the correct class definition. This means we
// require a copy of the RocksDB source code to be accessible at build-time. If
// no copy of the RocksDB source is found when this unit is built, it is not an
// error, but the mitigation will not be made and the result will be vulnerable
// to this deadlock.
#ifdef STORAGE_ROCKSDB_INCLUDE_DB_H_
#if __has_include("db/write_thread.h")
uint8_t
rocksdb::WriteThread::BlockingAwaitState(Writer *const w,
uint8_t goal_mask)
@ -92,4 +73,14 @@ rocksdb::WriteThread::BlockingAwaitState(Writer *const w,
assert((state & goal_mask) != 0);
return state;
}
#endif STORAGE_ROCKSDB_INCLUDE_DB_H_
#else
#warning "RocksDB source is not available. Cannot interpose bugfixes."
#endif
#if __has_include("table/block_fetcher.h") && defined(IRCD_DB_BYPASS_CHECKSUM)
void
rocksdb::BlockFetcher::CheckBlockChecksum()
{
//assert(false);
}
#endif