maxmustermann/dogecoin
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge #15118: Refactor block file logic

Browse source 
04cca33094 Style cleanup. (Jim Posen)
4c01e4e159 flatfile: Unit tests for FlatFileSeq methods. (Jim Posen)
65a489e93d scripted-diff: Rename CBlockDiskPos to FlatFilePos. (Jim Posen)
d6d8a78f26 Move CDiskBlockPos from chain to flatfile. (Jim Posen)
e0380933e3 validation: Refactor file flush logic into FlatFileSeq. (Jim Posen)
992404b31e validation: Refactor block file pre-allocation into FlatFileSeq. (Jim Posen)
e2d2abb99f validation: Refactor OpenDiskFile into method on FlatFileSeq. (Jim Posen)
9183d6ef65 validation: Extract basic block file logic into FlatFileSeq class. (Jim Posen)
62e7addb63 util: Move CheckDiskSpace to util. (Jim Posen)

Pull request description:

  This cleans up and refactors block file helpers so that they may be used by the block filter indexer. Per [design discussion](https://github.com/bitcoin/bitcoin/pull/14121#issuecomment-451252591) about storing BIP 157 block filters, it has been suggested that they are stored in the same way as block and undo data. This refactor is sufficient to simplify file operations for this use case, though in the future perhaps more pruning-related logic ought to be moved into the new classes.

  The basic abstraction is a `FlatFileSeq` which manages access to a sequence of numbered files into which raw data is written.

Tree-SHA512: b2108756777f2dad8964a1a2ef2764486e708a4a4a8cfac47b5de8bcb0625388964438eb096b10cfd9ea39212c299b5cb32fa943e768db2333cf49ea7def157e
This commit is contained in:
Wladimir J. van der Laan 2019-03-02 23:03:21 +01:00
parent 789b0bbf2a 04cca33094
commit 2d46f1be0c
No known key found for this signature in database
GPG key ID: 1E4AED62986CD25D
12 changed files with 417 additions and 181 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Makefile.am
View file

@ -128,6 +128,7 @@ BITCOIN_CORE_H = \
core_io.h \
core_memusage.h \
cuckoocache.h \
flatfile.h \
fs.h \
httprpc.h \
httpserver.h \
@ -247,6 +248,7 @@ libbitcoin_server_a_SOURCES = \
chain.cpp \
checkpoints.cpp \
consensus/tx_verify.cpp \
flatfile.cpp \
httprpc.cpp \
httpserver.cpp \
index/base.cpp \

1
src/Makefile.test.include
View file

@ -83,6 +83,7 @@ BITCOIN_TESTS =\
test/cuckoocache_tests.cpp \
test/denialofservice_tests.cpp \
test/descriptor_tests.cpp \
test/flatfile_tests.cpp \
test/fs_tests.cpp \
test/getarg_tests.cpp \
test/hash_tests.cpp \

49
src/chain.h
View file

@ -8,6 +8,7 @@
#include <arith_uint256.h>
#include <consensus/params.h>
#include <flatfile.h>
#include <primitives/block.h>
#include <tinyformat.h>
#include <uint256.h>
@ -90,46 +91,6 @@ public:
}
};
struct CDiskBlockPos
{
int nFile;
unsigned int nPos;
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
READWRITE(VARINT(nPos));
}
CDiskBlockPos() {
SetNull();
}
CDiskBlockPos(int nFileIn, unsigned int nPosIn) {
nFile = nFileIn;
nPos = nPosIn;
}
friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) {
return (a.nFile == b.nFile && a.nPos == b.nPos);
}
friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) {
return !(a == b);
}
void SetNull() { nFile = -1; nPos = 0; }
bool IsNull() const { return (nFile == -1); }
std::string ToString() const
{
return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos);
}
};
enum BlockStatus: uint32_t {
//! Unused.
BLOCK_VALID_UNKNOWN = 0,
@ -266,8 +227,8 @@ public:
nNonce = block.nNonce;
}
CDiskBlockPos GetBlockPos() const {
CDiskBlockPos ret;
FlatFilePos GetBlockPos() const {
FlatFilePos ret;
if (nStatus & BLOCK_HAVE_DATA) {
ret.nFile = nFile;
ret.nPos = nDataPos;
@ -275,8 +236,8 @@ public:
return ret;
}
CDiskBlockPos GetUndoPos() const {
CDiskBlockPos ret;
FlatFilePos GetUndoPos() const {
FlatFilePos ret;
if (nStatus & BLOCK_HAVE_UNDO) {
ret.nFile = nFile;
ret.nPos = nUndoPos;

98
src/flatfile.cpp Normal file
View file

@ -0,0 +1,98 @@
// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <stdexcept>
#include <flatfile.h>
#include <logging.h>
#include <tinyformat.h>
#include <util/system.h>
FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) :
m_dir(std::move(dir)),
m_prefix(prefix),
m_chunk_size(chunk_size)
{
if (chunk_size == 0) {
throw std::invalid_argument("chunk_size must be positive");
}
}
std::string FlatFilePos::ToString() const
{
return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos);
}
fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const
{
return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile);
}
FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only)
{
if (pos.IsNull()) {
return nullptr;
}
fs::path path = FileName(pos);
fs::create_directories(path.parent_path());
FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+");
if (!file && !read_only)
file = fsbridge::fopen(path, "wb+");
if (!file) {
LogPrintf("Unable to open file %s\n", path.string());
return nullptr;
}
if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) {
LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
fclose(file);
return nullptr;
}
return file;
}
size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space)
{
out_of_space = false;
unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size;
unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size;
if (n_new_chunks > n_old_chunks) {
size_t old_size = pos.nPos;
size_t new_size = n_new_chunks * m_chunk_size;
size_t inc_size = new_size - old_size;
if (CheckDiskSpace(m_dir, inc_size)) {
FILE *file = Open(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile);
AllocateFileRange(file, pos.nPos, inc_size);
fclose(file);
return inc_size;
}
} else {
out_of_space = true;
}
}
return 0;
}
bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize)
{
FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos
if (!file) {
return error("%s: failed to open file %d", __func__, pos.nFile);
}
if (finalize && !TruncateFile(file, pos.nPos)) {
fclose(file);
return error("%s: failed to truncate file %d", __func__, pos.nFile);
}
if (!FileCommit(file)) {
fclose(file);
return error("%s: failed to commit file %d", __func__, pos.nFile);
}
fclose(file);
return true;
}

96
src/flatfile.h Normal file
View file

@ -0,0 +1,96 @@
// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_FLATFILE_H
#define BITCOIN_FLATFILE_H
#include <string>
#include <fs.h>
#include <serialize.h>
struct FlatFilePos
{
int nFile;
unsigned int nPos;
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
READWRITE(VARINT(nPos));
}
FlatFilePos() : nFile(-1), nPos(0) {}
FlatFilePos(int nFileIn, unsigned int nPosIn) :
nFile(nFileIn),
nPos(nPosIn)
{}
friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) {
return (a.nFile == b.nFile && a.nPos == b.nPos);
}
friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) {
return !(a == b);
}
void SetNull() { nFile = -1; nPos = 0; }
bool IsNull() const { return (nFile == -1); }
std::string ToString() const;
};
/**
* FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates
* access to and efficient management of these files.
*/
class FlatFileSeq
{
private:
const fs::path m_dir;
const char* const m_prefix;
const size_t m_chunk_size;
public:
/**
* Constructor
*
* @param dir The base directory that all files live in.
* @param prefix A short prefix given to all file names.
* @param chunk_size Disk space is pre-allocated in multiples of this amount.
*/
FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size);
/** Get the name of the file at the given position. */
fs::path FileName(const FlatFilePos& pos) const;
/** Open a handle to the file at the given position. */
FILE* Open(const FlatFilePos& pos, bool read_only = false);
/**
* Allocate additional space in a file after the given starting position. The amount allocated
* will be the minimum multiple of the sequence chunk size greater than add_size.
*
* @param[in] pos The starting position that bytes will be allocated after.
* @param[in] add_size The minimum number of bytes to be allocated.
* @param[out] out_of_space Whether the allocation failed due to insufficient disk space.
* @return The number of bytes successfully allocated.
*/
size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space);
/**
* Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final.
*
* @param[in] pos The first unwritten position in the file to be flushed.
* @param[in] finalize True if no more data will be written to this file.
* @return true on success, false on failure.
*/
bool Flush(const FlatFilePos& pos, bool finalize = false);
};
#endif // BITCOIN_FLATFILE_H

8
src/index/txindex.cpp
View file

@ -16,7 +16,7 @@ constexpr char DB_TXINDEX_BLOCK = 'T';
std::unique_ptr<TxIndex> g_txindex;
struct CDiskTxPos : public CDiskBlockPos
struct CDiskTxPos : public FlatFilePos
{
unsigned int nTxOffset; // after header
@ -24,11 +24,11 @@ struct CDiskTxPos : public CDiskBlockPos
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITEAS(CDiskBlockPos, *this);
READWRITEAS(FlatFilePos, *this);
READWRITE(VARINT(nTxOffset));
}
CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
}
CDiskTxPos() {
@ -36,7 +36,7 @@ struct CDiskTxPos : public CDiskBlockPos
}
void SetNull() {
CDiskBlockPos::SetNull();
FlatFilePos::SetNull();
nTxOffset = 0;
}
};

8
src/init.cpp
View file

@ -668,8 +668,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles)
if (fReindex) {
int nFile = 0;
while (true) {
CDiskBlockPos pos(nFile, 0);
if (!fs::exists(GetBlockPosFilename(pos, "blk")))
FlatFilePos pos(nFile, 0);
if (!fs::exists(GetBlockPosFilename(pos)))
break; // No block files left to reindex
FILE *file = OpenBlockFile(pos, true);
if (!file)
@ -1669,11 +1669,11 @@ bool AppInitMain(InitInterfaces& interfaces)
// ********************************************************* Step 11: import blocks
if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) {
if (!CheckDiskSpace(GetDataDir())) {
InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir()));
return false;
}
if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) {
if (!CheckDiskSpace(GetBlocksDir())) {
InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir()));
return false;
}

123
src/test/flatfile_tests.cpp Normal file
View file

@ -0,0 +1,123 @@
// Copyright (c) 2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <flatfile.h>
#include <test/test_bitcoin.h>
#include <boost/test/unit_test.hpp>
BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(flatfile_filename)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFilePos pos(456, 789);
FlatFileSeq seq1(data_dir, "a", 16 * 1024);
BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat");
FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024);
BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat");
}
BOOST_AUTO_TEST_CASE(flatfile_open)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFileSeq seq(data_dir, "a", 16 * 1024);
std::string line1("A purely peer-to-peer version of electronic cash would allow online "
"payments to be sent directly from one party to another without going "
"through a financial institution.");
std::string line2("Digital signatures provide part of the solution, but the main benefits are "
"lost if a trusted third party is still required to prevent double-spending.");
size_t pos1 = 0;
size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION);
// Write first line to file.
{
CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION);
file << LIMITED_STRING(line1, 256);
}
// Attempt to append to file opened in read-only mode.
{
CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION);
BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure);
}
// Append second line to file.
{
CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION);
file << LIMITED_STRING(line2, 256);
}
// Read text from file in read-only mode.
{
std::string text;
CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION);
file >> LIMITED_STRING(text, 256);
BOOST_CHECK_EQUAL(text, line1);
file >> LIMITED_STRING(text, 256);
BOOST_CHECK_EQUAL(text, line2);
}
// Read text from file with position offset.
{
std::string text;
CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION);
file >> LIMITED_STRING(text, 256);
BOOST_CHECK_EQUAL(text, line2);
}
// Ensure another file in the sequence has no data.
{
std::string text;
CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION);
BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure);
}
}
BOOST_AUTO_TEST_CASE(flatfile_allocate)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFileSeq seq(data_dir, "a", 100);
bool out_of_space;
BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100);
BOOST_CHECK(!out_of_space);
BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100);
BOOST_CHECK(!out_of_space);
BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200);
BOOST_CHECK(!out_of_space);
}
BOOST_AUTO_TEST_CASE(flatfile_flush)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFileSeq seq(data_dir, "a", 100);
bool out_of_space;
seq.Allocate(FlatFilePos(0, 0), 1, out_of_space);
// Flush without finalize should not truncate file.
seq.Flush(FlatFilePos(0, 1));
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100);
// Flush with finalize should truncate file.
seq.Flush(FlatFilePos(0, 1), true);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1);
}
BOOST_AUTO_TEST_SUITE_END()

8
src/util/system.cpp
View file

@ -135,6 +135,14 @@ bool DirIsWritable(const fs::path& directory)
return true;
}
bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes)
{
constexpr uint64_t min_disk_space = 52428800; // 50 MiB
uint64_t free_bytes_available = fs::space(dir).available;
return free_bytes_available >= min_disk_space + additional_bytes;
}
/**
* Interpret a string argument as a boolean.
*

1
src/util/system.h
View file

@ -72,6 +72,7 @@ bool RenameOver(fs::path src, fs::path dest);
bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false);
void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name);
bool DirIsWritable(const fs::path& directory);
bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0);
/** Release all directory locks. This is used for unit testing only, at runtime
* the global destructor will take care of the locks.

189
src/validation.cpp
View file

@ -15,6 +15,7 @@
#include <consensus/tx_verify.h>
#include <consensus/validation.h>
#include <cuckoocache.h>
#include <flatfile.h>
#include <hash.h>
#include <index/txindex.h>
#include <policy/fees.h>
@ -165,7 +166,7 @@ public:
* that it doesn't descend from an invalid block, and then add it to mapBlockIndex.
*/
bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
// Block (dis)connection on a given view:
DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view);
@ -204,7 +205,7 @@ private:
void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main);
void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
@ -317,7 +318,9 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState &
static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight);
static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight);
bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr);
static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false);
static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false);
static FlatFileSeq BlockFileSeq();
static FlatFileSeq UndoFileSeq();
bool CheckFinalTx(const CTransaction &tx, int flags)
{
@ -1042,7 +1045,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus
// CBlock and CBlockIndex
//
static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart)
static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION);
@ -1063,7 +1066,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes
return true;
}
bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams)
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams)
{
block.SetNull();
@ -1089,7 +1092,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus:
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams)
{
CDiskBlockPos blockPos;
FlatFilePos blockPos;
{
LOCK(cs_main);
blockPos = pindex->GetBlockPos();
@ -1103,9 +1106,9 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus
return true;
}
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start)
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start)
{
CDiskBlockPos hpos = pos;
FlatFilePos hpos = pos;
hpos.nPos -= 8; // Seek back 8 bytes for meta header
CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION);
if (filein.IsNull()) {
@ -1140,7 +1143,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos,
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start)
{
CDiskBlockPos block_pos;
FlatFilePos block_pos;
{
LOCK(cs_main);
block_pos = pindex->GetBlockPos();
@ -1452,7 +1455,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi
namespace {
bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
@ -1481,7 +1484,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint
static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex)
{
CDiskBlockPos pos = pindex->GetUndoPos();
FlatFilePos pos = pindex->GetUndoPos();
if (pos.IsNull()) {
return error("%s: no undo data available", __func__);
}
@ -1627,37 +1630,24 @@ void static FlushBlockFile(bool fFinalize = false)
{
LOCK(cs_LastBlockFile);
CDiskBlockPos posOld(nLastBlockFile, 0);
FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize);
FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize);
bool status = true;
FILE *fileOld = OpenBlockFile(posOld);
if (fileOld) {
if (fFinalize)
status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize);
status &= FileCommit(fileOld);
fclose(fileOld);
}
fileOld = OpenUndoFile(posOld);
if (fileOld) {
if (fFinalize)
status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize);
status &= FileCommit(fileOld);
fclose(fileOld);
}
status &= BlockFileSeq().Flush(block_pos_old, fFinalize);
status &= UndoFileSeq().Flush(undo_pos_old, fFinalize);
if (!status) {
AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
}
}
static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize);
static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize);
static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams)
{
// Write undo information to disk
if (pindex->GetUndoPos().IsNull()) {
CDiskBlockPos _pos;
FlatFilePos _pos;
if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40))
return error("ConnectBlock(): FindUndoPos failed");
if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart()))
@ -2134,8 +2124,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// Write blocks and block index to disk.
if (fDoFullFlush || fPeriodicWrite) {
// Depend on nMinDiskSpace to ensure we can write block index
if (!CheckDiskSpace(0, true))
return state.Error("out of disk space");
if (!CheckDiskSpace(GetBlocksDir())) {
return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
}
// First make sure all block and undo data is flushed to disk.
FlushBlockFile();
// Then update all block file information (which may refer to block and undo files).
@ -2168,8 +2159,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// twice (once in the log, and once in the tables). This is already
// an overestimation, as most will delete an existing entry or
// overwrite one. Still, use a conservative safety factor of 2.
if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize()))
return state.Error("out of disk space");
if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) {
return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
}
// Flush the chainstate (which may refer to block index entries).
if (!pcoinsTip->Flush())
return AbortNode(state, "Failed to write to coin database");
@ -2922,7 +2914,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block)
}
/** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */
void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams)
void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams)
{
pindexNew->nTx = block.vtx.size();
pindexNew->nChainTx = 0;
@ -2968,7 +2960,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi
}
}
static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false)
static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false)
{
LOCK(cs_LastBlockFile);
@ -3003,21 +2995,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
vinfoBlockFile[nFile].nSize += nAddSize;
if (!fKnown) {
unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
if (fPruneMode)
fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) {
FILE *file = OpenBlockFile(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile);
AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos);
fclose(file);
}
}
else
return error("out of disk space");
bool out_of_space;
size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode("Disk space is low!", _("Error: Disk space is low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
}
@ -3025,32 +3009,23 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
return true;
}
static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize)
static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize)
{
pos.nFile = nFile;
LOCK(cs_LastBlockFile);
unsigned int nNewSize;
pos.nPos = vinfoBlockFile[nFile].nUndoSize;
nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize;
vinfoBlockFile[nFile].nUndoSize += nAddSize;
setDirtyFileInfo.insert(nFile);
unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
if (fPruneMode)
fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) {
FILE *file = OpenUndoFile(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile);
AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos);
fclose(file);
}
}
else
return state.Error("out of disk space");
bool out_of_space;
size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
return true;
@ -3435,26 +3410,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio
}
/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */
static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) {
static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) {
unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION);
CDiskBlockPos blockPos;
FlatFilePos blockPos;
if (dbp != nullptr)
blockPos = *dbp;
if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) {
error("%s: FindBlockPos failed", __func__);
return CDiskBlockPos();
return FlatFilePos();
}
if (dbp == nullptr) {
if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) {
AbortNode("Failed to write block");
return CDiskBlockPos();
return FlatFilePos();
}
}
return blockPos;
}
/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */
bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock)
bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock)
{
const CBlock& block = *pblock;
@ -3516,7 +3491,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali
// Write block to history file
if (fNewBlock) *fNewBlock = true;
try {
CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp);
FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp);
if (blockPos.IsNull()) {
state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__));
return false;
@ -3647,9 +3622,9 @@ void PruneOneBlockFile(const int fileNumber)
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
{
for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
CDiskBlockPos pos(*it, 0);
fs::remove(GetBlockPosFilename(pos, "blk"));
fs::remove(GetBlockPosFilename(pos, "rev"));
FlatFilePos pos(*it, 0);
fs::remove(BlockFileSeq().FileName(pos));
fs::remove(UndoFileSeq().FileName(pos));
LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
}
}
@ -3757,52 +3732,28 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte
nLastBlockWeCanPrune, count);
}
bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir)
static FlatFileSeq BlockFileSeq()
{
uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available;
// Check for nMinDiskSpace bytes (currently 50MB)
if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes)
return AbortNode("Disk space is low!", _("Error: Disk space is low!"));
return true;
return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE);
}
static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly)
static FlatFileSeq UndoFileSeq()
{
if (pos.IsNull())
return nullptr;
fs::path path = GetBlockPosFilename(pos, prefix);
fs::create_directories(path.parent_path());
FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+");
if (!file && !fReadOnly)
file = fsbridge::fopen(path, "wb+");
if (!file) {
LogPrintf("Unable to open file %s\n", path.string());
return nullptr;
}
if (pos.nPos) {
if (fseek(file, pos.nPos, SEEK_SET)) {
LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
fclose(file);
return nullptr;
}
}
return file;
return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE);
}
FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) {
return OpenDiskFile(pos, "blk", fReadOnly);
FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) {
return BlockFileSeq().Open(pos, fReadOnly);
}
/** Open an undo file (rev?????.dat) */
static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) {
return OpenDiskFile(pos, "rev", fReadOnly);
static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) {
return UndoFileSeq().Open(pos, fReadOnly);
}
fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix)
fs::path GetBlockPosFilename(const FlatFilePos &pos)
{
return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile);
return BlockFileSeq().FileName(pos);
}
CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash)
@ -3909,7 +3860,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE
}
for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++)
{
CDiskBlockPos pos(*it, 0);
FlatFilePos pos(*it, 0);
if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) {
return false;
}
@ -4342,7 +4293,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams)
try {
const CBlock& block = chainparams.GenesisBlock();
CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr);
FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr);
if (blockPos.IsNull())
return error("%s: writing genesis block to disk failed", __func__);
CBlockIndex *pindex = AddToBlockIndex(block);
@ -4359,10 +4310,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams)
return g_chainstate.LoadGenesisBlock(chainparams);
}
bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp)
bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp)
{
// Map of disk positions for blocks with unknown parent (only used for reindex)
static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent;
static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent;
int64_t nStart = GetTimeMillis();
int nLoaded = 0;
@ -4448,9 +4399,9 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB
while (!queue.empty()) {
uint256 head = queue.front();
queue.pop_front();
std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head);
std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head);
while (range.first != range.second) {
std::multimap<uint256, CDiskBlockPos>::iterator it = range.first;
std::multimap<uint256, FlatFilePos>::iterator it = range.first;
std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>();
if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus()))
{

15
src/validation.h
View file

@ -181,9 +181,6 @@ extern arith_uint256 nMinimumChainWork;
/** Best header we've seen so far (used for getheaders queries' starting points). */
extern CBlockIndex *pindexBestHeader;
/** Minimum disk space required - used in CheckDiskSpace() */
static const uint64_t nMinDiskSpace = 52428800;
/** Pruning-related variables and constants */
/** True if any block files have ever been pruned. */
extern bool fHavePruned;
@ -245,14 +242,12 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons
*/
bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main);
/** Check whether enough disk space is available for an incoming block */
bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false);
/** Open a block file (blk?????.dat) */
FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false);
FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false);
/** Translation to a filesystem path */
fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix);
fs::path GetBlockPosFilename(const FlatFilePos &pos);
/** Import blocks from an external file */
bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr);
bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr);
/** Ensures we have a genesis block in the block tree, possibly writing one to disk. */
bool LoadGenesisBlock(const CChainParams& chainparams);
/** Load the block tree and coins database from disk,
@ -391,9 +386,9 @@ void InitScriptExecutionCache();
/** Functions for disk access for blocks */
bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams);
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams);
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams);
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start);
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start);
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start);
/** Functions for validating blocks and updating the block tree */