From 707fde7b9ba522c22179e2db0ed7b462c65138d9 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Mar 2019 12:08:32 -0400 Subject: [PATCH 1/5] add unused SnapshotMetadata class --- src/Makefile.am | 1 + src/node/utxo_snapshot.h | 50 ++++++++++++++++++++++++++++++++++++++++ src/validation.h | 1 + 3 files changed, 52 insertions(+) create mode 100644 src/node/utxo_snapshot.h diff --git a/src/Makefile.am b/src/Makefile.am index 3fb1a1f98..19d41b78c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -159,6 +159,7 @@ BITCOIN_CORE_H = \ node/coinstats.h \ node/psbt.h \ node/transaction.h \ + node/utxo_snapshot.h \ noui.h \ optional.h \ outputtype.h \ diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 000000000..702a0cbe5 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,50 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include +#include + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata +{ +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress during + //! initial block download for the assumeutxo chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { } + SnapshotMetadata( + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) { } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } + +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/validation.h b/src/validation.h index d17a320a4..a539ab546 100644 --- a/src/validation.h +++ b/src/validation.h @@ -21,6 +21,7 @@ #include // For CTxMemPool::cs #include #include +#include #include #include From 92fafb3a7da66f737e960e541fcfbcadedf6043a Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Mar 2019 12:09:06 -0400 Subject: [PATCH 2/5] coinstats: add coins_count Also changes existing CCoinsStats attributes to be C++11 initialized. --- src/node/coinstats.cpp | 2 ++ src/node/coinstats.h | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 57fa158ad..23269cd4b 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -38,6 +38,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr pcursor(view->Cursor()); assert(pcursor); @@ -61,6 +62,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) } prevkey = key.hash; outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7c11aab8b..a19af0fd1 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -15,16 +15,17 @@ class CCoinsView; struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - CAmount nTotalAmount; + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint64_t nBogoSize{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + CAmount nTotalAmount{0}; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} + //! The number of coins contained. + uint64_t coins_count{0}; }; //! Calculate statistics about the unspent transaction output set From 57cf74c9918d10c69a46e6ceb3cb1a5e04edf5bc Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Mar 2019 12:10:33 -0400 Subject: [PATCH 3/5] rpc: add dumptxoutset Allows the creation of a UTXO snapshot to disk. --- src/rpc/blockchain.cpp | 109 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index fac6bcd60..de5649de2 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -2245,6 +2246,113 @@ static UniValue getblockfilter(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "dumptxoutset", + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + { + {"path", + RPCArg::Type::STR, + RPCArg::Optional::NO, + /* default_val */ "", + "path to the output file. If relative, will be prefixed by datadir."}, + }, + RPCResult{ + "{\n" + " \"coins_written\": n, (numeric) the number of coins written in the snapshot\n" + " \"base_hash\": \"...\", (string) the hash of the base of the snapshot\n" + " \"base_height\": n, (string) the height of the base of the snapshot\n" + " \"path\": \"...\" (string) the absolute path that the snapshot was written to\n" + "]\n" + }, + RPCExamples{ + HelpExampleCli("dumptxoutset", "utxo.dat") + } + }.Check(request); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + // Write to a temporary path and then move into `path` on completion + // to avoid confusion due to an interruption. + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " already exists. If you are sure this is what you want, " + "move it out of the way first"); + } + + FILE* file{fsbridge::fopen(temppath, "wb")}; + CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + std::unique_ptr pcursor; + CCoinsStats stats; + CBlockIndex* tip; + + { + // We need to lock cs_main to ensure that the coinsdb isn't written to + // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats + // based upon the coinsdb, and (iii) constructing a cursor to the + // coinsdb for use below this block. + // + // Cursors returned by leveldb iterate over snapshots, so the contents + // of the pcursor will not be affected by simultaneous writes during + // use below this block. + // + // See discussion here: + // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 + // + LOCK(::cs_main); + + ::ChainstateActive().ForceFlushStateToDisk(); + + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr(::ChainstateActive().CoinsDB().Cursor()); + tip = LookupBlockIndex(stats.hashBlock); + CHECK_NONFATAL(tip); + } + + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx}; + + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter{0}; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } + + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().ToString()); + result.pushKV("base_height", tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2281,6 +2389,7 @@ static const CRPCCommand commands[] = { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, + { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, }; // clang-format on From c1ccbc3ddef931896a7e9dcfa6704e305a69fbff Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 25 Apr 2019 15:32:09 -0400 Subject: [PATCH 4/5] devtools: add utxo_snapshot.sh to allow easy (if not time-consuming) generation and verification of snapshots. --- contrib/devtools/utxo_snapshot.sh | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 contrib/devtools/utxo_snapshot.sh diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 index 000000000..dee25ff67 --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh ' + echo + echo " if is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" From 92b2f5306ba0b3f031293cb8f415b67cb002c2f1 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Tue, 17 Sep 2019 10:57:46 -0400 Subject: [PATCH 5/5] test: add dumptxoutset RPC test --- test/functional/rpc_dumptxoutset.py | 51 +++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 2 files changed, 52 insertions(+) create mode 100755 test/functional/rpc_dumptxoutset.py diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py new file mode 100755 index 000000000..7527bdfb0 --- /dev/null +++ b/test/functional/rpc_dumptxoutset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the generation of UTXO snapshots using `dumptxoutset`. +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +import hashlib +from pathlib import Path + + +class DumptxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the dumptxoutset RPC command.""" + node = self.nodes[0] + mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 + node.setmocktime(mocktime) + node.generate(100) + + FILENAME = 'txoutset.dat' + out = node.dumptxoutset(FILENAME) + expected_path = Path(node.datadir) / 'regtest' / FILENAME + + assert expected_path.is_file() + + assert_equal(out['coins_written'], 100) + assert_equal(out['base_height'], 100) + assert_equal(out['path'], str(expected_path)) + # Blockhash should be deterministic based on mocked time. + assert_equal( + out['base_hash'], + '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + + with open(str(expected_path), 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + # UTXO snapshot hash should be deterministic based on mocked time. + assert_equal( + digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664') + + # Specifying a path to an existing file will fail. + assert_raises_rpc_error( + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + +if __name__ == '__main__': + DumptxoutsetTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index e0b523b71..2938c54a3 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -190,6 +190,7 @@ BASE_SCRIPTS = [ 'rpc_uptime.py', 'wallet_resendwallettransactions.py', 'wallet_fallbackfee.py', + 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_getblockstats.py', 'wallet_create_tx.py',