maxmustermann/dogecoin
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge #16899: UTXO snapshot creation (dumptxoutset)

Browse source 
92b2f5306b test: add dumptxoutset RPC test (James O'Beirne)
c1ccbc3dde devtools: add utxo_snapshot.sh (James O'Beirne)
57cf74c991 rpc: add dumptxoutset (James O'Beirne)
92fafb3a7d coinstats: add coins_count (James O'Beirne)
707fde7b9b add unused SnapshotMetadata class (James O'Beirne)

Pull request description:

  This is part of the [assumeutxo project](https://github.com/bitcoin/bitcoin/projects/11):

  Parent PR: #15606
  Issue: #15605
  Specification: https://github.com/jamesob/assumeutxo-docs/tree/master/proposal

  ---

  This changeset defines the serialization format for UTXO snapshots and adds an RPC command for creating them, `dumptxoutset`. It also adds a convenience script for generating and verifying snapshots at a certain height, since that requires doing a hacky rewind of the chain via `invalidateblock`.

  All of this is unused at the moment.

ACKs for top commit:
  laanwj:
    ACK 92b2f5306b

Tree-SHA512: 200dff87767f157d627e99506ec543465d9329860a6cd49363081619c437163a640a46d008faa92b1f44fd403bfc7a7c9e851c658b5a4849efa9a34ca976bf31
This commit is contained in:
Wladimir J. van der Laan 2019-11-05 19:38:45 +01:00
parent d35b12107e 92b2f5306b
commit b05b28183c
No known key found for this signature in database
GPG key ID: 1E4AED62986CD25D
9 changed files with 269 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
contrib/devtools/utxo_snapshot.sh Executable file
View file

@ -0,0 +1,44 @@
#!/usr/bin/env bash
#
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
export LC_ALL=C
set -ueo pipefail
if (( $# < 3 )); then
echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>'
echo
echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the "
echo " expected assumeutxo hash"
echo
echo 'Examples:'
echo
echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata"
echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli'
exit 1
fi
GENERATE_AT_HEIGHT="${1}"; shift;
OUTPUT_PATH="${1}"; shift;
# Most of the calls we make take a while to run, so pad with a lengthy timeout.
BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999"
# Block we'll invalidate/reconsider to rewind/fast-forward the chain.
PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) )
(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while")
${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}"
if [[ "${OUTPUT_PATH}" = "-" ]]; then
(>&2 echo "Generating txoutset info...")
${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g'
else
(>&2 echo "Generating UTXO snapshot...")
${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}"
fi
(>&2 echo "Restoring chain to original height; this may take a while")
${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}"

1
src/Makefile.am
View file

@ -161,6 +161,7 @@ BITCOIN_CORE_H = \
node/context.h \
node/psbt.h \
node/transaction.h \
node/utxo_snapshot.h \
noui.h \
optional.h \
outputtype.h \

2
src/node/coinstats.cpp
View file

@ -38,6 +38,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash,
//! Calculate statistics about the unspent transaction output set
bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
{
stats = CCoinsStats();
std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor());
assert(pcursor);
@ -61,6 +62,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
}
prevkey = key.hash;
outputs[key.n] = std::move(coin);
stats.coins_count++;
} else {
return error("%s: unable to read value", __func__);
}

19
src/node/coinstats.h
View file

@ -15,16 +15,17 @@ class CCoinsView;
struct CCoinsStats
{
int nHeight;
uint256 hashBlock;
uint64_t nTransactions;
uint64_t nTransactionOutputs;
uint64_t nBogoSize;
uint256 hashSerialized;
uint64_t nDiskSize;
CAmount nTotalAmount;
int nHeight{0};
uint256 hashBlock{};
uint64_t nTransactions{0};
uint64_t nTransactionOutputs{0};
uint64_t nBogoSize{0};
uint256 hashSerialized{};
uint64_t nDiskSize{0};
CAmount nTotalAmount{0};
CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {}
//! The number of coins contained.
uint64_t coins_count{0};
};
//! Calculate statistics about the unspent transaction output set

50
src/node/utxo_snapshot.h Normal file
View file

@ -0,0 +1,50 @@
// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H
#define BITCOIN_NODE_UTXO_SNAPSHOT_H
#include <uint256.h>
#include <serialize.h>
//! Metadata describing a serialized version of a UTXO set from which an
//! assumeutxo CChainState can be constructed.
class SnapshotMetadata
{
public:
//! The hash of the block that reflects the tip of the chain for the
//! UTXO set contained in this snapshot.
uint256 m_base_blockhash;
//! The number of coins in the UTXO set contained in this snapshot. Used
//! during snapshot load to estimate progress of UTXO set reconstruction.
uint64_t m_coins_count = 0;
//! Necessary to "fake" the base nChainTx so that we can estimate progress during
//! initial block download for the assumeutxo chainstate.
unsigned int m_nchaintx = 0;
SnapshotMetadata() { }
SnapshotMetadata(
const uint256& base_blockhash,
uint64_t coins_count,
unsigned int nchaintx) :
m_base_blockhash(base_blockhash),
m_coins_count(coins_count),
m_nchaintx(nchaintx) { }
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action)
{
READWRITE(m_base_blockhash);
READWRITE(m_coins_count);
READWRITE(m_nchaintx);
}
};
#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H

109
src/rpc/blockchain.cpp
View file

@ -15,6 +15,7 @@
#include <hash.h>
#include <index/blockfilterindex.h>
#include <node/coinstats.h>
#include <node/utxo_snapshot.h>
#include <policy/feerate.h>
#include <policy/policy.h>
#include <policy/rbf.h>
@ -2245,6 +2246,113 @@ static UniValue getblockfilter(const JSONRPCRequest& request)
return ret;
}
/**
* Serialize the UTXO set to a file for loading elsewhere.
*
* @see SnapshotMetadata
*/
UniValue dumptxoutset(const JSONRPCRequest& request)
{
RPCHelpMan{
"dumptxoutset",
"\nWrite the serialized UTXO set to disk.\n"
"Incidentally flushes the latest coinsdb (leveldb) to disk.\n",
{
{"path",
RPCArg::Type::STR,
RPCArg::Optional::NO,
/* default_val */ "",
"path to the output file. If relative, will be prefixed by datadir."},
},
RPCResult{
"{\n"
" \"coins_written\": n, (numeric) the number of coins written in the snapshot\n"
" \"base_hash\": \"...\", (string) the hash of the base of the snapshot\n"
" \"base_height\": n, (string) the height of the base of the snapshot\n"
" \"path\": \"...\" (string) the absolute path that the snapshot was written to\n"
"]\n"
},
RPCExamples{
HelpExampleCli("dumptxoutset", "utxo.dat")
}
}.Check(request);
fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir());
// Write to a temporary path and then move into `path` on completion
// to avoid confusion due to an interruption.
fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir());
if (fs::exists(path)) {
throw JSONRPCError(
RPC_INVALID_PARAMETER,
path.string() + " already exists. If you are sure this is what you want, "
"move it out of the way first");
}
FILE* file{fsbridge::fopen(temppath, "wb")};
CAutoFile afile{file, SER_DISK, CLIENT_VERSION};
std::unique_ptr<CCoinsViewCursor> pcursor;
CCoinsStats stats;
CBlockIndex* tip;
{
// We need to lock cs_main to ensure that the coinsdb isn't written to
// between (i) flushing coins cache to disk (coinsdb), (ii) getting stats
// based upon the coinsdb, and (iii) constructing a cursor to the
// coinsdb for use below this block.
//
// Cursors returned by leveldb iterate over snapshots, so the contents
// of the pcursor will not be affected by simultaneous writes during
// use below this block.
//
// See discussion here:
// https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369
//
LOCK(::cs_main);
::ChainstateActive().ForceFlushStateToDisk();
if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) {
throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set");
}
pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor());
tip = LookupBlockIndex(stats.hashBlock);
CHECK_NONFATAL(tip);
}
SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx};
afile << metadata;
COutPoint key;
Coin coin;
unsigned int iter{0};
while (pcursor->Valid()) {
if (iter % 5000 == 0 && !IsRPCRunning()) {
throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down");
}
++iter;
if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
afile << key;
afile << coin;
}
pcursor->Next();
}
afile.fclose();
fs::rename(temppath, path);
UniValue result(UniValue::VOBJ);
result.pushKV("coins_written", stats.coins_count);
result.pushKV("base_hash", tip->GetBlockHash().ToString());
result.pushKV("base_height", tip->nHeight);
result.pushKV("path", path.string());
return result;
}
// clang-format off
static const CRPCCommand commands[] =
{ // category name actor (function) argNames
@ -2281,6 +2389,7 @@ static const CRPCCommand commands[] =
{ "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} },
{ "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} },
{ "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} },
{ "hidden", "dumptxoutset", &dumptxoutset, {"path"} },
};
// clang-format on

1
src/validation.h
View file

@ -21,6 +21,7 @@
#include <txmempool.h> // For CTxMemPool::cs
#include <txdb.h>
#include <versionbits.h>
#include <serialize.h>
#include <algorithm>
#include <atomic>

51
test/functional/rpc_dumptxoutset.py Executable file
View file

@ -0,0 +1,51 @@
#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the generation of UTXO snapshots using `dumptxoutset`.
"""
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import assert_equal, assert_raises_rpc_error
import hashlib
from pathlib import Path
class DumptxoutsetTest(BitcoinTestFramework):
def set_test_params(self):
self.setup_clean_chain = True
self.num_nodes = 1
def run_test(self):
"""Test a trivial usage of the dumptxoutset RPC command."""
node = self.nodes[0]
mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1
node.setmocktime(mocktime)
node.generate(100)
FILENAME = 'txoutset.dat'
out = node.dumptxoutset(FILENAME)
expected_path = Path(node.datadir) / 'regtest' / FILENAME
assert expected_path.is_file()
assert_equal(out['coins_written'], 100)
assert_equal(out['base_height'], 100)
assert_equal(out['path'], str(expected_path))
# Blockhash should be deterministic based on mocked time.
assert_equal(
out['base_hash'],
'6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6')
with open(str(expected_path), 'rb') as f:
digest = hashlib.sha256(f.read()).hexdigest()
# UTXO snapshot hash should be deterministic based on mocked time.
assert_equal(
digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664')
# Specifying a path to an existing file will fail.
assert_raises_rpc_error(
-8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME)
if __name__ == '__main__':
DumptxoutsetTest().main()

1
test/functional/test_runner.py
View file

@ -191,6 +191,7 @@ BASE_SCRIPTS = [
'rpc_uptime.py',
'wallet_resendwallettransactions.py',
'wallet_fallbackfee.py',
'rpc_dumptxoutset.py',
'feature_minchainwork.py',
'rpc_getblockstats.py',
'wallet_create_tx.py',