Merge pull request #5863
f9ec3f0 Add block pruning functionality (mrbandrews)This commit is contained in:
Wladimir J. van der Laan
commit
c2713042a3
356
qa/rpc-tests/pruning.py
Executable file
356
qa/rpc-tests/pruning.py
Executable file
|
|
@ -0,0 +1,356 @@
|
|||
#!/usr/bin/env python2
|
||||
# Copyright (c) 2014 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#
|
||||
# Test pruning code
|
||||
# ********
|
||||
# WARNING:
|
||||
# This test uses 4GB of disk space and takes in excess of 30 mins to run
|
||||
# ********
|
||||
|
||||
from test_framework import BitcoinTestFramework
|
||||
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
|
||||
from util import *
|
||||
import os.path
|
||||
|
||||
def calc_usage(blockdir):
|
||||
return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f))/(1024*1024)
|
||||
|
||||
class PruneTest(BitcoinTestFramework):
|
||||
|
||||
def __init__(self):
|
||||
self.utxo = []
|
||||
self.address = ["",""]
|
||||
|
||||
# Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create
|
||||
# So we have big transactions and full blocks to fill up our block files
|
||||
|
||||
# create one script_pubkey
|
||||
script_pubkey = "6a4d0200" #OP_RETURN OP_PUSH2 512 bytes
|
||||
for i in xrange (512):
|
||||
script_pubkey = script_pubkey + "01"
|
||||
# concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change
|
||||
self.txouts = "81"
|
||||
for k in xrange(128):
|
||||
# add txout value
|
||||
self.txouts = self.txouts + "0000000000000000"
|
||||
# add length of script_pubkey
|
||||
self.txouts = self.txouts + "fd0402"
|
||||
# add script_pubkey
|
||||
self.txouts = self.txouts + script_pubkey
|
||||
|
||||
|
||||
def setup_chain(self):
|
||||
print("Initializing test directory "+self.options.tmpdir)
|
||||
initialize_chain_clean(self.options.tmpdir, 3)
|
||||
|
||||
def setup_network(self):
|
||||
self.nodes = []
|
||||
self.is_network_split = False
|
||||
|
||||
# Create nodes 0 and 1 to mine
|
||||
self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300))
|
||||
self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300))
|
||||
|
||||
# Create node 2 to test pruning
|
||||
self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=300))
|
||||
self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"
|
||||
|
||||
self.address[0] = self.nodes[0].getnewaddress()
|
||||
self.address[1] = self.nodes[1].getnewaddress()
|
||||
|
||||
connect_nodes(self.nodes[0], 1)
|
||||
connect_nodes(self.nodes[1], 2)
|
||||
connect_nodes(self.nodes[2], 0)
|
||||
sync_blocks(self.nodes[0:3])
|
||||
|
||||
def create_big_chain(self):
|
||||
# Start by creating some coinbases we can spend later
|
||||
self.nodes[1].generate(200)
|
||||
sync_blocks(self.nodes[0:2])
|
||||
self.nodes[0].generate(150)
|
||||
# Then mine enough full blocks to create more than 550MB of data
|
||||
for i in xrange(645):
|
||||
self.mine_full_block(self.nodes[0], self.address[0])
|
||||
|
||||
sync_blocks(self.nodes[0:3])
|
||||
|
||||
def test_height_min(self):
|
||||
if not os.path.isfile(self.prunedir+"blk00000.dat"):
|
||||
raise AssertionError("blk00000.dat is missing, pruning too early")
|
||||
print "Success"
|
||||
print "Though we're already using more than 550MB, current usage:", calc_usage(self.prunedir)
|
||||
print "Mining 25 more blocks should cause the first block file to be pruned"
|
||||
# Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
|
||||
for i in xrange(25):
|
||||
self.mine_full_block(self.nodes[0],self.address[0])
|
||||
|
||||
waitstart = time.time()
|
||||
while os.path.isfile(self.prunedir+"blk00000.dat"):
|
||||
time.sleep(0.1)
|
||||
if time.time() - waitstart > 10:
|
||||
raise AssertionError("blk00000.dat not pruned when it should be")
|
||||
|
||||
print "Success"
|
||||
usage = calc_usage(self.prunedir)
|
||||
print "Usage should be below target:", usage
|
||||
if (usage > 550):
|
||||
raise AssertionError("Pruning target not being met")
|
||||
|
||||
def create_chain_with_staleblocks(self):
|
||||
# Create stale blocks in manageable sized chunks
|
||||
print "Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds"
|
||||
|
||||
for j in xrange(12):
|
||||
# Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
|
||||
# Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
|
||||
# Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine
|
||||
stop_node(self.nodes[0],0)
|
||||
self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)
|
||||
# Mine 24 blocks in node 1
|
||||
self.utxo = self.nodes[1].listunspent()
|
||||
for i in xrange(24):
|
||||
if j == 0:
|
||||
self.mine_full_block(self.nodes[1],self.address[1])
|
||||
else:
|
||||
self.nodes[1].generate(1) #tx's already in mempool from previous disconnects
|
||||
|
||||
# Reorg back with 25 block chain from node 0
|
||||
self.utxo = self.nodes[0].listunspent()
|
||||
for i in xrange(25):
|
||||
self.mine_full_block(self.nodes[0],self.address[0])
|
||||
|
||||
# Create connections in the order so both nodes can see the reorg at the same time
|
||||
connect_nodes(self.nodes[1], 0)
|
||||
connect_nodes(self.nodes[2], 0)
|
||||
sync_blocks(self.nodes[0:3])
|
||||
|
||||
print "Usage can be over target because of high stale rate:", calc_usage(self.prunedir)
|
||||
|
||||
def reorg_test(self):
|
||||
# Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
|
||||
# This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
|
||||
# Reboot node 1 to clear its mempool (hopefully make the invalidate faster)
|
||||
# Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks)
|
||||
stop_node(self.nodes[1],1)
|
||||
self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300)
|
||||
|
||||
height = self.nodes[1].getblockcount()
|
||||
print "Current block height:", height
|
||||
|
||||
invalidheight = height-287
|
||||
badhash = self.nodes[1].getblockhash(invalidheight)
|
||||
print "Invalidating block at height:",invalidheight,badhash
|
||||
self.nodes[1].invalidateblock(badhash)
|
||||
|
||||
# We've now switched to our previously mined-24 block fork on node 1, but thats not what we want
|
||||
# So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
|
||||
mainchainhash = self.nodes[0].getblockhash(invalidheight - 1)
|
||||
curhash = self.nodes[1].getblockhash(invalidheight - 1)
|
||||
while curhash != mainchainhash:
|
||||
self.nodes[1].invalidateblock(curhash)
|
||||
curhash = self.nodes[1].getblockhash(invalidheight - 1)
|
||||
|
||||
assert(self.nodes[1].getblockcount() == invalidheight - 1)
|
||||
print "New best height", self.nodes[1].getblockcount()
|
||||
|
||||
# Reboot node1 to clear those giant tx's from mempool
|
||||
stop_node(self.nodes[1],1)
|
||||
self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300)
|
||||
|
||||
print "Generating new longer chain of 300 more blocks"
|
||||
self.nodes[1].generate(300)
|
||||
|
||||
print "Reconnect nodes"
|
||||
connect_nodes(self.nodes[0], 1)
|
||||
connect_nodes(self.nodes[2], 1)
|
||||
sync_blocks(self.nodes[0:3])
|
||||
|
||||
print "Verify height on node 2:",self.nodes[2].getblockcount()
|
||||
print "Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir)
|
||||
|
||||
print "Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)"
|
||||
self.nodes[0].generate(220) #node 0 has many large tx's in its mempool from the disconnects
|
||||
sync_blocks(self.nodes[0:3])
|
||||
|
||||
usage = calc_usage(self.prunedir)
|
||||
print "Usage should be below target:", usage
|
||||
if (usage > 550):
|
||||
raise AssertionError("Pruning target not being met")
|
||||
|
||||
return invalidheight,badhash
|
||||
|
||||
def reorg_back(self):
|
||||
# Verify that a block on the old main chain fork has been pruned away
|
||||
try:
|
||||
self.nodes[2].getblock(self.forkhash)
|
||||
raise AssertionError("Old block wasn't pruned so can't test redownload")
|
||||
except JSONRPCException as e:
|
||||
print "Will need to redownload block",self.forkheight
|
||||
|
||||
# Verify that we have enough history to reorg back to the fork point
|
||||
# Although this is more than 288 blocks, because this chain was written more recently
|
||||
# and only its other 299 small and 220 large block are in the block files after it,
|
||||
# its expected to still be retained
|
||||
self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight))
|
||||
|
||||
first_reorg_height = self.nodes[2].getblockcount()
|
||||
curchainhash = self.nodes[2].getblockhash(self.mainchainheight)
|
||||
self.nodes[2].invalidateblock(curchainhash)
|
||||
goalbestheight = self.mainchainheight
|
||||
goalbesthash = self.mainchainhash2
|
||||
|
||||
# As of 0.10 the current block download logic is not able to reorg to the original chain created in
|
||||
# create_chain_with_stale_blocks because it doesn't know of any peer thats on that chain from which to
|
||||
# redownload its missing blocks.
|
||||
# Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain
|
||||
# because it has all the block data.
|
||||
# However it must mine enough blocks to have a more work chain than the reorg_test chain in order
|
||||
# to trigger node 2's block download logic.
|
||||
# At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg
|
||||
if self.nodes[2].getblockcount() < self.mainchainheight:
|
||||
blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
|
||||
print "Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine
|
||||
self.nodes[0].invalidateblock(curchainhash)
|
||||
assert(self.nodes[0].getblockcount() == self.mainchainheight)
|
||||
assert(self.nodes[0].getbestblockhash() == self.mainchainhash2)
|
||||
goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
|
||||
goalbestheight = first_reorg_height + 1
|
||||
|
||||
print "Verify node 2 reorged back to the main chain, some blocks of which it had to redownload"
|
||||
waitstart = time.time()
|
||||
while self.nodes[2].getblockcount() < goalbestheight:
|
||||
time.sleep(0.1)
|
||||
if time.time() - waitstart > 300:
|
||||
raise AssertionError("Node 2 didn't reorg to proper height")
|
||||
assert(self.nodes[2].getbestblockhash() == goalbesthash)
|
||||
# Verify we can now have the data for a block previously pruned
|
||||
assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)
|
||||
|
||||
def mine_full_block(self, node, address):
|
||||
# Want to create a full block
|
||||
# We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit
|
||||
for j in xrange(14):
|
||||
if len(self.utxo) < 14:
|
||||
self.utxo = node.listunspent()
|
||||
inputs=[]
|
||||
outputs = {}
|
||||
t = self.utxo.pop()
|
||||
inputs.append({ "txid" : t["txid"], "vout" : t["vout"]})
|
||||
remchange = t["amount"] - Decimal("0.001000")
|
||||
outputs[address]=remchange
|
||||
# Create a basic transaction that will send change back to ourself after account for a fee
|
||||
# And then insert the 128 generated transaction outs in the middle rawtx[92] is where the #
|
||||
# of txouts is stored and is the only thing we overwrite from the original transaction
|
||||
rawtx = node.createrawtransaction(inputs, outputs)
|
||||
newtx = rawtx[0:92]
|
||||
newtx = newtx + self.txouts
|
||||
newtx = newtx + rawtx[94:]
|
||||
# Appears to be ever so slightly faster to sign with SIGHASH_NONE
|
||||
signresult = node.signrawtransaction(newtx,None,None,"NONE")
|
||||
txid = node.sendrawtransaction(signresult["hex"], True)
|
||||
# Mine a full sized block which will be these transactions we just created
|
||||
node.generate(1)
|
||||
|
||||
|
||||
def run_test(self):
|
||||
print "Warning! This test requires 4GB of disk space and takes over 30 mins"
|
||||
print "Mining a big blockchain of 995 blocks"
|
||||
self.create_big_chain()
|
||||
# Chain diagram key:
|
||||
# * blocks on main chain
|
||||
# +,&,$,@ blocks on other forks
|
||||
# X invalidated block
|
||||
# N1 Node 1
|
||||
#
|
||||
# Start by mining a simple chain that all nodes have
|
||||
# N0=N1=N2 **...*(995)
|
||||
|
||||
print "Check that we haven't started pruning yet because we're below PruneAfterHeight"
|
||||
self.test_height_min()
|
||||
# Extend this chain past the PruneAfterHeight
|
||||
# N0=N1=N2 **...*(1020)
|
||||
|
||||
print "Check that we'll exceed disk space target if we have a very high stale block rate"
|
||||
self.create_chain_with_staleblocks()
|
||||
# Disconnect N0
|
||||
# And mine a 24 block chain on N1 and a separate 25 block chain on N0
|
||||
# N1=N2 **...*+...+(1044)
|
||||
# N0 **...**...**(1045)
|
||||
#
|
||||
# reconnect nodes causing reorg on N1 and N2
|
||||
# N1=N2 **...*(1020) *...**(1045)
|
||||
# \
|
||||
# +...+(1044)
|
||||
#
|
||||
# repeat this process until you have 12 stale forks hanging off the
|
||||
# main chain on N1 and N2
|
||||
# N0 *************************...***************************(1320)
|
||||
#
|
||||
# N1=N2 **...*(1020) *...**(1045) *.. ..**(1295) *...**(1320)
|
||||
# \ \ \
|
||||
# +...+(1044) &.. $...$(1319)
|
||||
|
||||
# Save some current chain state for later use
|
||||
self.mainchainheight = self.nodes[2].getblockcount() #1320
|
||||
self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
|
||||
|
||||
print "Check that we can survive a 288 block reorg still"
|
||||
(self.forkheight,self.forkhash) = self.reorg_test() #(1033, )
|
||||
# Now create a 288 block reorg by mining a longer chain on N1
|
||||
# First disconnect N1
|
||||
# Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain
|
||||
# N1 **...*(1020) **...**(1032)X..
|
||||
# \
|
||||
# ++...+(1031)X..
|
||||
#
|
||||
# Now mine 300 more blocks on N1
|
||||
# N1 **...*(1020) **...**(1032) @@...@(1332)
|
||||
# \ \
|
||||
# \ X...
|
||||
# \ \
|
||||
# ++...+(1031)X.. ..
|
||||
#
|
||||
# Reconnect nodes and mine 220 more blocks on N1
|
||||
# N1 **...*(1020) **...**(1032) @@...@@@(1552)
|
||||
# \ \
|
||||
# \ X...
|
||||
# \ \
|
||||
# ++...+(1031)X.. ..
|
||||
#
|
||||
# N2 **...*(1020) **...**(1032) @@...@@@(1552)
|
||||
# \ \
|
||||
# \ *...**(1320)
|
||||
# \ \
|
||||
# ++...++(1044) ..
|
||||
#
|
||||
# N0 ********************(1032) @@...@@@(1552)
|
||||
# \
|
||||
# *...**(1320)
|
||||
|
||||
print "Test that we can rerequest a block we previously pruned if needed for a reorg"
|
||||
self.reorg_back()
|
||||
# Verify that N2 still has block 1033 on current chain (@), but not on main chain (*)
|
||||
# Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to
|
||||
# original main chain (*), but will require redownload of some blocks
|
||||
# In order to have a peer we think we can download from, must also perform this invalidation
|
||||
# on N0 and mine a new longest chain to trigger.
|
||||
# Final result:
|
||||
# N0 ********************(1032) **...****(1553)
|
||||
# \
|
||||
# X@...@@@(1552)
|
||||
#
|
||||
# N2 **...*(1020) **...**(1032) **...****(1553)
|
||||
# \ \
|
||||
# \ X@...@@@(1552)
|
||||
# \
|
||||
# +..
|
||||
#
|
||||
# N1 doesn't change because 1033 on main chain (*) is invalid
|
||||
|
||||
print "Done"
|
||||
|
||||
if __name__ == '__main__':
|
||||
PruneTest().main()
|
||||
|
|
@ -158,7 +158,7 @@ def _rpchost_to_args(rpchost):
|
|||
rv += ['-rpcport=' + rpcport]
|
||||
return rv
|
||||
|
||||
def start_node(i, dirname, extra_args=None, rpchost=None):
|
||||
def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None):
|
||||
"""
|
||||
Start a bitcoind and return RPC connection to it
|
||||
"""
|
||||
|
|
@ -172,7 +172,10 @@ def start_node(i, dirname, extra_args=None, rpchost=None):
|
|||
["-rpcwait", "getblockcount"], stdout=devnull)
|
||||
devnull.close()
|
||||
url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i))
|
||||
proxy = AuthServiceProxy(url)
|
||||
if timewait is not None:
|
||||
proxy = AuthServiceProxy(url, timeout=timewait)
|
||||
else:
|
||||
proxy = AuthServiceProxy(url)
|
||||
proxy.url = url # store URL on proxy for info
|
||||
return proxy
|
||||
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ public:
|
|||
vAlertPubKey = ParseHex("04fc9702847840aaf195de8442ebecedf5b095cdbb9bc716bda9110971b28a49e0ead8564ff0db22209e0374782c093bb899692d524e9d6a6956e7c5ecbcd68284");
|
||||
nDefaultPort = 8333;
|
||||
nMinerThreads = 0;
|
||||
nPruneAfterHeight = 100000;
|
||||
|
||||
/**
|
||||
* Build the genesis block. Note that the output of the genesis coinbase cannot
|
||||
|
|
@ -198,6 +199,7 @@ public:
|
|||
vAlertPubKey = ParseHex("04302390343f91cc401d56d68b123028bf52e5fca1939df127f63c6467cdf9c8e2c14b61104cf817d0b780da337893ecc4aaff1309e536162dabbdb45200ca2b0a");
|
||||
nDefaultPort = 18333;
|
||||
nMinerThreads = 0;
|
||||
nPruneAfterHeight = 1000;
|
||||
|
||||
//! Modify the testnet genesis block so the timestamp is valid for a later start.
|
||||
genesis.nTime = 1296688602;
|
||||
|
|
@ -257,6 +259,7 @@ public:
|
|||
consensus.hashGenesisBlock = genesis.GetHash();
|
||||
nDefaultPort = 18444;
|
||||
assert(consensus.hashGenesisBlock == uint256S("0x0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"));
|
||||
nPruneAfterHeight = 1000;
|
||||
|
||||
vFixedSeeds.clear(); //! Regtest mode doesn't have any fixed seeds.
|
||||
vSeeds.clear(); //! Regtest mode doesn't have any DNS seeds.
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ public:
|
|||
bool DefaultConsistencyChecks() const { return fDefaultConsistencyChecks; }
|
||||
/** Make standard checks */
|
||||
bool RequireStandard() const { return fRequireStandard; }
|
||||
int64_t PruneAfterHeight() const { return nPruneAfterHeight; }
|
||||
/** Make miner stop after a block is found. In RPC, don't return until nGenProcLimit blocks are generated */
|
||||
bool MineBlocksOnDemand() const { return fMineBlocksOnDemand; }
|
||||
/** In the future use NetworkIDString() for RPC fields */
|
||||
|
|
@ -77,6 +78,7 @@ protected:
|
|||
std::vector<unsigned char> vAlertPubKey;
|
||||
int nDefaultPort;
|
||||
int nMinerThreads;
|
||||
uint64_t nPruneAfterHeight;
|
||||
std::vector<CDNSSeedData> vSeeds;
|
||||
std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES];
|
||||
std::string strNetworkID;
|
||||
|
|
|
|||
90
src/init.cpp
90
src/init.cpp
|
|
@ -275,7 +275,12 @@ std::string HelpMessage(HelpMessageMode mode)
|
|||
#ifndef WIN32
|
||||
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), "bitcoind.pid"));
|
||||
#endif
|
||||
strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup"));
|
||||
strUsage += HelpMessageOpt("-prune=<n>", _("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex.") + " " +
|
||||
_("Warning: Reverting this setting requires re-downloading the entire blockchain.") + " " +
|
||||
_("(default: 0 = disable pruning blocks,") + " " +
|
||||
strprintf(_(">%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
|
||||
strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup"));
|
||||
|
||||
#if !defined(WIN32)
|
||||
strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)"));
|
||||
#endif
|
||||
|
|
@ -352,7 +357,7 @@ std::string HelpMessage(HelpMessageMode mode)
|
|||
strUsage += HelpMessageOpt("-flushwallet", strprintf(_("Run a thread to flush wallet periodically (default: %u)"), 1));
|
||||
strUsage += HelpMessageOpt("-stopafterblockimport", strprintf(_("Stop running after importing blocks from disk (default: %u)"), 0));
|
||||
}
|
||||
string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy"; // Don't translate these and qt below
|
||||
string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy, prune"; // Don't translate these and qt below
|
||||
if (mode == HMM_BITCOIN_QT)
|
||||
debugCategories += ", qt";
|
||||
strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " +
|
||||
|
|
@ -458,10 +463,33 @@ struct CImportingNow
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
// If we're using -prune with -reindex, then delete block files that will be ignored by the
|
||||
// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
|
||||
// is missing, and since pruning works by deleting the oldest block file first, just check
|
||||
// for block file 0, and if it doesn't exist, delete all the block files in the
|
||||
// directory (since they won't be read by the reindex but will take up disk space).
|
||||
void DeleteAllBlockFiles()
|
||||
{
|
||||
if (boost::filesystem::exists(GetBlockPosFilename(CDiskBlockPos(0, 0), "blk")))
|
||||
return;
|
||||
|
||||
LogPrintf("Removing all blk?????.dat and rev?????.dat files for -reindex with -prune\n");
|
||||
boost::filesystem::path blocksdir = GetDataDir() / "blocks";
|
||||
for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) {
|
||||
if (is_regular_file(*it)) {
|
||||
if ((it->path().filename().string().length() == 12) &&
|
||||
(it->path().filename().string().substr(8,4) == ".dat") &&
|
||||
((it->path().filename().string().substr(0,3) == "blk") ||
|
||||
(it->path().filename().string().substr(0,3) == "rev")))
|
||||
boost::filesystem::remove(it->path());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
|
||||
{
|
||||
RenameThread("bitcoin-loadblk");
|
||||
|
||||
// -reindex
|
||||
if (fReindex) {
|
||||
CImportingNow imp;
|
||||
|
|
@ -674,6 +702,21 @@ bool AppInit2(boost::thread_group& threadGroup)
|
|||
if (nFD - MIN_CORE_FILEDESCRIPTORS < nMaxConnections)
|
||||
nMaxConnections = nFD - MIN_CORE_FILEDESCRIPTORS;
|
||||
|
||||
// if using block pruning, then disable txindex
|
||||
// also disable the wallet (for now, until SPV support is implemented in wallet)
|
||||
if (GetArg("-prune", 0)) {
|
||||
if (GetBoolArg("-txindex", false))
|
||||
return InitError(_("Prune mode is incompatible with -txindex."));
|
||||
#ifdef ENABLE_WALLET
|
||||
if (!GetBoolArg("-disablewallet", false)) {
|
||||
if (SoftSetBoolArg("-disablewallet", true))
|
||||
LogPrintf("%s : parameter interaction: -prune -> setting -disablewallet=1\n", __func__);
|
||||
else
|
||||
return InitError(_("Can't run with a wallet in prune mode."));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// ********************************************************* Step 3: parameter-to-internal-flags
|
||||
|
||||
fDebug = !mapMultiArgs["-debug"].empty();
|
||||
|
|
@ -710,6 +753,21 @@ bool AppInit2(boost::thread_group& threadGroup)
|
|||
nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS;
|
||||
|
||||
fServer = GetBoolArg("-server", false);
|
||||
|
||||
// block pruning; get the amount of disk space (in MB) to allot for block & undo files
|
||||
int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024;
|
||||
if (nSignedPruneTarget < 0) {
|
||||
return InitError(_("Prune cannot be configured with a negative value."));
|
||||
}
|
||||
nPruneTarget = (uint64_t) nSignedPruneTarget;
|
||||
if (nPruneTarget) {
|
||||
if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) {
|
||||
return InitError(strprintf(_("Prune configured below the minimum of %d MB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
|
||||
}
|
||||
LogPrintf("Prune configured to target %uMiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024);
|
||||
fPruneMode = true;
|
||||
}
|
||||
|
||||
#ifdef ENABLE_WALLET
|
||||
bool fDisableWallet = GetBoolArg("-disablewallet", false);
|
||||
#endif
|
||||
|
|
@ -1030,8 +1088,12 @@ bool AppInit2(boost::thread_group& threadGroup)
|
|||
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
|
||||
pcoinsTip = new CCoinsViewCache(pcoinscatcher);
|
||||
|
||||
if (fReindex)
|
||||
if (fReindex) {
|
||||
pblocktree->WriteReindexing(true);
|
||||
//If we're reindexing in prune mode, wipe away all our block and undo data files
|
||||
if (fPruneMode)
|
||||
DeleteAllBlockFiles();
|
||||
}
|
||||
|
||||
if (!LoadBlockIndex()) {
|
||||
strLoadError = _("Error loading block database");
|
||||
|
|
@ -1055,7 +1117,18 @@ bool AppInit2(boost::thread_group& threadGroup)
|
|||
break;
|
||||
}
|
||||
|
||||
// Check for changed -prune state. What we are concerned about is a user who has pruned blocks
|
||||
// in the past, but is now trying to run unpruned.
|
||||
if (fHavePruned && !fPruneMode) {
|
||||
strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain");
|
||||
break;
|
||||
}
|
||||
|
||||
uiInterface.InitMessage(_("Verifying blocks..."));
|
||||
if (fHavePruned && GetArg("-checkblocks", 288) > MIN_BLOCKS_TO_KEEP) {
|
||||
LogPrintf("Prune: pruned datadir may not have more than %d blocks; -checkblocks=%d may fail\n",
|
||||
MIN_BLOCKS_TO_KEEP, GetArg("-checkblocks", 288));
|
||||
}
|
||||
if (!CVerifyDB().VerifyDB(pcoinsdbview, GetArg("-checklevel", 3),
|
||||
GetArg("-checkblocks", 288))) {
|
||||
strLoadError = _("Corrupted block database detected");
|
||||
|
|
@ -1106,6 +1179,15 @@ bool AppInit2(boost::thread_group& threadGroup)
|
|||
mempool.ReadFeeEstimates(est_filein);
|
||||
fFeeEstimatesInitialized = true;
|
||||
|
||||
// if prune mode, unset NODE_NETWORK and prune block files
|
||||
if (fPruneMode) {
|
||||
LogPrintf("Unsetting NODE_NETWORK on prune mode\n");
|
||||
nLocalServices &= ~NODE_NETWORK;
|
||||
if (!fReindex) {
|
||||
PruneAndFlush();
|
||||
}
|
||||
}
|
||||
|
||||
// ********************************************************* Step 8: load wallet
|
||||
#ifdef ENABLE_WALLET
|
||||
if (fDisableWallet) {
|
||||
|
|
|
|||
278
src/main.cpp
278
src/main.cpp
|
|
@ -52,9 +52,12 @@ int nScriptCheckThreads = 0;
|
|||
bool fImporting = false;
|
||||
bool fReindex = false;
|
||||
bool fTxIndex = false;
|
||||
bool fHavePruned = false;
|
||||
bool fPruneMode = false;
|
||||
bool fIsBareMultisigStd = true;
|
||||
bool fCheckBlockIndex = false;
|
||||
unsigned int nCoinCacheSize = 5000;
|
||||
uint64_t nPruneTarget = 0;
|
||||
|
||||
/** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */
|
||||
CFeeRate minRelayTxFee = CFeeRate(1000);
|
||||
|
|
@ -110,17 +113,25 @@ namespace {
|
|||
|
||||
/**
|
||||
* The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and
|
||||
* as good as our current tip or better. Entries may be failed, though.
|
||||
* as good as our current tip or better. Entries may be failed, though, and pruning nodes may be
|
||||
* missing the data for the block.
|
||||
*/
|
||||
set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates;
|
||||
/** Number of nodes with fSyncStarted. */
|
||||
int nSyncStarted = 0;
|
||||
/** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. */
|
||||
/** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions.
|
||||
* Pruned nodes may have entries where B is missing data.
|
||||
*/
|
||||
multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked;
|
||||
|
||||
CCriticalSection cs_LastBlockFile;
|
||||
std::vector<CBlockFileInfo> vinfoBlockFile;
|
||||
int nLastBlockFile = 0;
|
||||
/** Global flag to indicate we should check to see if there are
|
||||
* block/undo files that should be deleted. Set on startup
|
||||
* or if we allocate more file space when we're in prune mode
|
||||
*/
|
||||
bool fCheckForPruning = false;
|
||||
|
||||
/**
|
||||
* Every received block is assigned a unique and increasing identifier, so we
|
||||
|
|
@ -1849,6 +1860,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
|
|||
}
|
||||
|
||||
enum FlushStateMode {
|
||||
FLUSH_STATE_NONE,
|
||||
FLUSH_STATE_IF_NEEDED,
|
||||
FLUSH_STATE_PERIODIC,
|
||||
FLUSH_STATE_ALWAYS
|
||||
|
|
@ -1856,16 +1868,30 @@ enum FlushStateMode {
|
|||
|
||||
/**
|
||||
* Update the on-disk chain state.
|
||||
* The caches and indexes are flushed if either they're too large, forceWrite is set, or
|
||||
* fast is not set and it's been a while since the last write.
|
||||
* The caches and indexes are flushed depending on the mode we're called with
|
||||
* if they're too large, if it's been a while since the last write,
|
||||
* or always and in all cases if we're in prune mode and are deleting files.
|
||||
*/
|
||||
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
|
||||
LOCK(cs_main);
|
||||
LOCK2(cs_main, cs_LastBlockFile);
|
||||
static int64_t nLastWrite = 0;
|
||||
std::set<int> setFilesToPrune;
|
||||
bool fFlushForPrune = false;
|
||||
try {
|
||||
if (fPruneMode && fCheckForPruning) {
|
||||
FindFilesToPrune(setFilesToPrune);
|
||||
if (!setFilesToPrune.empty()) {
|
||||
fFlushForPrune = true;
|
||||
if (!fHavePruned) {
|
||||
pblocktree->WriteFlag("prunedblockfiles", true);
|
||||
fHavePruned = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((mode == FLUSH_STATE_ALWAYS) ||
|
||||
((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) ||
|
||||
(mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000)) {
|
||||
(mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000) ||
|
||||
fFlushForPrune) {
|
||||
// Typical CCoins structures on disk are around 100 bytes in size.
|
||||
// Pushing a new one to the database can cause it to be written
|
||||
// twice (once in the log, and once in the tables). This is already
|
||||
|
|
@ -1893,9 +1919,16 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
|
|||
return state.Abort("Files to write to block index database");
|
||||
}
|
||||
}
|
||||
// Finally flush the chainstate (which may refer to block index entries).
|
||||
// Flush the chainstate (which may refer to block index entries).
|
||||
if (!pcoinsTip->Flush())
|
||||
return state.Abort("Failed to write to coin database");
|
||||
|
||||
// Finally remove any pruned files
|
||||
if (fFlushForPrune) {
|
||||
UnlinkPrunedFiles(setFilesToPrune);
|
||||
fCheckForPruning = false;
|
||||
}
|
||||
|
||||
// Update best block in wallet (so we can detect restored wallets).
|
||||
if (mode != FLUSH_STATE_IF_NEEDED) {
|
||||
GetMainSignals().SetBestChain(chainActive.GetLocator());
|
||||
|
|
@ -1913,6 +1946,12 @@ void FlushStateToDisk() {
|
|||
FlushStateToDisk(state, FLUSH_STATE_ALWAYS);
|
||||
}
|
||||
|
||||
void PruneAndFlush() {
|
||||
CValidationState state;
|
||||
fCheckForPruning = true;
|
||||
FlushStateToDisk(state, FLUSH_STATE_NONE);
|
||||
}
|
||||
|
||||
/** Update chainActive and related internal data structures. */
|
||||
void static UpdateTip(CBlockIndex *pindexNew) {
|
||||
chainActive.SetTip(pindexNew);
|
||||
|
|
@ -2083,15 +2122,29 @@ static CBlockIndex* FindMostWorkChain() {
|
|||
CBlockIndex *pindexTest = pindexNew;
|
||||
bool fInvalidAncestor = false;
|
||||
while (pindexTest && !chainActive.Contains(pindexTest)) {
|
||||
assert(pindexTest->nStatus & BLOCK_HAVE_DATA);
|
||||
assert(pindexTest->nChainTx || pindexTest->nHeight == 0);
|
||||
if (pindexTest->nStatus & BLOCK_FAILED_MASK) {
|
||||
// Candidate has an invalid ancestor, remove entire chain from the set.
|
||||
if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)
|
||||
|
||||
// Pruned nodes may have entries in setBlockIndexCandidates for
|
||||
// which block files have been deleted. Remove those as candidates
|
||||
// for the most work chain if we come across them; we can't switch
|
||||
// to a chain unless we have all the non-active-chain parent blocks.
|
||||
bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK;
|
||||
bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA);
|
||||
if (fFailedChain || fMissingData) {
|
||||
// Candidate chain is not usable (either invalid or missing data)
|
||||
if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork))
|
||||
pindexBestInvalid = pindexNew;
|
||||
CBlockIndex *pindexFailed = pindexNew;
|
||||
// Remove the entire chain from the set.
|
||||
while (pindexTest != pindexFailed) {
|
||||
pindexFailed->nStatus |= BLOCK_FAILED_CHILD;
|
||||
if (fFailedChain) {
|
||||
pindexFailed->nStatus |= BLOCK_FAILED_CHILD;
|
||||
} else if (fMissingData) {
|
||||
// If we're missing data, then add back to mapBlocksUnlinked,
|
||||
// so that if the block arrives in the future we can try adding
|
||||
// to setBlockIndexCandidates again.
|
||||
mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed));
|
||||
}
|
||||
setBlockIndexCandidates.erase(pindexFailed);
|
||||
pindexFailed = pindexFailed->pprev;
|
||||
}
|
||||
|
|
@ -2219,7 +2272,9 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) {
|
|||
uint256 hashNewTip = pindexNewTip->GetBlockHash();
|
||||
// Relay inventory, but don't relay old inventory during initial block download.
|
||||
int nBlockEstimate = Checkpoints::GetTotalBlocksEstimate();
|
||||
{
|
||||
// Don't relay blocks if pruning -- could cause a peer to try to download, resulting
|
||||
// in a stalled download if the block file is pruned before the request.
|
||||
if (nLocalServices & NODE_NETWORK) {
|
||||
LOCK(cs_vNodes);
|
||||
BOOST_FOREACH(CNode* pnode, vNodes)
|
||||
if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate))
|
||||
|
|
@ -2419,6 +2474,8 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd
|
|||
unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
|
||||
unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
|
||||
if (nNewChunks > nOldChunks) {
|
||||
if (fPruneMode)
|
||||
fCheckForPruning = true;
|
||||
if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) {
|
||||
FILE *file = OpenBlockFile(pos);
|
||||
if (file) {
|
||||
|
|
@ -2450,6 +2507,8 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne
|
|||
unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
|
||||
unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
|
||||
if (nNewChunks > nOldChunks) {
|
||||
if (fPruneMode)
|
||||
fCheckForPruning = true;
|
||||
if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) {
|
||||
FILE *file = OpenUndoFile(pos);
|
||||
if (file) {
|
||||
|
|
@ -2665,7 +2724,10 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex,
|
|||
if (!AcceptBlockHeader(block, state, &pindex))
|
||||
return false;
|
||||
|
||||
if (pindex->nStatus & BLOCK_HAVE_DATA) {
|
||||
// If we're pruning, ensure that we don't allow a peer to dump a copy
|
||||
// of old blocks. But we might need blocks that are not on the main chain
|
||||
// to handle a reorg, even if we've processed once.
|
||||
if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) {
|
||||
// TODO: deal better with duplicate blocks.
|
||||
// return state.DoS(20, error("AcceptBlock(): already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate");
|
||||
return true;
|
||||
|
|
@ -2698,6 +2760,9 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex,
|
|||
return state.Abort(std::string("System error: ") + e.what());
|
||||
}
|
||||
|
||||
if (fCheckForPruning)
|
||||
FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -2785,6 +2850,112 @@ bool AbortNode(const std::string &strMessage, const std::string &userMessage) {
|
|||
return false;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* BLOCK PRUNING CODE
|
||||
*/
|
||||
|
||||
/* Calculate the amount of disk space the block & undo files currently use */
|
||||
uint64_t CalculateCurrentUsage()
|
||||
{
|
||||
uint64_t retval = 0;
|
||||
BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) {
|
||||
retval += file.nSize + file.nUndoSize;
|
||||
}
|
||||
return retval;
|
||||
}
|
||||
|
||||
/* Prune a block file (modify associated database entries)*/
|
||||
void PruneOneBlockFile(const int fileNumber)
|
||||
{
|
||||
for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) {
|
||||
CBlockIndex* pindex = it->second;
|
||||
if (pindex->nFile == fileNumber) {
|
||||
pindex->nStatus &= ~BLOCK_HAVE_DATA;
|
||||
pindex->nStatus &= ~BLOCK_HAVE_UNDO;
|
||||
pindex->nFile = 0;
|
||||
pindex->nDataPos = 0;
|
||||
pindex->nUndoPos = 0;
|
||||
setDirtyBlockIndex.insert(pindex);
|
||||
|
||||
// Prune from mapBlocksUnlinked -- any block we prune would have
|
||||
// to be downloaded again in order to consider its chain, at which
|
||||
// point it would be considered as a candidate for
|
||||
// mapBlocksUnlinked or setBlockIndexCandidates.
|
||||
std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev);
|
||||
while (range.first != range.second) {
|
||||
std::multimap<CBlockIndex *, CBlockIndex *>::iterator it = range.first;
|
||||
range.first++;
|
||||
if (it->second == pindex) {
|
||||
mapBlocksUnlinked.erase(it);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vinfoBlockFile[fileNumber].SetNull();
|
||||
setDirtyFileInfo.insert(fileNumber);
|
||||
}
|
||||
|
||||
|
||||
void UnlinkPrunedFiles(std::set<int>& setFilesToPrune)
|
||||
{
|
||||
for (set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
|
||||
CDiskBlockPos pos(*it, 0);
|
||||
boost::filesystem::remove(GetBlockPosFilename(pos, "blk"));
|
||||
boost::filesystem::remove(GetBlockPosFilename(pos, "rev"));
|
||||
LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
|
||||
}
|
||||
}
|
||||
|
||||
/* Calculate the block/rev files that should be deleted to remain under target*/
|
||||
void FindFilesToPrune(std::set<int>& setFilesToPrune)
|
||||
{
|
||||
LOCK2(cs_main, cs_LastBlockFile);
|
||||
if (chainActive.Tip() == NULL || nPruneTarget == 0) {
|
||||
return;
|
||||
}
|
||||
if (chainActive.Tip()->nHeight <= Params().PruneAfterHeight()) {
|
||||
return;
|
||||
}
|
||||
|
||||
unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP;
|
||||
uint64_t nCurrentUsage = CalculateCurrentUsage();
|
||||
// We don't check to prune until after we've allocated new space for files
|
||||
// So we should leave a buffer under our target to account for another allocation
|
||||
// before the next pruning.
|
||||
uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
|
||||
uint64_t nBytesToPrune;
|
||||
int count=0;
|
||||
|
||||
if (nCurrentUsage + nBuffer >= nPruneTarget) {
|
||||
for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
|
||||
nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize;
|
||||
|
||||
if (vinfoBlockFile[fileNumber].nSize == 0)
|
||||
continue;
|
||||
|
||||
if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target?
|
||||
break;
|
||||
|
||||
// don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip
|
||||
if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep)
|
||||
break;
|
||||
|
||||
PruneOneBlockFile(fileNumber);
|
||||
// Queue up the files for removal
|
||||
setFilesToPrune.insert(fileNumber);
|
||||
nCurrentUsage -= nBytesToPrune;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n",
|
||||
nPruneTarget/1024/1024, nCurrentUsage/1024/1024,
|
||||
((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024,
|
||||
nLastBlockWeMustKeep, count);
|
||||
}
|
||||
|
||||
bool CheckDiskSpace(uint64_t nAdditionalBytes)
|
||||
{
|
||||
uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available;
|
||||
|
|
@ -2872,7 +3043,9 @@ bool static LoadBlockIndexDB()
|
|||
{
|
||||
CBlockIndex* pindex = item.second;
|
||||
pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
|
||||
if (pindex->nStatus & BLOCK_HAVE_DATA) {
|
||||
// We can link the chain of blocks for which we've received transactions at some point.
|
||||
// Pruned nodes may have deleted the block.
|
||||
if (pindex->nTx > 0) {
|
||||
if (pindex->pprev) {
|
||||
if (pindex->pprev->nChainTx) {
|
||||
pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
|
||||
|
|
@ -2929,6 +3102,11 @@ bool static LoadBlockIndexDB()
|
|||
}
|
||||
}
|
||||
|
||||
// Check whether we have ever pruned block & undo files
|
||||
pblocktree->ReadFlag("prunedblockfiles", fHavePruned);
|
||||
if (fHavePruned)
|
||||
LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
|
||||
|
||||
// Check whether we need to continue reindexing
|
||||
bool fReindexing = false;
|
||||
pblocktree->ReadReindexing(fReindexing);
|
||||
|
|
@ -3069,6 +3247,7 @@ void UnloadBlockIndex()
|
|||
delete entry.second;
|
||||
}
|
||||
mapBlockIndex.clear();
|
||||
fHavePruned = false;
|
||||
}
|
||||
|
||||
bool LoadBlockIndex()
|
||||
|
|
@ -3260,6 +3439,7 @@ void static CheckBlockIndex()
|
|||
int nHeight = 0;
|
||||
CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid.
|
||||
CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA.
|
||||
CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0.
|
||||
CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not).
|
||||
CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not).
|
||||
CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not).
|
||||
|
|
@ -3268,6 +3448,7 @@ void static CheckBlockIndex()
|
|||
nNodes++;
|
||||
if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex;
|
||||
if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex;
|
||||
if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex;
|
||||
if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex;
|
||||
if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex;
|
||||
if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex;
|
||||
|
|
@ -3279,12 +3460,21 @@ void static CheckBlockIndex()
|
|||
assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match.
|
||||
assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block.
|
||||
}
|
||||
// HAVE_DATA is equivalent to VALID_TRANSACTIONS and equivalent to nTx > 0 (we stored the number of transactions in the block)
|
||||
assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0));
|
||||
assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0));
|
||||
if (pindex->nChainTx == 0) assert(pindex->nSequenceId == 0); // nSequenceId can't be set for blocks that aren't linked
|
||||
// All parents having data is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set.
|
||||
assert((pindexFirstMissing != NULL) == (pindex->nChainTx == 0)); // nChainTx == 0 is used to signal that all parent block's transaction data is available.
|
||||
// VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred).
|
||||
// HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred.
|
||||
if (!fHavePruned) {
|
||||
// If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0
|
||||
assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0));
|
||||
assert(pindexFirstMissing == pindexFirstNeverProcessed);
|
||||
} else {
|
||||
// If we have pruned, then we can only say that HAVE_DATA implies nTx > 0
|
||||
if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0);
|
||||
}
|
||||
if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA);
|
||||
assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent.
|
||||
// All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set.
|
||||
assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned).
|
||||
assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0));
|
||||
assert(pindex->nHeight == nHeight); // nHeight must be consistent.
|
||||
assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's.
|
||||
|
|
@ -3297,11 +3487,20 @@ void static CheckBlockIndex()
|
|||
// Checks for not-invalid blocks.
|
||||
assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents.
|
||||
}
|
||||
if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstMissing == NULL) {
|
||||
if (pindexFirstInvalid == NULL) { // If this block sorts at least as good as the current tip and is valid, it must be in setBlockIndexCandidates.
|
||||
assert(setBlockIndexCandidates.count(pindex));
|
||||
if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) {
|
||||
if (pindexFirstInvalid == NULL) {
|
||||
// If this block sorts at least as good as the current tip and
|
||||
// is valid and we have all data for its parents, it must be in
|
||||
// setBlockIndexCandidates. chainActive.Tip() must also be there
|
||||
// even if some data has been pruned.
|
||||
if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) {
|
||||
assert(setBlockIndexCandidates.count(pindex));
|
||||
}
|
||||
// If some parent is missing, then it could be that this block was in
|
||||
// setBlockIndexCandidates but had to be removed because of the missing data.
|
||||
// In this case it must be in mapBlocksUnlinked -- see test below.
|
||||
}
|
||||
} else { // If this block sorts worse than the current tip, it cannot be in setBlockIndexCandidates.
|
||||
} else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates.
|
||||
assert(setBlockIndexCandidates.count(pindex) == 0);
|
||||
}
|
||||
// Check whether this block is in mapBlocksUnlinked.
|
||||
|
|
@ -3315,12 +3514,28 @@ void static CheckBlockIndex()
|
|||
}
|
||||
rangeUnlinked.first++;
|
||||
}
|
||||
if (pindex->pprev && pindex->nStatus & BLOCK_HAVE_DATA && pindexFirstMissing != NULL) {
|
||||
if (pindexFirstInvalid == NULL) { // If this block has block data available, some parent doesn't, and has no invalid parents, it must be in mapBlocksUnlinked.
|
||||
assert(foundInUnlinked);
|
||||
if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) {
|
||||
// If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked.
|
||||
assert(foundInUnlinked);
|
||||
}
|
||||
if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA
|
||||
if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked.
|
||||
if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) {
|
||||
// We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent.
|
||||
assert(fHavePruned); // We must have pruned.
|
||||
// This block may have entered mapBlocksUnlinked if:
|
||||
// - it has a descendant that at some point had more work than the
|
||||
// tip, and
|
||||
// - we tried switching to that descendant but were missing
|
||||
// data for some intermediate block between chainActive and the
|
||||
// tip.
|
||||
// So if this block is itself better than chainActive.Tip() and it wasn't in
|
||||
// setBlockIndexCandidates, then it must be in mapBlocksUnlinked.
|
||||
if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) {
|
||||
if (pindexFirstInvalid == NULL) {
|
||||
assert(foundInUnlinked);
|
||||
}
|
||||
}
|
||||
} else { // If this block does not have block data available, or all parents do, it cannot be in mapBlocksUnlinked.
|
||||
assert(!foundInUnlinked);
|
||||
}
|
||||
// assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow
|
||||
// End: actual consistency checks.
|
||||
|
|
@ -3340,6 +3555,7 @@ void static CheckBlockIndex()
|
|||
// If pindex was the first with a certain property, unset the corresponding variable.
|
||||
if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL;
|
||||
if (pindex == pindexFirstMissing) pindexFirstMissing = NULL;
|
||||
if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL;
|
||||
if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL;
|
||||
if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL;
|
||||
if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL;
|
||||
|
|
@ -3497,7 +3713,9 @@ void static ProcessGetData(CNode* pfrom)
|
|||
}
|
||||
}
|
||||
}
|
||||
if (send)
|
||||
// Pruned nodes may have deleted the block, so check whether
|
||||
// it's available before trying to send.
|
||||
if (send && (mi->second->nStatus & BLOCK_HAVE_DATA))
|
||||
{
|
||||
// Send block from disk
|
||||
CBlock block;
|
||||
|
|
|
|||
45
src/main.h
45
src/main.h
|
|
@ -132,6 +132,26 @@ extern CBlockIndex *pindexBestHeader;
|
|||
/** Minimum disk space required - used in CheckDiskSpace() */
|
||||
static const uint64_t nMinDiskSpace = 52428800;
|
||||
|
||||
/** Pruning-related variables and constants */
|
||||
/** True if any block files have ever been pruned. */
|
||||
extern bool fHavePruned;
|
||||
/** True if we're running in -prune mode. */
|
||||
extern bool fPruneMode;
|
||||
/** Number of MiB of block files that we're trying to stay below. */
|
||||
extern uint64_t nPruneTarget;
|
||||
/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of chainActive.Tip() will not be pruned. */
|
||||
static const signed int MIN_BLOCKS_TO_KEEP = 288;
|
||||
|
||||
// Require that user allocate at least 550MB for block & undo files (blk???.dat and rev???.dat)
|
||||
// At 1MB per block, 288 blocks = 288MB.
|
||||
// Add 15% for Undo data = 331MB
|
||||
// Add 20% for Orphan block rate = 397MB
|
||||
// We want the low water mark after pruning to be at least 397 MB and since we prune in
|
||||
// full block file chunks, we need the high water mark which triggers the prune to be
|
||||
// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB
|
||||
// Setting the target to > than 550MB will make it likely we can respect the target.
|
||||
static const signed int MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024;
|
||||
|
||||
/** Register with a network node to receive its signals */
|
||||
void RegisterNodeSignals(CNodeSignals& nodeSignals);
|
||||
/** Unregister a network node */
|
||||
|
|
@ -186,6 +206,28 @@ bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock, b
|
|||
bool ActivateBestChain(CValidationState &state, CBlock *pblock = NULL);
|
||||
CAmount GetBlockValue(int nHeight, const CAmount& nFees);
|
||||
|
||||
/**
|
||||
* Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target.
|
||||
* The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
|
||||
* space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
|
||||
* (which in this case means the blockchain must be re-downloaded.)
|
||||
*
|
||||
* Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set.
|
||||
* Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
|
||||
* Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 10 on regtest).
|
||||
* Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
|
||||
* The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
|
||||
* A db flag records the fact that at least some block files have been pruned.
|
||||
*
|
||||
* @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
|
||||
*/
|
||||
void FindFilesToPrune(std::set<int>& setFilesToPrune);
|
||||
|
||||
/**
|
||||
* Actually unlink the specified files
|
||||
*/
|
||||
void UnlinkPrunedFiles(std::set<int>& setFilesToPrune);
|
||||
|
||||
/** Create a new block index entry for a given block hash */
|
||||
CBlockIndex * InsertBlockIndex(uint256 hash);
|
||||
/** Abort with a message */
|
||||
|
|
@ -196,7 +238,8 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats);
|
|||
void Misbehaving(NodeId nodeid, int howmuch);
|
||||
/** Flush all state, indexes and buffers to disk. */
|
||||
void FlushStateToDisk();
|
||||
|
||||
/** Prune block files and flush state to disk. */
|
||||
void PruneAndFlush();
|
||||
|
||||
/** (try to) add transaction to memory pool **/
|
||||
bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree,
|
||||
|
|
|
|||
Loading…
Reference in a new issue