Rewrite CreateNewBlock

Use the score index on the mempool to only add sorted txs in order.  Remove much of the validation while building the block, relying on mempool to be consistent and only contain txs that can be mined.
The mempool is assumed to be consistent as far as not containing txs which spend non-existent outputs or double spends, and scripts are valid.  Finality of txs is still checked (except not coinbase maturity, assumed in mempool).
Still TestBlockValidity in case mempool consistency breaks and return error state if an invalid block was created.
Unit tests are modified to realize that invalid blocks can now be constructed if the mempool breaks its consistency assumptions and also updated to have the right fees, since the cached value is now used for block construction.

Conflicts:
	src/miner.cpp
This commit is contained in:
Alex Morcos 2015-11-03 10:35:39 -05:00
parent 5f12263302
commit 553cad94e2
2 changed files with 162 additions and 211 deletions

View file

@ -27,6 +27,7 @@
#include <boost/thread.hpp>
#include <boost/tuple/tuple.hpp>
#include <queue>
using namespace std;
@ -40,48 +41,18 @@ using namespace std;
// transactions in the memory pool. When we select transactions from the
// pool, we select by highest priority or fee rate, so we might consider
// transactions that depend on transactions that aren't yet in the block.
// The COrphan class keeps track of these 'temporary orphans' while
// CreateBlock is figuring out which transactions to include.
//
class COrphan
{
public:
const CTransaction* ptx;
set<uint256> setDependsOn;
CFeeRate feeRate;
double dPriority;
COrphan(const CTransaction* ptxIn) : ptx(ptxIn), feeRate(0), dPriority(0)
{
}
};
uint64_t nLastBlockTx = 0;
uint64_t nLastBlockSize = 0;
// We want to sort transactions by priority and fee rate, so:
typedef boost::tuple<double, CFeeRate, const CTransaction*> TxPriority;
class TxPriorityCompare
class ScoreCompare
{
bool byFee;
public:
TxPriorityCompare(bool _byFee) : byFee(_byFee) { }
ScoreCompare() {}
bool operator()(const TxPriority& a, const TxPriority& b)
bool operator()(const CTxMemPool::txiter a, const CTxMemPool::txiter b)
{
if (byFee)
{
if (a.get<1>() == b.get<1>())
return a.get<0>() < b.get<0>();
return a.get<1>() < b.get<1>();
}
else
{
if (a.get<0>() == b.get<0>())
return a.get<1>() < b.get<1>();
return a.get<0>() < b.get<0>();
}
return CompareTxMemPoolEntryByScore()(*b,*a); // Convert to less than
}
};
@ -141,6 +112,22 @@ CBlockTemplate* CreateNewBlock(const CChainParams& chainparams, const CScript& s
nBlockMinSize = std::min(nBlockMaxSize, nBlockMinSize);
// Collect memory pool transactions into the block
CTxMemPool::setEntries inBlock;
CTxMemPool::setEntries waitSet;
// This vector will be sorted into a priority queue:
vector<TxCoinAgePriority> vecPriority;
TxCoinAgePriorityCompare pricomparer;
std::map<CTxMemPool::txiter, double, CTxMemPool::CompareIteratorByHash> waitPriMap;
typedef std::map<CTxMemPool::txiter, double, CTxMemPool::CompareIteratorByHash>::iterator waitPriIter;
double actualPriority = -1;
std::priority_queue<CTxMemPool::txiter, std::vector<CTxMemPool::txiter>, ScoreCompare> clearedTxs;
bool fPrintPriority = GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY);
uint64_t nBlockSize = 1000;
uint64_t nBlockTx = 0;
unsigned int nBlockSigOps = 100;
int lastFewTxs = 0;
CAmount nFees = 0;
{
@ -149,157 +136,102 @@ CBlockTemplate* CreateNewBlock(const CChainParams& chainparams, const CScript& s
const int nHeight = pindexPrev->nHeight + 1;
pblock->nTime = GetAdjustedTime();
const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast();
CCoinsViewCache view(pcoinsTip);
// Priority order to process transactions
list<COrphan> vOrphan; // list memory doesn't move
map<uint256, vector<COrphan*> > mapDependers;
bool fPrintPriority = GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY);
int64_t nLockTimeCutoff = (STANDARD_LOCKTIME_VERIFY_FLAGS & LOCKTIME_MEDIAN_TIME_PAST)
? nMedianTimePast
: pblock->GetBlockTime();
// This vector will be sorted into a priority queue:
vector<TxPriority> vecPriority;
vecPriority.reserve(mempool.mapTx.size());
for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin();
mi != mempool.mapTx.end(); ++mi)
{
const CTransaction& tx = mi->GetTx();
int64_t nLockTimeCutoff = (STANDARD_LOCKTIME_VERIFY_FLAGS & LOCKTIME_MEDIAN_TIME_PAST)
? nMedianTimePast
: pblock->GetBlockTime();
if (tx.IsCoinBase() || !IsFinalTx(tx, nHeight, nLockTimeCutoff))
continue;
COrphan* porphan = NULL;
double dPriority = 0;
CAmount nTotalIn = 0;
bool fMissingInputs = false;
BOOST_FOREACH(const CTxIn& txin, tx.vin)
bool fPriorityBlock = nBlockPrioritySize > 0;
if (fPriorityBlock) {
vecPriority.reserve(mempool.mapTx.size());
for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin();
mi != mempool.mapTx.end(); ++mi)
{
// Read prev transaction
if (!view.HaveCoins(txin.prevout.hash))
{
// This should never happen; all transactions in the memory
// pool should connect to either transactions in the chain
// or other transactions in the memory pool.
if (!mempool.mapTx.count(txin.prevout.hash))
{
LogPrintf("ERROR: mempool transaction missing input\n");
if (fDebug) assert("mempool transaction missing input" == 0);
fMissingInputs = true;
if (porphan)
vOrphan.pop_back();
break;
}
// Has to wait for dependencies
if (!porphan)
{
// Use list for automatic deletion
vOrphan.push_back(COrphan(&tx));
porphan = &vOrphan.back();
}
mapDependers[txin.prevout.hash].push_back(porphan);
porphan->setDependsOn.insert(txin.prevout.hash);
nTotalIn += mempool.mapTx.find(txin.prevout.hash)->GetTx().vout[txin.prevout.n].nValue;
continue;
}
const CCoins* coins = view.AccessCoins(txin.prevout.hash);
assert(coins);
CAmount nValueIn = coins->vout[txin.prevout.n].nValue;
nTotalIn += nValueIn;
int nConf = nHeight - coins->nHeight;
dPriority += (double)nValueIn * nConf;
double dPriority = mi->GetPriority(nHeight);
CAmount dummy;
mempool.ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy);
vecPriority.push_back(TxCoinAgePriority(dPriority, mi));
}
if (fMissingInputs) continue;
// Priority is sum(valuein * age) / modified_txsize
unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
dPriority = tx.ComputePriority(dPriority, nTxSize);
uint256 hash = tx.GetHash();
mempool.ApplyDeltas(hash, dPriority, nTotalIn);
CFeeRate feeRate(nTotalIn-tx.GetValueOut(), nTxSize);
if (porphan)
{
porphan->dPriority = dPriority;
porphan->feeRate = feeRate;
}
else
vecPriority.push_back(TxPriority(dPriority, feeRate, &(mi->GetTx())));
std::make_heap(vecPriority.begin(), vecPriority.end(), pricomparer);
}
// Collect transactions into block
uint64_t nBlockSize = 1000;
uint64_t nBlockTx = 0;
int nBlockSigOps = 100;
bool fSortedByFee = (nBlockPrioritySize <= 0);
CTxMemPool::indexed_transaction_set::nth_index<3>::type::iterator mi = mempool.mapTx.get<3>().begin();
CTxMemPool::txiter iter;
TxPriorityCompare comparer(fSortedByFee);
std::make_heap(vecPriority.begin(), vecPriority.end(), comparer);
while (!vecPriority.empty())
while (mi != mempool.mapTx.get<3>().end() || !clearedTxs.empty())
{
// Take highest priority transaction off the priority queue:
double dPriority = vecPriority.front().get<0>();
CFeeRate feeRate = vecPriority.front().get<1>();
const CTransaction& tx = *(vecPriority.front().get<2>());
std::pop_heap(vecPriority.begin(), vecPriority.end(), comparer);
vecPriority.pop_back();
// Size limits
unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
if (nBlockSize + nTxSize >= nBlockMaxSize)
continue;
// Legacy limits on sigOps:
unsigned int nTxSigOps = GetLegacySigOpCount(tx);
if (nBlockSigOps + nTxSigOps >= MAX_BLOCK_SIGOPS)
continue;
// Skip free transactions if we're past the minimum block size:
const uint256& hash = tx.GetHash();
double dPriorityDelta = 0;
CAmount nFeeDelta = 0;
mempool.ApplyDeltas(hash, dPriorityDelta, nFeeDelta);
if (fSortedByFee && (dPriorityDelta <= 0) && (nFeeDelta <= 0) && (feeRate < ::minRelayTxFee) && (nBlockSize + nTxSize >= nBlockMinSize))
continue;
// Prioritise by fee once past the priority size or we run out of high-priority
// transactions:
if (!fSortedByFee &&
((nBlockSize + nTxSize >= nBlockPrioritySize) || !AllowFree(dPriority)))
{
fSortedByFee = true;
comparer = TxPriorityCompare(fSortedByFee);
std::make_heap(vecPriority.begin(), vecPriority.end(), comparer);
bool priorityTx = false;
if (fPriorityBlock && !vecPriority.empty()) { // add a tx from priority queue to fill the blockprioritysize
priorityTx = true;
iter = vecPriority.front().second;
actualPriority = vecPriority.front().first;
std::pop_heap(vecPriority.begin(), vecPriority.end(), pricomparer);
vecPriority.pop_back();
}
else if (clearedTxs.empty()) { // add tx with next highest score
iter = mempool.mapTx.project<0>(mi);
mi++;
}
else { // try to add a previously postponed child tx
iter = clearedTxs.top();
clearedTxs.pop();
}
if (!view.HaveInputs(tx))
if (inBlock.count(iter))
continue; // could have been added to the priorityBlock
const CTransaction& tx = iter->GetTx();
bool fOrphan = false;
BOOST_FOREACH(CTxMemPool::txiter parent, mempool.GetMemPoolParents(iter))
{
if (!inBlock.count(parent)) {
fOrphan = true;
break;
}
}
if (fOrphan) {
if (priorityTx)
waitPriMap.insert(std::make_pair(iter,actualPriority));
else
waitSet.insert(iter);
continue;
}
unsigned int nTxSize = iter->GetTxSize();
if (fPriorityBlock &&
(nBlockSize + nTxSize >= nBlockPrioritySize || !AllowFree(actualPriority))) {
fPriorityBlock = false;
waitPriMap.clear();
}
if (!priorityTx &&
(iter->GetModifiedFee() < ::minRelayTxFee.GetFee(nTxSize) && nBlockSize >= nBlockMinSize)) {
break;
}
if (nBlockSize + nTxSize >= nBlockMaxSize) {
if (nBlockSize > nBlockMaxSize - 100 || lastFewTxs > 50) {
break;
}
// Once we're within 1000 bytes of a full block, only look at 50 more txs
// to try to fill the remaining space.
if (nBlockSize > nBlockMaxSize - 1000) {
lastFewTxs++;
}
continue;
}
if (!IsFinalTx(tx, nHeight, nLockTimeCutoff))
continue;
CAmount nTxFees = view.GetValueIn(tx)-tx.GetValueOut();
nTxSigOps += GetP2SHSigOpCount(tx, view);
if (nBlockSigOps + nTxSigOps >= MAX_BLOCK_SIGOPS)
unsigned int nTxSigOps = iter->GetSigOpCount();
if (nBlockSigOps + nTxSigOps >= MAX_BLOCK_SIGOPS) {
if (nBlockSigOps > MAX_BLOCK_SIGOPS - 2) {
break;
}
continue;
}
// Note that flags: we don't want to set mempool/IsStandard()
// policy here, but we still have to ensure that the block we
// create only contains transactions that are valid in new blocks.
CValidationState state;
if (!CheckInputs(tx, state, view, true, MANDATORY_SCRIPT_VERIFY_FLAGS, true))
continue;
UpdateCoins(tx, state, view, nHeight);
CAmount nTxFees = iter->GetFee();
// Added
pblock->vtx.push_back(tx);
pblocktemplate->vTxFees.push_back(nTxFees);
@ -311,31 +243,37 @@ CBlockTemplate* CreateNewBlock(const CChainParams& chainparams, const CScript& s
if (fPrintPriority)
{
double dPriority = iter->GetPriority(nHeight);
CAmount dummy;
mempool.ApplyDeltas(tx.GetHash(), dPriority, dummy);
LogPrintf("priority %.1f fee %s txid %s\n",
dPriority, feeRate.ToString(), tx.GetHash().ToString());
dPriority , CFeeRate(iter->GetModifiedFee(), nTxSize).ToString(), tx.GetHash().ToString());
}
inBlock.insert(iter);
// Add transactions that depend on this one to the priority queue
if (mapDependers.count(hash))
BOOST_FOREACH(CTxMemPool::txiter child, mempool.GetMemPoolChildren(iter))
{
BOOST_FOREACH(COrphan* porphan, mapDependers[hash])
{
if (!porphan->setDependsOn.empty())
{
porphan->setDependsOn.erase(hash);
if (porphan->setDependsOn.empty())
{
vecPriority.push_back(TxPriority(porphan->dPriority, porphan->feeRate, porphan->ptx));
std::push_heap(vecPriority.begin(), vecPriority.end(), comparer);
}
if (fPriorityBlock) {
waitPriIter wpiter = waitPriMap.find(child);
if (wpiter != waitPriMap.end()) {
vecPriority.push_back(TxCoinAgePriority(wpiter->second,child));
std::push_heap(vecPriority.begin(), vecPriority.end(), pricomparer);
waitPriMap.erase(wpiter);
}
}
else {
if (waitSet.count(child)) {
clearedTxs.push(child);
waitSet.erase(child);
}
}
}
}
nLastBlockTx = nBlockTx;
nLastBlockSize = nBlockSize;
LogPrintf("CreateNewBlock(): total size %u\n", nBlockSize);
LogPrintf("CreateNewBlock(): total size %u txs: %u fees: %ld sigops %d\n", nBlockSize, nBlockTx, nFees, nBlockSigOps);
// Compute final coinbase transaction.
txNew.vout[0].nValue = nFees + GetBlockSubsidy(nHeight, chainparams.GetConsensus());
@ -351,8 +289,9 @@ CBlockTemplate* CreateNewBlock(const CChainParams& chainparams, const CScript& s
pblocktemplate->vTxSigOps[0] = GetLegacySigOpCount(pblock->vtx[0]);
CValidationState state;
if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false))
throw std::runtime_error("CreateNewBlock(): TestBlockValidity failed");
if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) {
throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state)));
}
}
return pblocktemplate.release();

View file

@ -120,7 +120,22 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
tx.vout[0].nValue -= 1000000;
hash = tx.GetHash();
bool spendsCoinbase = (i == 0) ? true : false; // only first tx spends coinbase
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx));
// If we don't set the # of sig ops in the CTxMemPoolEntry, template creation fails
mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx));
tx.vin[0].prevout.hash = hash;
}
BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error);
mempool.clear();
tx.vin[0].prevout.hash = txFirst[0]->GetHash();
tx.vout[0].nValue = 5000000000LL;
for (unsigned int i = 0; i < 1001; ++i)
{
tx.vout[0].nValue -= 1000000;
hash = tx.GetHash();
bool spendsCoinbase = (i == 0) ? true : false; // only first tx spends coinbase
// If we do set the # of sig ops in the CTxMemPoolEntry, template creation passes
mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOps(20).FromTx(tx));
tx.vin[0].prevout.hash = hash;
}
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));
@ -141,18 +156,17 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
tx.vout[0].nValue -= 10000000;
hash = tx.GetHash();
bool spendsCoinbase = (i == 0) ? true : false; // only first tx spends coinbase
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx));
mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx));
tx.vin[0].prevout.hash = hash;
}
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));
delete pblocktemplate;
mempool.clear();
// orphan in mempool
// orphan in mempool, template creation fails
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).FromTx(tx));
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));
delete pblocktemplate;
mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).FromTx(tx));
BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error);
mempool.clear();
// child with higher priority than parent
@ -160,7 +174,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
tx.vin[0].prevout.hash = txFirst[1]->GetHash();
tx.vout[0].nValue = 4900000000LL;
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
mempool.addUnchecked(hash, entry.Fee(100000000LL).Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
tx.vin[0].prevout.hash = hash;
tx.vin.resize(2);
tx.vin[1].scriptSig = CScript() << OP_1;
@ -168,23 +182,23 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
tx.vin[1].prevout.n = 0;
tx.vout[0].nValue = 5900000000LL;
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
mempool.addUnchecked(hash, entry.Fee(400000000LL).Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));
delete pblocktemplate;
mempool.clear();
// coinbase in mempool
// coinbase in mempool, template creation fails
tx.vin.resize(1);
tx.vin[0].prevout.SetNull();
tx.vin[0].scriptSig = CScript() << OP_0 << OP_1;
tx.vout[0].nValue = 0;
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(false).FromTx(tx));
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));
delete pblocktemplate;
// give it a fee so it'll get mined
mempool.addUnchecked(hash, entry.Fee(100000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx));
BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error);
mempool.clear();
// invalid (pre-p2sh) txn in mempool
// invalid (pre-p2sh) txn in mempool, template creation fails
tx.vin[0].prevout.hash = txFirst[0]->GetHash();
tx.vin[0].prevout.n = 0;
tx.vin[0].scriptSig = CScript() << OP_1;
@ -192,28 +206,26 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
script = CScript() << OP_0;
tx.vout[0].scriptPubKey = GetScriptForDestination(CScriptID(script));
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
mempool.addUnchecked(hash, entry.Fee(10000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
tx.vin[0].prevout.hash = hash;
tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end());
tx.vout[0].nValue -= 1000000;
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(false).FromTx(tx));
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));
delete pblocktemplate;
mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx));
BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error);
mempool.clear();
// double spend txn pair in mempool
// double spend txn pair in mempool, template creation fails
tx.vin[0].prevout.hash = txFirst[0]->GetHash();
tx.vin[0].scriptSig = CScript() << OP_1;
tx.vout[0].nValue = 4900000000LL;
tx.vout[0].scriptPubKey = CScript() << OP_1;
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
mempool.addUnchecked(hash, entry.Fee(100000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
tx.vout[0].scriptPubKey = CScript() << OP_2;
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));
delete pblocktemplate;
mempool.addUnchecked(hash, entry.Fee(100000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error);
mempool.clear();
// subsidy changing
@ -237,7 +249,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
tx.vout[0].scriptPubKey = CScript() << OP_1;
tx.nLockTime = chainActive.Tip()->nHeight+1;
hash = tx.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
mempool.addUnchecked(hash, entry.Fee(100000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx));
BOOST_CHECK(!CheckFinalTx(tx, LOCKTIME_MEDIAN_TIME_PAST));
// time locked
@ -251,7 +263,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
tx2.vout[0].scriptPubKey = CScript() << OP_1;
tx2.nLockTime = chainActive.Tip()->GetMedianTimePast()+1;
hash = tx2.GetHash();
mempool.addUnchecked(hash, entry.Time(GetTime()).SpendsCoinbase(true).FromTx(tx2));
mempool.addUnchecked(hash, entry.Fee(100000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx2));
BOOST_CHECK(!CheckFinalTx(tx2, LOCKTIME_MEDIAN_TIME_PAST));
BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey));