maxmustermann/dogecoin
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Switch blocks to a constant-space Merkle root/branch algorithm.

Browse source 
This switches the Merkle tree logic for blocks to one that runs in constant (small) space.
The old code is moved to tests, and a new test is added that for various combinations of
block sizes, transaction positions to compute a branch for, and mutations:
 * Verifies that the old code and new code agree for the Merkle root.
 * Verifies that the old code and new code agree for the Merkle branch.
 * Verifies that the computed Merkle branch is valid.
 * Verifies that mutations don't change the Merkle root.
 * Verifies that mutations are correctly detected.
This commit is contained in:
Pieter Wuille 2015-11-17 17:35:44 +01:00
parent ee60e5625b
commit eece63fa72
12 changed files with 182 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Makefile.test.include
View file

@ -57,6 +57,7 @@ BITCOIN_TESTS =\
test/dbwrapper_tests.cpp \
test/main_tests.cpp \
test/mempool_tests.cpp \
test/merkle_tests.cpp \
test/miner_tests.cpp \
test/mruset_tests.cpp \
test/multisig_tests.cpp \

3
src/chainparams.cpp
View file

@ -4,6 +4,7 @@
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "chainparams.h"
#include "consensus/merkle.h"
#include "tinyformat.h"
#include "util.h"
@ -32,7 +33,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi
genesis.nVersion = nVersion;
genesis.vtx.push_back(txNew);
genesis.hashPrevBlock.SetNull();
genesis.hashMerkleRoot = genesis.ComputeMerkleRoot();
genesis.hashMerkleRoot = BlockMerkleRoot(genesis);
return genesis;
}

20
src/consensus/merkle.cpp
View file

@ -150,3 +150,23 @@ uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint2
}
return hash;
}
uint256 BlockMerkleRoot(const CBlock& block, bool* mutated)
{
std::vector<uint256> leaves;
leaves.resize(block.vtx.size());
for (size_t s = 0; s < block.vtx.size(); s++) {
leaves[s] = block.vtx[s].GetHash();
}
return ComputeMerkleRoot(leaves, mutated);
}
std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position)
{
std::vector<uint256> leaves;
leaves.resize(block.vtx.size());
for (size_t s = 0; s < block.vtx.size(); s++) {
leaves[s] = block.vtx[s].GetHash();
}
return ComputeMerkleBranch(leaves, position);
}

15
src/consensus/merkle.h
View file

@ -8,10 +8,25 @@
#include <stdint.h>
#include <vector>
#include "primitives/transaction.h"
#include "primitives/block.h"
#include "uint256.h"
uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated = NULL);
std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position);
uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint256>& branch, uint32_t position);
/*
* Compute the Merkle root of the transactions in a block.
* *mutated is set to true if a duplicated subtree was found.
*/
uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = NULL);
/*
* Compute the Merkle branch for the tree of transactions in a block, for a
* given position.
* This can be verified using ComputeMerkleRootFromBranch.
*/
std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position);
#endif

3
src/main.cpp
View file

@ -12,6 +12,7 @@
#include "checkpoints.h"
#include "checkqueue.h"
#include "consensus/consensus.h"
#include "consensus/merkle.h"
#include "consensus/validation.h"
#include "hash.h"
#include "init.h"
@ -2876,7 +2877,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo
// Check the merkle root.
if (fCheckMerkleRoot) {
bool mutated;
uint256 hashMerkleRoot2 = block.ComputeMerkleRoot(&mutated);
uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated);
if (block.hashMerkleRoot != hashMerkleRoot2)
return state.DoS(100, error("CheckBlock(): hashMerkleRoot mismatch"),
REJECT_INVALID, "bad-txnmrklroot", true);

3
src/miner.cpp
View file

@ -10,6 +10,7 @@
#include "chainparams.h"
#include "coins.h"
#include "consensus/consensus.h"
#include "consensus/merkle.h"
#include "consensus/validation.h"
#include "hash.h"
#include "main.h"
@ -373,7 +374,7 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned
assert(txCoinbase.vin[0].scriptSig.size() <= 100);
pblock->vtx[0] = txCoinbase;
pblock->hashMerkleRoot = pblock->ComputeMerkleRoot();
pblock->hashMerkleRoot = BlockMerkleRoot(*pblock);
}
//////////////////////////////////////////////////////////////////////////////

63
src/primitives/block.cpp
View file

@ -15,69 +15,6 @@ uint256 CBlockHeader::GetHash() const
return SerializeHash(*this);
}
uint256 CBlock::ComputeMerkleRoot(bool* fMutated) const
{
/* WARNING! If you're reading this because you're learning about crypto
and/or designing a new system that will use merkle trees, keep in mind
that the following merkle tree algorithm has a serious flaw related to
duplicate txids, resulting in a vulnerability (CVE-2012-2459).
The reason is that if the number of hashes in the list at a given time
is odd, the last one is duplicated before computing the next level (which
is unusual in Merkle trees). This results in certain sequences of
transactions leading to the same merkle root. For example, these two
trees:
A A
/ \ / \
B C B C
/ \ | / \ / \
D E F D E F F
/ \ / \ / \ / \ / \ / \ / \
1 2 3 4 5 6 1 2 3 4 5 6 5 6
for transaction lists [1,2,3,4,5,6] and [1,2,3,4,5,6,5,6] (where 5 and
6 are repeated) result in the same root hash A (because the hash of both
of (F) and (F,F) is C).
The vulnerability results from being able to send a block with such a
transaction list, with the same merkle root, and the same block hash as
the original without duplication, resulting in failed validation. If the
receiving node proceeds to mark that block as permanently invalid
however, it will fail to accept further unmodified (and thus potentially
valid) versions of the same block. We defend against this by detecting
the case where we would hash two identical hashes at the end of the list
together, and treating that identically to the block having an invalid
merkle root. Assuming no double-SHA256 collisions, this will detect all
known ways of changing the transactions without affecting the merkle
root.
*/
std::vector<uint256> vMerkleTree;
vMerkleTree.reserve(vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes.
for (std::vector<CTransaction>::const_iterator it(vtx.begin()); it != vtx.end(); ++it)
vMerkleTree.push_back(it->GetHash());
int j = 0;
bool mutated = false;
for (int nSize = vtx.size(); nSize > 1; nSize = (nSize + 1) / 2)
{
for (int i = 0; i < nSize; i += 2)
{
int i2 = std::min(i+1, nSize-1);
if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) {
// Two identical hashes at the end of the list at a particular level.
mutated = true;
}
vMerkleTree.push_back(Hash(BEGIN(vMerkleTree[j+i]), END(vMerkleTree[j+i]),
BEGIN(vMerkleTree[j+i2]), END(vMerkleTree[j+i2])));
}
j += nSize;
}
if (fMutated) {
*fMutated = mutated;
}
return (vMerkleTree.empty() ? uint256() : vMerkleTree.back());
}
std::string CBlock::ToString() const
{
std::stringstream s;

6
src/primitives/block.h
View file

@ -118,12 +118,6 @@ public:
return block;
}
// Build the merkle tree for this block and return the merkle root.
// If non-NULL, *mutated is set to whether mutation was detected in the merkle
// tree (a duplication of transactions in the block leading to an identical
// merkle root).
uint256 ComputeMerkleRoot(bool* mutated = NULL) const;
std::string ToString() const;
};

1
src/test/main_tests.cpp
View file

@ -72,5 +72,4 @@ BOOST_AUTO_TEST_CASE(test_combiner_all)
Test.disconnect(&ReturnTrue);
BOOST_CHECK(Test());
}
BOOST_AUTO_TEST_SUITE_END()

136
src/test/merkle_tests.cpp Normal file
View file

@ -0,0 +1,136 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "consensus/merkle.h"
#include "test/test_bitcoin.h"
#include "random.h"
#include <boost/test/unit_test.hpp>
BOOST_FIXTURE_TEST_SUITE(merkle_tests, TestingSetup)
// Older version of the merkle root computation code, for comparison.
static uint256 BlockBuildMerkleTree(const CBlock& block, bool* fMutated, std::vector<uint256>& vMerkleTree)
{
vMerkleTree.clear();
vMerkleTree.reserve(block.vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes.
for (std::vector<CTransaction>::const_iterator it(block.vtx.begin()); it != block.vtx.end(); ++it)
vMerkleTree.push_back(it->GetHash());
int j = 0;
bool mutated = false;
for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2)
{
for (int i = 0; i < nSize; i += 2)
{
int i2 = std::min(i+1, nSize-1);
if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) {
// Two identical hashes at the end of the list at a particular level.
mutated = true;
}
vMerkleTree.push_back(Hash(vMerkleTree[j+i].begin(), vMerkleTree[j+i].end(),
vMerkleTree[j+i2].begin(), vMerkleTree[j+i2].end()));
}
j += nSize;
}
if (fMutated) {
*fMutated = mutated;
}
return (vMerkleTree.empty() ? uint256() : vMerkleTree.back());
}
// Older version of the merkle branch computation code, for comparison.
static std::vector<uint256> BlockGetMerkleBranch(const CBlock& block, const std::vector<uint256>& vMerkleTree, int nIndex)
{
std::vector<uint256> vMerkleBranch;
int j = 0;
for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2)
{
int i = std::min(nIndex^1, nSize-1);
vMerkleBranch.push_back(vMerkleTree[j+i]);
nIndex >>= 1;
j += nSize;
}
return vMerkleBranch;
}
static inline int ctz(uint32_t i) {
if (i == 0) return 0;
int j = 0;
while (!(i & 1)) {
j++;
i >>= 1;
}
return j;
}
BOOST_AUTO_TEST_CASE(merkle_test)
{
for (int i = 0; i < 32; i++) {
// Try 32 block sizes: all sizes from 0 to 16 inclusive, and then 15 random sizes.
int ntx = (i <= 16) ? i : 17 + (insecure_rand() % 4000);
// Try up to 3 mutations.
for (int mutate = 0; mutate <= 3; mutate++) {
int duplicate1 = mutate >= 1 ? 1 << ctz(ntx) : 0; // The last how many transactions to duplicate first.
if (duplicate1 >= ntx) break; // Duplication of the entire tree results in a different root (it adds a level).
int ntx1 = ntx + duplicate1; // The resulting number of transactions after the first duplication.
int duplicate2 = mutate >= 2 ? 1 << ctz(ntx1) : 0; // Likewise for the second mutation.
if (duplicate2 >= ntx1) break;
int ntx2 = ntx1 + duplicate2;
int duplicate3 = mutate >= 3 ? 1 << ctz(ntx2) : 0; // And for the the third mutation.
if (duplicate3 >= ntx2) break;
int ntx3 = ntx2 + duplicate3;
// Build a block with ntx different transactions.
CBlock block;
block.vtx.resize(ntx);
for (int j = 0; j < ntx; j++) {
CMutableTransaction mtx;
mtx.nLockTime = j;
block.vtx[j] = mtx;
}
// Compute the root of the block before mutating it.
bool unmutatedMutated = false;
uint256 unmutatedRoot = BlockMerkleRoot(block, &unmutatedMutated);
BOOST_CHECK(unmutatedMutated == false);
// Optionally mutate by duplicating the last transactions, resulting in the same merkle root.
block.vtx.resize(ntx3);
for (int j = 0; j < duplicate1; j++) {
block.vtx[ntx + j] = block.vtx[ntx + j - duplicate1];
}
for (int j = 0; j < duplicate2; j++) {
block.vtx[ntx1 + j] = block.vtx[ntx1 + j - duplicate2];
}
for (int j = 0; j < duplicate3; j++) {
block.vtx[ntx2 + j] = block.vtx[ntx2 + j - duplicate3];
}
// Compute the merkle root and merkle tree using the old mechanism.
bool oldMutated = false;
std::vector<uint256> merkleTree;
uint256 oldRoot = BlockBuildMerkleTree(block, &oldMutated, merkleTree);
// Compute the merkle root using the new mechanism.
bool newMutated = false;
uint256 newRoot = BlockMerkleRoot(block, &newMutated);
BOOST_CHECK(oldRoot == newRoot);
BOOST_CHECK(newRoot == unmutatedRoot);
BOOST_CHECK((newRoot == uint256()) == (ntx == 0));
BOOST_CHECK(oldMutated == newMutated);
BOOST_CHECK(newMutated == !!mutate);
// If no mutation was done (once for every ntx value), try up to 16 branches.
if (mutate == 0) {
for (int loop = 0; loop < std::min(ntx, 16); loop++) {
// If ntx <= 16, try all branches. Otherise, try 16 random ones.
int mtx = loop;
if (ntx > 16) {
mtx = insecure_rand() % ntx;
}
std::vector<uint256> newBranch = BlockMerkleBranch(block, mtx);
std::vector<uint256> oldBranch = BlockGetMerkleBranch(block, merkleTree, mtx);
BOOST_CHECK(oldBranch == newBranch);
BOOST_CHECK(ComputeMerkleRootFromBranch(block.vtx[mtx].GetHash(), newBranch, mtx) == oldRoot);
}
}
}
}
}
BOOST_AUTO_TEST_SUITE_END()

3
src/test/miner_tests.cpp
View file

@ -5,6 +5,7 @@
#include "chainparams.h"
#include "coins.h"
#include "consensus/consensus.h"
#include "consensus/merkle.h"
#include "consensus/validation.h"
#include "main.h"
#include "miner.h"
@ -93,7 +94,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
pblock->vtx[0] = CTransaction(txCoinbase);
if (txFirst.size() < 2)
txFirst.push_back(new CTransaction(pblock->vtx[0]));
pblock->hashMerkleRoot = pblock->ComputeMerkleRoot();
pblock->hashMerkleRoot = BlockMerkleRoot(*pblock);
pblock->nNonce = blockinfo[i].nonce;
CValidationState state;
BOOST_CHECK(ProcessNewBlock(state, chainparams, NULL, pblock, true, NULL));

3
src/test/pmt_tests.cpp
View file

@ -2,6 +2,7 @@
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "consensus/merkle.h"
#include "merkleblock.h"
#include "serialize.h"
#include "streams.h"
@ -48,7 +49,7 @@ BOOST_AUTO_TEST_CASE(pmt_test1)
}
// calculate actual merkle root and height
uint256 merkleRoot1 = block.ComputeMerkleRoot();
uint256 merkleRoot1 = BlockMerkleRoot(block);
std::vector<uint256> vTxid(nTx, uint256());
for (unsigned int j=0; j<nTx; j++)
vTxid[j] = block.vtx[j].GetHash();