diff --git a/src/Makefile.test.include b/src/Makefile.test.include index fafc1a294..c377183ad 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -57,6 +57,7 @@ BITCOIN_TESTS =\ test/dbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ + test/merkle_tests.cpp \ test/miner_tests.cpp \ test/mruset_tests.cpp \ test/multisig_tests.cpp \ diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 5d6d1ef9d..a46866a2b 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "chainparams.h" +#include "consensus/merkle.h" #include "tinyformat.h" #include "util.h" @@ -32,7 +33,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi genesis.nVersion = nVersion; genesis.vtx.push_back(txNew); genesis.hashPrevBlock.SetNull(); - genesis.hashMerkleRoot = genesis.ComputeMerkleRoot(); + genesis.hashMerkleRoot = BlockMerkleRoot(genesis); return genesis; } diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 6be9c26df..9a8afa8a3 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -150,3 +150,23 @@ uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector leaves; + leaves.resize(block.vtx.size()); + for (size_t s = 0; s < block.vtx.size(); s++) { + leaves[s] = block.vtx[s].GetHash(); + } + return ComputeMerkleRoot(leaves, mutated); +} + +std::vector BlockMerkleBranch(const CBlock& block, uint32_t position) +{ + std::vector leaves; + leaves.resize(block.vtx.size()); + for (size_t s = 0; s < block.vtx.size(); s++) { + leaves[s] = block.vtx[s].GetHash(); + } + return ComputeMerkleBranch(leaves, position); +} diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index 7fd13d3e4..6ef59745a 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -8,10 +8,25 @@ #include #include +#include "primitives/transaction.h" +#include "primitives/block.h" #include "uint256.h" uint256 ComputeMerkleRoot(const std::vector& leaves, bool* mutated = NULL); std::vector ComputeMerkleBranch(const std::vector& leaves, uint32_t position); uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector& branch, uint32_t position); +/* + * Compute the Merkle root of the transactions in a block. + * *mutated is set to true if a duplicated subtree was found. + */ +uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = NULL); + +/* + * Compute the Merkle branch for the tree of transactions in a block, for a + * given position. + * This can be verified using ComputeMerkleRootFromBranch. + */ +std::vector BlockMerkleBranch(const CBlock& block, uint32_t position); + #endif diff --git a/src/main.cpp b/src/main.cpp index ceb5cb66f..191b2b3c1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -12,6 +12,7 @@ #include "checkpoints.h" #include "checkqueue.h" #include "consensus/consensus.h" +#include "consensus/merkle.h" #include "consensus/validation.h" #include "hash.h" #include "init.h" @@ -2876,7 +2877,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo // Check the merkle root. if (fCheckMerkleRoot) { bool mutated; - uint256 hashMerkleRoot2 = block.ComputeMerkleRoot(&mutated); + uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) return state.DoS(100, error("CheckBlock(): hashMerkleRoot mismatch"), REJECT_INVALID, "bad-txnmrklroot", true); diff --git a/src/miner.cpp b/src/miner.cpp index bb6b51337..8187e5818 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -10,6 +10,7 @@ #include "chainparams.h" #include "coins.h" #include "consensus/consensus.h" +#include "consensus/merkle.h" #include "consensus/validation.h" #include "hash.h" #include "main.h" @@ -373,7 +374,7 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned assert(txCoinbase.vin[0].scriptSig.size() <= 100); pblock->vtx[0] = txCoinbase; - pblock->hashMerkleRoot = pblock->ComputeMerkleRoot(); + pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); } ////////////////////////////////////////////////////////////////////////////// diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 7a58074d2..7280c18f7 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -15,69 +15,6 @@ uint256 CBlockHeader::GetHash() const return SerializeHash(*this); } -uint256 CBlock::ComputeMerkleRoot(bool* fMutated) const -{ - /* WARNING! If you're reading this because you're learning about crypto - and/or designing a new system that will use merkle trees, keep in mind - that the following merkle tree algorithm has a serious flaw related to - duplicate txids, resulting in a vulnerability (CVE-2012-2459). - - The reason is that if the number of hashes in the list at a given time - is odd, the last one is duplicated before computing the next level (which - is unusual in Merkle trees). This results in certain sequences of - transactions leading to the same merkle root. For example, these two - trees: - - A A - / \ / \ - B C B C - / \ | / \ / \ - D E F D E F F - / \ / \ / \ / \ / \ / \ / \ - 1 2 3 4 5 6 1 2 3 4 5 6 5 6 - - for transaction lists [1,2,3,4,5,6] and [1,2,3,4,5,6,5,6] (where 5 and - 6 are repeated) result in the same root hash A (because the hash of both - of (F) and (F,F) is C). - - The vulnerability results from being able to send a block with such a - transaction list, with the same merkle root, and the same block hash as - the original without duplication, resulting in failed validation. If the - receiving node proceeds to mark that block as permanently invalid - however, it will fail to accept further unmodified (and thus potentially - valid) versions of the same block. We defend against this by detecting - the case where we would hash two identical hashes at the end of the list - together, and treating that identically to the block having an invalid - merkle root. Assuming no double-SHA256 collisions, this will detect all - known ways of changing the transactions without affecting the merkle - root. - */ - std::vector vMerkleTree; - vMerkleTree.reserve(vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes. - for (std::vector::const_iterator it(vtx.begin()); it != vtx.end(); ++it) - vMerkleTree.push_back(it->GetHash()); - int j = 0; - bool mutated = false; - for (int nSize = vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) - { - for (int i = 0; i < nSize; i += 2) - { - int i2 = std::min(i+1, nSize-1); - if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) { - // Two identical hashes at the end of the list at a particular level. - mutated = true; - } - vMerkleTree.push_back(Hash(BEGIN(vMerkleTree[j+i]), END(vMerkleTree[j+i]), - BEGIN(vMerkleTree[j+i2]), END(vMerkleTree[j+i2]))); - } - j += nSize; - } - if (fMutated) { - *fMutated = mutated; - } - return (vMerkleTree.empty() ? uint256() : vMerkleTree.back()); -} - std::string CBlock::ToString() const { std::stringstream s; diff --git a/src/primitives/block.h b/src/primitives/block.h index 54731ff55..5c017d436 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -118,12 +118,6 @@ public: return block; } - // Build the merkle tree for this block and return the merkle root. - // If non-NULL, *mutated is set to whether mutation was detected in the merkle - // tree (a duplication of transactions in the block leading to an identical - // merkle root). - uint256 ComputeMerkleRoot(bool* mutated = NULL) const; - std::string ToString() const; }; diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index 21ae46d6e..2b92d239e 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -72,5 +72,4 @@ BOOST_AUTO_TEST_CASE(test_combiner_all) Test.disconnect(&ReturnTrue); BOOST_CHECK(Test()); } - BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp new file mode 100644 index 000000000..1e31f2e67 --- /dev/null +++ b/src/test/merkle_tests.cpp @@ -0,0 +1,136 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "consensus/merkle.h" +#include "test/test_bitcoin.h" +#include "random.h" + +#include + +BOOST_FIXTURE_TEST_SUITE(merkle_tests, TestingSetup) + +// Older version of the merkle root computation code, for comparison. +static uint256 BlockBuildMerkleTree(const CBlock& block, bool* fMutated, std::vector& vMerkleTree) +{ + vMerkleTree.clear(); + vMerkleTree.reserve(block.vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes. + for (std::vector::const_iterator it(block.vtx.begin()); it != block.vtx.end(); ++it) + vMerkleTree.push_back(it->GetHash()); + int j = 0; + bool mutated = false; + for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) + { + for (int i = 0; i < nSize; i += 2) + { + int i2 = std::min(i+1, nSize-1); + if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) { + // Two identical hashes at the end of the list at a particular level. + mutated = true; + } + vMerkleTree.push_back(Hash(vMerkleTree[j+i].begin(), vMerkleTree[j+i].end(), + vMerkleTree[j+i2].begin(), vMerkleTree[j+i2].end())); + } + j += nSize; + } + if (fMutated) { + *fMutated = mutated; + } + return (vMerkleTree.empty() ? uint256() : vMerkleTree.back()); +} + +// Older version of the merkle branch computation code, for comparison. +static std::vector BlockGetMerkleBranch(const CBlock& block, const std::vector& vMerkleTree, int nIndex) +{ + std::vector vMerkleBranch; + int j = 0; + for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) + { + int i = std::min(nIndex^1, nSize-1); + vMerkleBranch.push_back(vMerkleTree[j+i]); + nIndex >>= 1; + j += nSize; + } + return vMerkleBranch; +} + +static inline int ctz(uint32_t i) { + if (i == 0) return 0; + int j = 0; + while (!(i & 1)) { + j++; + i >>= 1; + } + return j; +} + +BOOST_AUTO_TEST_CASE(merkle_test) +{ + for (int i = 0; i < 32; i++) { + // Try 32 block sizes: all sizes from 0 to 16 inclusive, and then 15 random sizes. + int ntx = (i <= 16) ? i : 17 + (insecure_rand() % 4000); + // Try up to 3 mutations. + for (int mutate = 0; mutate <= 3; mutate++) { + int duplicate1 = mutate >= 1 ? 1 << ctz(ntx) : 0; // The last how many transactions to duplicate first. + if (duplicate1 >= ntx) break; // Duplication of the entire tree results in a different root (it adds a level). + int ntx1 = ntx + duplicate1; // The resulting number of transactions after the first duplication. + int duplicate2 = mutate >= 2 ? 1 << ctz(ntx1) : 0; // Likewise for the second mutation. + if (duplicate2 >= ntx1) break; + int ntx2 = ntx1 + duplicate2; + int duplicate3 = mutate >= 3 ? 1 << ctz(ntx2) : 0; // And for the the third mutation. + if (duplicate3 >= ntx2) break; + int ntx3 = ntx2 + duplicate3; + // Build a block with ntx different transactions. + CBlock block; + block.vtx.resize(ntx); + for (int j = 0; j < ntx; j++) { + CMutableTransaction mtx; + mtx.nLockTime = j; + block.vtx[j] = mtx; + } + // Compute the root of the block before mutating it. + bool unmutatedMutated = false; + uint256 unmutatedRoot = BlockMerkleRoot(block, &unmutatedMutated); + BOOST_CHECK(unmutatedMutated == false); + // Optionally mutate by duplicating the last transactions, resulting in the same merkle root. + block.vtx.resize(ntx3); + for (int j = 0; j < duplicate1; j++) { + block.vtx[ntx + j] = block.vtx[ntx + j - duplicate1]; + } + for (int j = 0; j < duplicate2; j++) { + block.vtx[ntx1 + j] = block.vtx[ntx1 + j - duplicate2]; + } + for (int j = 0; j < duplicate3; j++) { + block.vtx[ntx2 + j] = block.vtx[ntx2 + j - duplicate3]; + } + // Compute the merkle root and merkle tree using the old mechanism. + bool oldMutated = false; + std::vector merkleTree; + uint256 oldRoot = BlockBuildMerkleTree(block, &oldMutated, merkleTree); + // Compute the merkle root using the new mechanism. + bool newMutated = false; + uint256 newRoot = BlockMerkleRoot(block, &newMutated); + BOOST_CHECK(oldRoot == newRoot); + BOOST_CHECK(newRoot == unmutatedRoot); + BOOST_CHECK((newRoot == uint256()) == (ntx == 0)); + BOOST_CHECK(oldMutated == newMutated); + BOOST_CHECK(newMutated == !!mutate); + // If no mutation was done (once for every ntx value), try up to 16 branches. + if (mutate == 0) { + for (int loop = 0; loop < std::min(ntx, 16); loop++) { + // If ntx <= 16, try all branches. Otherise, try 16 random ones. + int mtx = loop; + if (ntx > 16) { + mtx = insecure_rand() % ntx; + } + std::vector newBranch = BlockMerkleBranch(block, mtx); + std::vector oldBranch = BlockGetMerkleBranch(block, merkleTree, mtx); + BOOST_CHECK(oldBranch == newBranch); + BOOST_CHECK(ComputeMerkleRootFromBranch(block.vtx[mtx].GetHash(), newBranch, mtx) == oldRoot); + } + } + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index dc20e3463..1d7c9f65c 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -5,6 +5,7 @@ #include "chainparams.h" #include "coins.h" #include "consensus/consensus.h" +#include "consensus/merkle.h" #include "consensus/validation.h" #include "main.h" #include "miner.h" @@ -93,7 +94,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->vtx[0] = CTransaction(txCoinbase); if (txFirst.size() < 2) txFirst.push_back(new CTransaction(pblock->vtx[0])); - pblock->hashMerkleRoot = pblock->ComputeMerkleRoot(); + pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); pblock->nNonce = blockinfo[i].nonce; CValidationState state; BOOST_CHECK(ProcessNewBlock(state, chainparams, NULL, pblock, true, NULL)); diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index d9f3c3e46..0d7fb2bc3 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "consensus/merkle.h" #include "merkleblock.h" #include "serialize.h" #include "streams.h" @@ -48,7 +49,7 @@ BOOST_AUTO_TEST_CASE(pmt_test1) } // calculate actual merkle root and height - uint256 merkleRoot1 = block.ComputeMerkleRoot(); + uint256 merkleRoot1 = BlockMerkleRoot(block); std::vector vTxid(nTx, uint256()); for (unsigned int j=0; j