maxmustermann/dogecoin
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge #18021: Serialization improvements step 4 (undo.h)

Browse source 
3c94b0039d Convert undo.h to new serialization framework (Pieter Wuille)
3cd8ab9d11 Make std::vector and prevector reuse the VectorFormatter logic (Pieter Wuille)
abf8624356 Add custom vector-element formatter (Pieter Wuille)
37d800bea0 Add a constant for the maximum vector allocation (5 Mbyte) (Pieter Wuille)

Pull request description:

  The next step of changes from #10785.

  This one adds:
  * A meta-formatter for vectors, which serializes the vector elements using another formatter
  * Switch the undo.h code to the new framework, using the above (where undo entries are serialized as a vector, each of which uses a modified serializer for the UTXOs).

ACKs for top commit:
  laanwj:
    code review ACK 3c94b0039d
  jonatack:
    Qualified ACK 3c94b0039d
  ryanofsky:
    Code review ACK 3c94b0039d. Changes since last review: renaming formatter classes, adding suggested static_assert, and removing temporary in VectorFormatter

Tree-SHA512: 44eebf51a303f6adbbc1ca2b9d043e8ae7fd37e06778e026590892f8d09f8253067862a68ba8ca5d733fd2f8e7c84edd255370f5a4b6560259427a65f94632df
This commit is contained in:
Wladimir J. van der Laan 2020-02-10 15:45:37 +01:00
commit 4c2578706c
No known key found for this signature in database
GPG key ID: 1E4AED62986CD25D
2 changed files with 81 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
src/serialize.h
View file

@ -25,6 +25,9 @@
static const unsigned int MAX_SIZE = 0x02000000;
/** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
/**
* Dummy data type to identify deserializing constructors.
*
@ -593,6 +596,53 @@ public:
template<typename I>
BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
/** Formatter to serialize/deserialize vector elements using another formatter
*
* Example:
* struct X {
* std::vector<uint64_t> v;
* SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
* };
* will define a struct that contains a vector of uint64_t, which is serialized
* as a vector of VarInt-encoded integers.
*
* V is not required to be an std::vector type. It works for any class that
* exposes a value_type, size, reserve, push_back, and const iterators.
*/
template<class Formatter>
struct VectorFormatter
{
template<typename Stream, typename V>
void Ser(Stream& s, const V& v)
{
WriteCompactSize(s, v.size());
for (const typename V::value_type& elem : v) {
s << Using<Formatter>(elem);
}
}
template<typename Stream, typename V>
void Unser(Stream& s, V& v)
{
v.clear();
size_t size = ReadCompactSize(s);
size_t allocated = 0;
while (allocated < size) {
// For DoS prevention, do not blindly allocate as much as the stream claims to contain.
// Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
// X MiB of data to make us allocate X+5 Mib.
static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
v.reserve(allocated);
while (v.size() < allocated) {
typename V::value_type val;
s >> Using<Formatter>(val);
v.push_back(std::move(val));
}
}
};
};
/**
* Forward declarations
*/
@ -673,6 +723,20 @@ inline void Unserialize(Stream& is, T&& a)
a.Unserialize(is);
}
/** Default formatter. Serializes objects as themselves.
*
* The vector/prevector serialization code passes this to VectorFormatter
* to enable reusing that logic. It shouldn't be needed elsewhere.
*/
struct DefaultFormatter
{
template<typename Stream, typename T>
static void Ser(Stream& s, const T& t) { Serialize(s, t); }
template<typename Stream, typename T>
static void Unser(Stream& s, T& t) { Unserialize(s, t); }
};
@ -713,9 +777,7 @@ void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
template<typename Stream, unsigned int N, typename T, typename V>
void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
{
WriteCompactSize(os, v.size());
for (typename prevector<N, T>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
::Serialize(os, (*vi));
Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
}
template<typename Stream, unsigned int N, typename T>
@ -744,19 +806,7 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
template<typename Stream, unsigned int N, typename T, typename V>
void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
{
v.clear();
unsigned int nSize = ReadCompactSize(is);
unsigned int i = 0;
unsigned int nMid = 0;
while (nMid < nSize)
{
nMid += 5000000 / sizeof(T);
if (nMid > nSize)
nMid = nSize;
v.resize_uninitialized(nMid);
for (; i < nMid; ++i)
Unserialize(is, v[i]);
}
Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
}
template<typename Stream, unsigned int N, typename T>
@ -793,9 +843,7 @@ void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
template<typename Stream, typename T, typename A, typename V>
void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
{
WriteCompactSize(os, v.size());
for (typename std::vector<T, A>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
::Serialize(os, (*vi));
Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
}
template<typename Stream, typename T, typename A>
@ -824,19 +872,7 @@ void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
template<typename Stream, typename T, typename A, typename V>
void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
{
v.clear();
unsigned int nSize = ReadCompactSize(is);
unsigned int i = 0;
unsigned int nMid = 0;
while (nMid < nSize)
{
nMid += 5000000 / sizeof(T);
if (nMid > nSize)
nMid = nSize;
v.resize(nMid);
for (; i < nMid; i++)
Unserialize(is, v[i]);
}
Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
}
template<typename Stream, typename T, typename A>

69
src/undo.h
View file

@ -13,58 +13,42 @@
#include <serialize.h>
#include <version.h>
/** Undo information for a CTxIn
/** Formatter for undo information for a CTxIn
*
* Contains the prevout's CTxOut being spent, and its metadata as well
* (coinbase or not, height). The serialization contains a dummy value of
* zero. This is compatible with older versions which expect to see
* the transaction version there.
*/
class TxInUndoSerializer
struct TxInUndoFormatter
{
const Coin* txout;
public:
template<typename Stream>
void Serialize(Stream &s) const {
::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1u : 0u)));
if (txout->nHeight > 0) {
void Ser(Stream &s, const Coin& txout) {
::Serialize(s, VARINT(txout.nHeight * 2 + (txout.fCoinBase ? 1u : 0u)));
if (txout.nHeight > 0) {
// Required to maintain compatibility with older undo format.
::Serialize(s, (unsigned char)0);
}
::Serialize(s, Using<TxOutCompression>(REF(txout->out)));
::Serialize(s, Using<TxOutCompression>(txout.out));
}
explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {}
};
class TxInUndoDeserializer
{
Coin* txout;
public:
template<typename Stream>
void Unserialize(Stream &s) {
void Unser(Stream &s, Coin& txout) {
unsigned int nCode = 0;
::Unserialize(s, VARINT(nCode));
txout->nHeight = nCode / 2;
txout->fCoinBase = nCode & 1;
if (txout->nHeight > 0) {
txout.nHeight = nCode / 2;
txout.fCoinBase = nCode & 1;
if (txout.nHeight > 0) {
// Old versions stored the version number for the last spend of
// a transaction's outputs. Non-final spends were indicated with
// height = 0.
unsigned int nVersionDummy;
::Unserialize(s, VARINT(nVersionDummy));
}
::Unserialize(s, Using<TxOutCompression>(REF(txout->out)));
::Unserialize(s, Using<TxOutCompression>(txout.out));
}
explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {}
};
static const size_t MIN_TRANSACTION_INPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxIn(), PROTOCOL_VERSION);
static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_INPUT_WEIGHT;
/** Undo information for a CTransaction */
class CTxUndo
{
@ -72,29 +56,7 @@ public:
// undo information for all txins
std::vector<Coin> vprevout;
template <typename Stream>
void Serialize(Stream& s) const {
// TODO: avoid reimplementing vector serializer
uint64_t count = vprevout.size();
::Serialize(s, COMPACTSIZE(REF(count)));
for (const auto& prevout : vprevout) {
::Serialize(s, TxInUndoSerializer(&prevout));
}
}
template <typename Stream>
void Unserialize(Stream& s) {
// TODO: avoid reimplementing vector deserializer
uint64_t count = 0;
::Unserialize(s, COMPACTSIZE(count));
if (count > MAX_INPUTS_PER_BLOCK) {
throw std::ios_base::failure("Too many input undo records");
}
vprevout.resize(count);
for (auto& prevout : vprevout) {
::Unserialize(s, TxInUndoDeserializer(&prevout));
}
}
SERIALIZE_METHODS(CTxUndo, obj) { READWRITE(Using<VectorFormatter<TxInUndoFormatter>>(obj.vprevout)); }
};
/** Undo information for a CBlock */
@ -103,12 +65,7 @@ class CBlockUndo
public:
std::vector<CTxUndo> vtxundo; // for all but the coinbase
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(vtxundo);
}
SERIALIZE_METHODS(CBlockUndo, obj) { READWRITE(obj.vtxundo); }
};
#endif // BITCOIN_UNDO_H