Simple benchmarking framework

Benchmarking framework, loosely based on google's micro-benchmarking
library (https://github.com/google/benchmark)

Wny not use the Google Benchmark framework? Because adding Even More Dependencies
isn't worth it. If we get a dozen or three benchmarks and need nanosecond-accurate
timings of threaded code then switching to the full-blown Google Benchmark library
should be considered.

The benchmark framework is hard-coded to run each benchmark for one wall-clock second,
and then spits out .csv-format timing information to stdout. It is left as an
exercise for later (or maybe never) to add command-line arguments to specify which
benchmark(s) to run, how long to run them for, how to format results, etc etc etc.
Again, see the Google Benchmark framework for where that might end up.

See src/bench/MilliSleep.cpp for a sanity-test benchmark that just benchmarks
'sleep 100 milliseconds.'

To compile and run benchmarks:
  cd src; make bench

Sample output:

Benchmark,count,min,max,average
Sleep100ms,10,0.101854,0.105059,0.103881
This commit is contained in:
Gavin Andresen 2015-09-24 13:13:38 -04:00
parent 1a9f19a78d
commit 535ed9223d
No known key found for this signature in database
GPG key ID: 7588242FBE38D3A8
7 changed files with 222 additions and 0 deletions

View file

@ -91,6 +91,11 @@ AC_ARG_ENABLE(tests,
[use_tests=$enableval], [use_tests=$enableval],
[use_tests=yes]) [use_tests=yes])
AC_ARG_ENABLE(bench,
AS_HELP_STRING([--enable-bench],[compile benchmarks (default is yes)]),
[use_bench=$enableval],
[use_bench=yes])
AC_ARG_WITH([comparison-tool], AC_ARG_WITH([comparison-tool],
AS_HELP_STRING([--with-comparison-tool],[path to java comparison tool (requires --enable-tests)]), AS_HELP_STRING([--with-comparison-tool],[path to java comparison tool (requires --enable-tests)]),
[use_comparison_tool=$withval], [use_comparison_tool=$withval],
@ -881,6 +886,7 @@ AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes])
AM_CONDITIONAL([ENABLE_TESTS],[test x$use_tests = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$use_tests = xyes])
AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes])
AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$use_tests$bitcoin_enable_qt_test = xyesyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$use_tests$bitcoin_enable_qt_test = xyesyes])
AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes])
AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes])
AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes])
AM_CONDITIONAL([USE_COMPARISON_TOOL],[test x$use_comparison_tool != xno]) AM_CONDITIONAL([USE_COMPARISON_TOOL],[test x$use_comparison_tool != xno])

View file

@ -61,6 +61,7 @@ endif
bin_PROGRAMS = bin_PROGRAMS =
TESTS = TESTS =
BENCHMARKS =
if BUILD_BITCOIND if BUILD_BITCOIND
bin_PROGRAMS += bitcoind bin_PROGRAMS += bitcoind
@ -445,6 +446,10 @@ if ENABLE_TESTS
include Makefile.test.include include Makefile.test.include
endif endif
if ENABLE_BENCH
include Makefile.bench.include
endif
if ENABLE_QT if ENABLE_QT
include Makefile.qt.include include Makefile.qt.include
endif endif

View file

@ -0,0 +1,45 @@
bin_PROGRAMS += bench/bench_bitcoin
BENCH_SRCDIR = bench
BENCH_BINARY = bench/bench_bitcoin$(EXEEXT)
bench_bench_bitcoin_SOURCES = \
bench/bench_bitcoin.cpp \
bench/bench.cpp \
bench/bench.h \
bench/MilliSleep.cpp
bench_bench_bitcoin_CPPFLAGS = $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/
bench_bench_bitcoin_LDADD = \
$(LIBBITCOIN_SERVER) \
$(LIBBITCOIN_COMMON) \
$(LIBBITCOIN_UNIVALUE) \
$(LIBBITCOIN_UTIL) \
$(LIBBITCOIN_CRYPTO) \
$(LIBLEVELDB) \
$(LIBMEMENV) \
$(LIBSECP256K1)
if ENABLE_ZMQ
bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
if ENABLE_WALLET
bench_bench_bitcoin_LDADD += $(LIBBITCOIN_WALLET)
endif
bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno
CLEANFILES += $(CLEAN_BITCOIN_BENCH)
bitcoin_bench: $(BENCH_BINARY)
bench: $(BENCH_BINARY) FORCE
$(BENCH_BINARY)
bitcoin_bench_clean : FORCE
rm -f $(CLEAN_BITCOIN_BENCH) $(bench_bench_bitcoin_OBJECTS) $(BENCH_BINARY)

16
src/bench/MilliSleep.cpp Normal file
View file

@ -0,0 +1,16 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "bench.h"
#include "main.h"
#include "utiltime.h"
static void Sleep100ms(benchmark::State& state)
{
while (state.KeepRunning()) {
MilliSleep(100);
}
}
BENCHMARK(Sleep100ms);

60
src/bench/bench.cpp Normal file
View file

@ -0,0 +1,60 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "bench.h"
#include <iostream>
#include <sys/time.h>
using namespace benchmark;
std::map<std::string, BenchFunction> BenchRunner::benchmarks;
static double gettimedouble(void) {
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_usec * 0.000001 + tv.tv_sec;
}
BenchRunner::BenchRunner(std::string name, BenchFunction func)
{
benchmarks.insert(std::make_pair(name, func));
}
void
BenchRunner::RunAll(double elapsedTimeForOne)
{
std::cout << "Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "\n";
for (std::map<std::string,BenchFunction>::iterator it = benchmarks.begin();
it != benchmarks.end(); ++it) {
State state(it->first, elapsedTimeForOne);
BenchFunction& func = it->second;
func(state);
}
}
bool State::KeepRunning()
{
double now = gettimedouble();
if (count == 0) {
beginTime = now;
}
else {
double elapsedOne = now - lastTime;
if (elapsedOne < minTime) minTime = elapsedOne;
if (elapsedOne > maxTime) maxTime = elapsedOne;
}
lastTime = now;
++count;
if (now - beginTime < maxElapsed) return true; // Keep going
--count;
// Output results
double average = (now-beginTime)/count;
std::cout << name << "," << count << "," << minTime << "," << maxTime << "," << average << "\n";
return false;
}

69
src/bench/bench.h Normal file
View file

@ -0,0 +1,69 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_BENCH_H
#define BITCOIN_BENCH_H
// Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark
// framework (see https://github.com/google/benchmark)
// Wny not use the Google Benchmark framework? Because adding Yet Another Dependency
// (that uses cmake as its build system and has lots of features we don't need) isn't
// worth it.
/*
* Usage:
static void CODE_TO_TIME(benchmark::State& state)
{
... do any setup needed...
while (state.KeepRunning()) {
... do stuff you want to time...
}
... do any cleanup needed...
}
BENCHMARK(CODE_TO_TIME);
*/
#include <boost/function.hpp>
#include <boost/preprocessor/cat.hpp>
#include <boost/preprocessor/stringize.hpp>
#include <map>
#include <string>
namespace benchmark {
class State {
std::string name;
double maxElapsed;
double beginTime;
double lastTime, minTime, maxTime;
int64_t count;
public:
State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) {
minTime = std::numeric_limits<double>::max();
maxTime = std::numeric_limits<double>::min();
}
bool KeepRunning();
};
typedef boost::function<void(State&)> BenchFunction;
class BenchRunner
{
static std::map<std::string, BenchFunction> benchmarks;
public:
BenchRunner(std::string name, BenchFunction func);
static void RunAll(double elapsedTimeForOne=1.0);
};
}
// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo);
#define BENCHMARK(n) \
benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n);
#endif // BITCOIN_BENCH_H

View file

@ -0,0 +1,21 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "bench.h"
#include "key.h"
#include "main.h"
#include "util.h"
int
main(int argc, char** argv)
{
ECC_Start();
SetupEnvironment();
fPrintToDebugLog = false; // don't want to write to debug.log file
benchmark::BenchRunner::RunAll();
ECC_Stop();
}