mirror of
https://git.wownero.com/wownero/wownero.git
synced 2024-08-15 01:03:23 +00:00
performance_tests: better stats, and keep track of timing history
This commit is contained in:
moneromooo-monero
parent
4a0e4c7d70
commit
1eef056588
8 changed files with 638 additions and 62 deletions
|
|
@ -37,7 +37,9 @@
|
|||
#include <boost/regex.hpp>
|
||||
|
||||
#include "misc_language.h"
|
||||
#include "stats.h"
|
||||
#include "common/perf_timer.h"
|
||||
#include "common/timings.h"
|
||||
|
||||
class performance_timer
|
||||
{
|
||||
|
|
@ -67,6 +69,7 @@ private:
|
|||
|
||||
struct Params
|
||||
{
|
||||
TimingsDatabase td;
|
||||
bool verbose;
|
||||
bool stats;
|
||||
unsigned loop_multiplier;
|
||||
|
|
@ -85,6 +88,8 @@ public:
|
|||
|
||||
bool run()
|
||||
{
|
||||
static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
|
||||
|
||||
T test;
|
||||
if (!test.init())
|
||||
return false;
|
||||
|
|
@ -106,11 +111,13 @@ public:
|
|||
m_per_call_timers[i].pause();
|
||||
}
|
||||
m_elapsed = timer.elapsed_ms();
|
||||
m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int elapsed_time() const { return m_elapsed; }
|
||||
size_t get_size() const { return m_stats->get_size(); }
|
||||
|
||||
int time_per_call(int scale = 1) const
|
||||
{
|
||||
|
|
@ -118,59 +125,19 @@ public:
|
|||
return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier);
|
||||
}
|
||||
|
||||
uint64_t per_call_min() const
|
||||
{
|
||||
uint64_t v = std::numeric_limits<uint64_t>::max();
|
||||
for (const auto &pt: m_per_call_timers)
|
||||
v = std::min(v, pt.value());
|
||||
return v;
|
||||
}
|
||||
uint64_t get_min() const { return m_stats->get_min(); }
|
||||
uint64_t get_max() const { return m_stats->get_max(); }
|
||||
double get_mean() const { return m_stats->get_mean(); }
|
||||
uint64_t get_median() const { return m_stats->get_median(); }
|
||||
double get_stddev() const { return m_stats->get_standard_deviation(); }
|
||||
double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); }
|
||||
std::vector<uint64_t> get_quantiles(size_t n) const { return m_stats->get_quantiles(n); }
|
||||
|
||||
uint64_t per_call_max() const
|
||||
bool is_same_distribution(size_t npoints, double mean, double stddev) const
|
||||
{
|
||||
uint64_t v = std::numeric_limits<uint64_t>::min();
|
||||
for (const auto &pt: m_per_call_timers)
|
||||
v = std::max(v, pt.value());
|
||||
return v;
|
||||
return m_stats->is_same_distribution_99(npoints, mean, stddev);
|
||||
}
|
||||
|
||||
uint64_t per_call_mean() const
|
||||
{
|
||||
uint64_t v = 0;
|
||||
for (const auto &pt: m_per_call_timers)
|
||||
v += pt.value();
|
||||
return v / m_per_call_timers.size();
|
||||
}
|
||||
|
||||
uint64_t per_call_median() const
|
||||
{
|
||||
std::vector<uint64_t> values;
|
||||
values.reserve(m_per_call_timers.size());
|
||||
for (const auto &pt: m_per_call_timers)
|
||||
values.push_back(pt.value());
|
||||
return epee::misc_utils::median(values);
|
||||
}
|
||||
|
||||
uint64_t per_call_stddev() const
|
||||
{
|
||||
if (m_per_call_timers.size() <= 1)
|
||||
return 0;
|
||||
const uint64_t mean = per_call_mean();
|
||||
uint64_t acc = 0;
|
||||
for (const auto &pt: m_per_call_timers)
|
||||
{
|
||||
int64_t dv = pt.value() - mean;
|
||||
acc += dv * dv;
|
||||
}
|
||||
acc /= m_per_call_timers.size () - 1;
|
||||
return sqrt(acc);
|
||||
}
|
||||
|
||||
uint64_t min_time_ns() const { return tools::ticks_to_ns(per_call_min()); }
|
||||
uint64_t max_time_ns() const { return tools::ticks_to_ns(per_call_max()); }
|
||||
uint64_t median_time_ns() const { return tools::ticks_to_ns(per_call_median()); }
|
||||
uint64_t standard_deviation_time_ns() const { return tools::ticks_to_ns(per_call_stddev()); }
|
||||
|
||||
private:
|
||||
/**
|
||||
* Warm up processor core, enabling turbo boost, etc.
|
||||
|
|
@ -191,10 +158,11 @@ private:
|
|||
int m_elapsed;
|
||||
Params m_params;
|
||||
std::vector<tools::PerformanceTimer> m_per_call_timers;
|
||||
std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
void run_test(const std::string &filter, const Params ¶ms, const char* test_name)
|
||||
void run_test(const std::string &filter, Params ¶ms, const char* test_name)
|
||||
{
|
||||
boost::smatch match;
|
||||
if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter)))
|
||||
|
|
@ -210,10 +178,10 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_
|
|||
std::cout << " elapsed: " << runner.elapsed_time() << " ms\n";
|
||||
if (params.stats)
|
||||
{
|
||||
std::cout << " min: " << runner.min_time_ns() << " ns\n";
|
||||
std::cout << " max: " << runner.max_time_ns() << " ns\n";
|
||||
std::cout << " median: " << runner.median_time_ns() << " ns\n";
|
||||
std::cout << " std dev: " << runner.standard_deviation_time_ns() << " ns\n";
|
||||
std::cout << " min: " << runner.get_min() << " ns\n";
|
||||
std::cout << " max: " << runner.get_max() << " ns\n";
|
||||
std::cout << " median: " << runner.get_median() << " ns\n";
|
||||
std::cout << " std dev: " << runner.get_stddev() << " ns\n";
|
||||
}
|
||||
}
|
||||
else
|
||||
|
|
@ -221,24 +189,48 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_
|
|||
std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:";
|
||||
}
|
||||
const char *unit = "ms";
|
||||
uint64_t scale = 1000000;
|
||||
int time_per_call = runner.time_per_call();
|
||||
if (time_per_call < 30000) {
|
||||
double scale = 1000000;
|
||||
uint64_t time_per_call = runner.time_per_call();
|
||||
if (time_per_call < 100) {
|
||||
scale = 1000;
|
||||
time_per_call = runner.time_per_call(1000);
|
||||
#ifdef _WIN32
|
||||
unit = "\xb5s";
|
||||
#else
|
||||
unit = "µs";
|
||||
#endif
|
||||
scale = 1000;
|
||||
}
|
||||
const auto quantiles = runner.get_quantiles(10);
|
||||
double min = runner.get_min();
|
||||
double max = runner.get_max();
|
||||
double med = runner.get_median();
|
||||
double mean = runner.get_mean();
|
||||
double stddev = runner.get_stddev();
|
||||
double npskew = runner.get_non_parametric_skew();
|
||||
|
||||
std::vector<TimingsDatabase::instance> prev_instances = params.td.get(test_name);
|
||||
params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles});
|
||||
|
||||
std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : "");
|
||||
if (params.stats)
|
||||
{
|
||||
uint64_t min_ns = runner.min_time_ns() / scale;
|
||||
uint64_t med_ns = runner.median_time_ns() / scale;
|
||||
uint64_t stddev_ns = runner.standard_deviation_time_ns() / scale;
|
||||
std::cout << " (min " << min_ns << " " << unit << ", median " << med_ns << " " << unit << ", std dev " << stddev_ns << " " << unit << ")";
|
||||
uint64_t mins = min / scale;
|
||||
uint64_t maxs = max / scale;
|
||||
uint64_t meds = med / scale;
|
||||
uint64_t p95s = quantiles[9] / scale;
|
||||
uint64_t stddevs = stddev / scale;
|
||||
std::string cmp;
|
||||
if (!prev_instances.empty())
|
||||
{
|
||||
const TimingsDatabase::instance &prev_instance = prev_instances.back();
|
||||
if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev))
|
||||
{
|
||||
double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean);
|
||||
cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster");
|
||||
}
|
||||
cmp += " -- " + std::to_string(prev_instance.mean);
|
||||
}
|
||||
std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp;
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue