From 1eef05658812a881af9b143aed723102e015908b Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 28 Dec 2018 12:38:46 +0000 Subject: [PATCH] performance_tests: better stats, and keep track of timing history --- contrib/epee/include/stats.h | 58 ++++ contrib/epee/include/stats.inl | 359 ++++++++++++++++++++ src/common/CMakeLists.txt | 2 + src/common/perf_timer.h | 1 + src/common/timings.cc | 125 +++++++ src/common/timings.h | 34 ++ tests/performance_tests/main.cpp | 5 + tests/performance_tests/performance_tests.h | 116 +++---- 8 files changed, 638 insertions(+), 62 deletions(-) create mode 100644 contrib/epee/include/stats.h create mode 100644 contrib/epee/include/stats.inl create mode 100644 src/common/timings.cc create mode 100644 src/common/timings.h diff --git a/contrib/epee/include/stats.h b/contrib/epee/include/stats.h new file mode 100644 index 000000000..1cf9c68fb --- /dev/null +++ b/contrib/epee/include/stats.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +template +class Stats +{ +public: + Stats(const std::vector &v): values(v), cached(0) {} + ~Stats() {} + + size_t get_size() const; + Tpod get_min() const; + Tpod get_max() const; + Tpod get_median() const; + double get_mean() const; + double get_confidence_interval_95() const; + double get_confidence_interval_99() const; + double get_standard_deviation() const; + double get_standard_error() const; + double get_variance() const; + double get_kurtosis() const; + double get_non_parametric_skew() const; + double get_t_test(T t) const; + double get_t_test(size_t npoints, double mean, double stddev) const; + double get_t_test(const Stats &other) const; + double get_z_test(const Stats &other) const; + double get_test(const Stats &other) const; + std::vector get_quantiles(unsigned int quantiles) const; + std::vector get_bins(unsigned int bins) const; + bool is_same_distribution_95(size_t npoints, double mean, double stddev) const; + bool is_same_distribution_95(const Stats &other) const; + bool is_same_distribution_99(size_t npoints, double mean, double stddev) const; + bool is_same_distribution_99(const Stats &other) const; + + double get_cdf95(size_t df) const; + double get_cdf95(const Stats &other) const; + double get_cdf99(size_t df) const; + double get_cdf99(const Stats &other) const; + +private: + inline bool is_cached(int bit) const; + inline void set_cached(int bit) const; + + const std::vector &values; + + mutable uint64_t cached; + mutable Tpod min; + mutable Tpod max; + mutable Tpod median; + mutable double mean; + mutable double standard_deviation; + mutable double standard_error; + mutable double variance; + mutable double kurtosis; +}; + +#include "stats.inl" diff --git a/contrib/epee/include/stats.inl b/contrib/epee/include/stats.inl new file mode 100644 index 000000000..5a5cd0b93 --- /dev/null +++ b/contrib/epee/include/stats.inl @@ -0,0 +1,359 @@ +#include +#include +#include +#include "stats.h" + +enum +{ + bit_min = 0, + bit_max, + bit_median, + bit_mean, + bit_standard_deviation, + bit_standard_error, + bit_variance, + bit_kurtosis, +}; + +static inline double square(double x) +{ + return x * x; +} + +template +static inline double interpolate(T v, T v0, double i0, T v1, double i1) +{ + return i0 + (i1 - i0) * (v - v0) / (v1 - v0); +} + +template +inline bool Stats::is_cached(int bit) const +{ + return cached & (1< +inline void Stats::set_cached(int bit) const +{ + cached |= 1< +size_t Stats::get_size() const +{ + return values.size(); +} + +template +Tpod Stats::get_min() const +{ + if (!is_cached(bit_min)) + { + min = std::numeric_limits::max(); + for (const T &v: values) + min = std::min(min, v); + set_cached(bit_min); + } + return min; +} + +template +Tpod Stats::get_max() const +{ + if (!is_cached(bit_max)) + { + max = std::numeric_limits::min(); + for (const T &v: values) + max = std::max(max, v); + set_cached(bit_max); + } + return max; +} + +template +Tpod Stats::get_median() const +{ + if (!is_cached(bit_median)) + { + std::vector sorted; + sorted.reserve(values.size()); + for (const T &v: values) + sorted.push_back(v); + std::sort(sorted.begin(), sorted.end()); + if (sorted.size() & 1) + { + median = sorted[sorted.size() / 2]; + } + else + { + median = (sorted[(sorted.size() - 1) / 2] + sorted[sorted.size() / 2]) / 2; + } + set_cached(bit_median); + } + return median; +} + +template +double Stats::get_mean() const +{ + if (values.empty()) + return 0.0; + if (!is_cached(bit_mean)) + { + mean = 0.0; + for (const T &v: values) + mean += v; + mean /= values.size(); + set_cached(bit_mean); + } + return mean; +} + +template +double Stats::get_cdf95(size_t df) const +{ + static const double p[101] = { + -1, 12.706, 4.3027, 3.1824, 2.7765, 2.5706, 2.4469, 2.3646, 2.3060, 2.2622, 2.2281, 2.2010, 2.1788, 2.1604, 2.1448, 2.1315, + 2.1199, 2.1098, 2.1009, 2.0930, 2.0860, 2.0796, 2.0739, 2.0687, 2.0639, 2.0595, 2.0555, 2.0518, 2.0484, 2.0452, 2.0423, 2.0395, + 2.0369, 2.0345, 2.0322, 2.0301, 2.0281, 2.0262, 2.0244, 2.0227, 2.0211, 2.0195, 2.0181, 2.0167, 2.0154, 2.0141, 2.0129, 2.0117, + 2.0106, 2.0096, 2.0086, 2.0076, 2.0066, 2.0057, 2.0049, 2.0040, 2.0032, 2.0025, 2.0017, 2.0010, 2.0003, 1.9996, 1.9990, 1.9983, + 1.9977, 1.9971, 1.9966, 1.9960, 1.9955, 1.9949, 1.9944, 1.9939, 1.9935, 1.9930, 1.9925, 1.9921, 1.9917, 1.9913, 1.9908, 1.9905, + 1.9901, 1.9897, 1.9893, 1.9890, 1.9886, 1.9883, 1.9879, 1.9876, 1.9873, 1.9870, 1.9867, 1.9864, 1.9861, 1.9858, 1.9855, 1.9852, + 1.9850, 1.9847, 1.9845, 1.9842, 1.9840, + }; + if (df <= 100) + return p[df]; + if (df <= 120) + return interpolate(df, 100, 1.9840, 120, 1.98); + return 1.96; +} + +template +double Stats::get_cdf95(const Stats &other) const +{ + return get_cdf95(get_size() + other.get_size() - 2); +} + +template +double Stats::get_cdf99(size_t df) const +{ + static const double p[101] = { + -1, 9.9250, 5.8408, 4.6041, 4.0321, 3.7074, 3.4995, 3.3554, 3.2498, 3.1693, 3.1058, 3.0545, 3.0123, 2.9768, 2.9467, 2.9208, 2.8982, + 2.8784, 2.8609, 2.8453, 2.8314, 2.8188, 2.8073, 2.7970, 2.7874, 2.7787, 2.7707, 2.7633, 2.7564, 2.7500, 2.7440, 2.7385, 2.7333, + 2.7284, 2.7238, 2.7195, 2.7154, 2.7116, 2.7079, 2.7045, 2.7012, 2.6981, 2.6951, 2.6923, 2.6896, 2.6870, 2.6846, 2.6822, 2.6800, + 2.6778, 2.6757, 2.6737, 2.6718, 2.6700, 2.6682, 2.6665, 2.6649, 2.6633, 2.6618, 2.6603, 2.6589, 2.6575, 2.6561, 2.6549, 2.6536, + 2.6524, 2.6512, 2.6501, 2.6490, 2.6479, 2.6469, 2.6458, 2.6449, 2.6439, 2.6430, 2.6421, 2.6412, 2.6403, 2.6395, 2.6387, 2.6379, + 2.6371, 2.6364, 2.6356, 2.6349, 2.6342, 2.6335, 2.6329, 2.6322, 2.6316, 2.6309, 2.6303, 2.6297, 2.6291, 2.6286, 2.6280, 2.6275, + 2.6269, 2.6264, 2.6259, + }; + if (df <= 100) + return p[df]; + if (df <= 120) + return interpolate(df, 100, 2.6529, 120, 2.617); + return 2.576; +} + +template +double Stats::get_cdf99(const Stats &other) const +{ + return get_cdf99(get_size() + other.get_size() - 2); +} + +template +double Stats::get_confidence_interval_95() const +{ + const size_t df = get_size() - 1; + return get_standard_error() * get_cdf95(df); +} + +template +double Stats::get_confidence_interval_99() const +{ + const size_t df = get_size() - 1; + return get_standard_error() * get_cdf99(df); +} + +template +bool Stats::is_same_distribution_95(size_t npoints, double mean, double stddev) const +{ + return fabs(get_t_test(npoints, mean, stddev)) < get_cdf95(get_size() + npoints - 2); +} + +template +bool Stats::is_same_distribution_95(const Stats &other) const +{ + return fabs(get_t_test(other)) < get_cdf95(other); +} + +template +bool Stats::is_same_distribution_99(size_t npoints, double mean, double stddev) const +{ + return fabs(get_t_test(npoints, mean, stddev)) < get_cdf99(get_size() + npoints - 2); +} + +template +bool Stats::is_same_distribution_99(const Stats &other) const +{ + return fabs(get_t_test(other)) < get_cdf99(other); +} + +template +double Stats::get_standard_deviation() const +{ + if (values.size() <= 1) + return 0.0; + if (!is_cached(bit_standard_deviation)) + { + Tpod m = get_mean(), t = 0; + for (const T &v: values) + t += ((T)v - m) * ((T)v - m); + standard_deviation = sqrt(t / ((double)values.size() - 1)); + set_cached(bit_standard_deviation); + } + return standard_deviation; +} + +template +double Stats::get_standard_error() const +{ + if (!is_cached(bit_standard_error)) + { + standard_error = get_standard_deviation() / sqrt(get_size()); + set_cached(bit_standard_error); + } + return standard_error; +} + +template +double Stats::get_variance() const +{ + if (!is_cached(bit_variance)) + { + double stddev = get_standard_deviation(); + variance = stddev * stddev; + set_cached(bit_variance); + } + return variance; +} + +template +double Stats::get_kurtosis() const +{ + if (values.empty()) + return 0.0; + if (!is_cached(bit_kurtosis)) + { + double m = get_mean(); + double n = 0, d = 0; + for (const T &v: values) + { + T p2 = (v - m) * (v - m); + T p4 = p2 * p2; + n += p4; + d += p2; + } + n /= values.size(); + d /= values.size(); + d *= d; + kurtosis = n / d; + set_cached(bit_kurtosis); + } + return kurtosis; +} + +template +double Stats::get_non_parametric_skew() const +{ + return (get_mean() - get_median()) / get_standard_deviation(); +} + +template +double Stats::get_t_test(T t) const +{ + const double n = get_mean() - t; + const double d = get_standard_deviation() / sqrt(get_size()); + return n / d; +} + +template +double Stats::get_t_test(size_t npoints, double mean, double stddev) const +{ + const double n = get_mean() - mean; + const double d = sqrt(get_variance() / get_size() + square(stddev) / npoints); + return n / d; +} + +template +double Stats::get_t_test(const Stats &other) const +{ + const double n = get_mean() - other.get_mean(); + const double d = sqrt(get_variance() / get_size() + other.get_variance() / other.get_size()); + return n / d; +} + +template +double Stats::get_z_test(const Stats &other) const +{ + const double m0 = get_mean(); + const double m1 = other.get_mean(); + const double sd0 = get_standard_deviation(); + const double sd1 = other.get_standard_deviation(); + const size_t s0 = get_size(); + const size_t s1 = other.get_size(); + + const double n = m0 - m1; + const double d = sqrt(square(sd0 / sqrt(s0)) + square(sd1 / sqrt(s1))); + + return n / d; +} + +template +double Stats::get_test(const Stats &other) const +{ + if (get_size() >= 30 && other.get_size() >= 30) + return get_z_test(other); + else + return get_t_test(other); +} + +template +std::vector Stats::get_quantiles(unsigned int quantiles) const +{ + std::vector sorted; + sorted.reserve(values.size()); + for (const T &v: values) + sorted.push_back(v); + std::sort(sorted.begin(), sorted.end()); + std::vector q(quantiles + 1, 0); + for (unsigned int i = 1; i <= quantiles; ++i) + { + unsigned idx = (unsigned)ceil(values.size() * i / (double)quantiles); + q[i] = sorted[idx - 1]; + } + if (!is_cached(bit_min)) + { + min = sorted.front(); + set_cached(bit_min); + } + q[0] = min; + if (!is_cached(bit_max)) + { + max = sorted.back(); + set_cached(bit_max); + } + return q; +} + +template +std::vector Stats::get_bins(unsigned int bins) const +{ + std::vector b(bins, 0); + const double scale = 1.0 / (get_max() - get_min()); + const T base = get_min(); + for (const T &v: values) + { + unsigned int idx = (v - base) * scale; + ++b[idx]; + } + return b; +} diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 212a1891e..bcf9cbce7 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -45,6 +45,7 @@ set(common_sources threadpool.cpp updates.cpp aligned.c + timings.cc combinator.cpp) if (STACK_TRACE) @@ -84,6 +85,7 @@ set(common_private_headers threadpool.h updates.h aligned.h + timings.h combinator.h) monero_private_headers(common diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index d859cf576..5203da205 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -53,6 +53,7 @@ public: void resume(); void reset(); uint64_t value() const; + operator uint64_t() const { return value(); } protected: uint64_t ticks; diff --git a/src/common/timings.cc b/src/common/timings.cc new file mode 100644 index 000000000..cb8deff2a --- /dev/null +++ b/src/common/timings.cc @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include +#include "misc_log_ex.h" +#include "timings.h" + +#define N_EXPECTED_FIELDS (8+11) + +TimingsDatabase::TimingsDatabase() +{ +} + +TimingsDatabase::TimingsDatabase(const std::string &filename): + filename(filename) +{ + load(); +} + +TimingsDatabase::~TimingsDatabase() +{ + save(); +} + +bool TimingsDatabase::load() +{ + instances.clear(); + + if (filename.empty()) + return true; + + FILE *f = fopen(filename.c_str(), "r"); + if (!f) + { + MDEBUG("Failed to load timings file " << filename << ": " << strerror(errno)); + return false; + } + while (1) + { + char s[4096]; + if (!fgets(s, sizeof(s), f)) + break; + char *tab = strchr(s, '\t'); + if (!tab) + { + MWARNING("Bad format: no tab found"); + continue; + } + const std::string name = std::string(s, tab - s); + std::vector fields; + char *ptr = tab + 1; + boost::split(fields, ptr, boost::is_any_of(" ")); + if (fields.size() != N_EXPECTED_FIELDS) + { + MERROR("Bad format: wrong number of fields: got " << fields.size() << " expected " << N_EXPECTED_FIELDS); + continue; + } + + instance i; + + unsigned int idx = 0; + i.t = atoi(fields[idx++].c_str()); + i.npoints = atoi(fields[idx++].c_str()); + i.min = atof(fields[idx++].c_str()); + i.max = atof(fields[idx++].c_str()); + i.mean = atof(fields[idx++].c_str()); + i.median = atof(fields[idx++].c_str()); + i.stddev = atof(fields[idx++].c_str()); + i.npskew = atof(fields[idx++].c_str()); + i.deciles.reserve(11); + for (int n = 0; n < 11; ++n) + { + i.deciles.push_back(atoi(fields[idx++].c_str())); + } + instances.insert(std::make_pair(name, i)); + } + fclose(f); + return true; +} + +bool TimingsDatabase::save() +{ + if (filename.empty()) + return true; + + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to write to file " << filename << ": " << strerror(errno)); + return false; + } + for (const auto &i: instances) + { + fprintf(f, "%s", i.first.c_str()); + fprintf(f, "\t%lu", (unsigned long)i.second.t); + fprintf(f, " %zu", i.second.npoints); + fprintf(f, " %f", i.second.min); + fprintf(f, " %f", i.second.max); + fprintf(f, " %f", i.second.mean); + fprintf(f, " %f", i.second.median); + fprintf(f, " %f", i.second.stddev); + fprintf(f, " %f", i.second.npskew); + for (uint64_t v: i.second.deciles) + fprintf(f, " %lu", (unsigned long)v); + fputc('\n', f); + } + fclose(f); + return true; +} + +std::vector TimingsDatabase::get(const char *name) const +{ + std::vector ret; + auto range = instances.equal_range(name); + for (auto i = range.first; i != range.second; ++i) + ret.push_back(i->second); + std::sort(ret.begin(), ret.end(), [](const instance &e0, const instance &e1){ return e0.t < e1.t; }); + return ret; +} + +void TimingsDatabase::add(const char *name, const instance &i) +{ + instances.insert(std::make_pair(name, i)); +} diff --git a/src/common/timings.h b/src/common/timings.h new file mode 100644 index 000000000..fb905611f --- /dev/null +++ b/src/common/timings.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include + +class TimingsDatabase +{ +public: + struct instance + { + time_t t; + size_t npoints; + double min, max, mean, median, stddev, npskew; + std::vector deciles; + }; + +public: + TimingsDatabase(); + TimingsDatabase(const std::string &filename); + ~TimingsDatabase(); + + std::vector get(const char *name) const; + void add(const char *name, const instance &data); + +private: + bool load(); + bool save(); + +private: + std::string filename; + std::multimap instances; +}; diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index 6749b71e4..bffbf699b 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -77,10 +77,12 @@ int main(int argc, char** argv) const command_line::arg_descriptor arg_verbose = { "verbose", "Verbose output", false }; const command_line::arg_descriptor arg_stats = { "stats", "Including statistics (min/median)", false }; const command_line::arg_descriptor arg_loop_multiplier = { "loop-multiplier", "Run for that many times more loops", 1 }; + const command_line::arg_descriptor arg_timings_database = { "timings-database", "Keep timings history in a file" }; command_line::add_arg(desc_options, arg_filter); command_line::add_arg(desc_options, arg_verbose); command_line::add_arg(desc_options, arg_stats); command_line::add_arg(desc_options, arg_loop_multiplier); + command_line::add_arg(desc_options, arg_timings_database); po::variables_map vm; bool r = command_line::handle_error_helper(desc_options, [&]() @@ -93,7 +95,10 @@ int main(int argc, char** argv) return 1; const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter)); + const std::string timings_database = command_line::get_arg(vm, arg_timings_database); Params p; + if (!timings_database.empty()) + p.td = TimingsDatabase(timings_database); p.verbose = command_line::get_arg(vm, arg_verbose); p.stats = command_line::get_arg(vm, arg_stats); p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier); diff --git a/tests/performance_tests/performance_tests.h b/tests/performance_tests/performance_tests.h index d37dda729..17d16b0f6 100644 --- a/tests/performance_tests/performance_tests.h +++ b/tests/performance_tests/performance_tests.h @@ -37,7 +37,9 @@ #include #include "misc_language.h" +#include "stats.h" #include "common/perf_timer.h" +#include "common/timings.h" class performance_timer { @@ -67,6 +69,7 @@ private: struct Params { + TimingsDatabase td; bool verbose; bool stats; unsigned loop_multiplier; @@ -85,6 +88,8 @@ public: bool run() { + static_assert(0 < T::loop_count, "T::loop_count must be greater than 0"); + T test; if (!test.init()) return false; @@ -106,11 +111,13 @@ public: m_per_call_timers[i].pause(); } m_elapsed = timer.elapsed_ms(); + m_stats.reset(new Stats(m_per_call_timers)); return true; } int elapsed_time() const { return m_elapsed; } + size_t get_size() const { return m_stats->get_size(); } int time_per_call(int scale = 1) const { @@ -118,59 +125,19 @@ public: return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier); } - uint64_t per_call_min() const - { - uint64_t v = std::numeric_limits::max(); - for (const auto &pt: m_per_call_timers) - v = std::min(v, pt.value()); - return v; - } + uint64_t get_min() const { return m_stats->get_min(); } + uint64_t get_max() const { return m_stats->get_max(); } + double get_mean() const { return m_stats->get_mean(); } + uint64_t get_median() const { return m_stats->get_median(); } + double get_stddev() const { return m_stats->get_standard_deviation(); } + double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); } + std::vector get_quantiles(size_t n) const { return m_stats->get_quantiles(n); } - uint64_t per_call_max() const + bool is_same_distribution(size_t npoints, double mean, double stddev) const { - uint64_t v = std::numeric_limits::min(); - for (const auto &pt: m_per_call_timers) - v = std::max(v, pt.value()); - return v; + return m_stats->is_same_distribution_99(npoints, mean, stddev); } - uint64_t per_call_mean() const - { - uint64_t v = 0; - for (const auto &pt: m_per_call_timers) - v += pt.value(); - return v / m_per_call_timers.size(); - } - - uint64_t per_call_median() const - { - std::vector values; - values.reserve(m_per_call_timers.size()); - for (const auto &pt: m_per_call_timers) - values.push_back(pt.value()); - return epee::misc_utils::median(values); - } - - uint64_t per_call_stddev() const - { - if (m_per_call_timers.size() <= 1) - return 0; - const uint64_t mean = per_call_mean(); - uint64_t acc = 0; - for (const auto &pt: m_per_call_timers) - { - int64_t dv = pt.value() - mean; - acc += dv * dv; - } - acc /= m_per_call_timers.size () - 1; - return sqrt(acc); - } - - uint64_t min_time_ns() const { return tools::ticks_to_ns(per_call_min()); } - uint64_t max_time_ns() const { return tools::ticks_to_ns(per_call_max()); } - uint64_t median_time_ns() const { return tools::ticks_to_ns(per_call_median()); } - uint64_t standard_deviation_time_ns() const { return tools::ticks_to_ns(per_call_stddev()); } - private: /** * Warm up processor core, enabling turbo boost, etc. @@ -191,10 +158,11 @@ private: int m_elapsed; Params m_params; std::vector m_per_call_timers; + std::unique_ptr> m_stats; }; template -void run_test(const std::string &filter, const Params ¶ms, const char* test_name) +void run_test(const std::string &filter, Params ¶ms, const char* test_name) { boost::smatch match; if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter))) @@ -210,10 +178,10 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_ std::cout << " elapsed: " << runner.elapsed_time() << " ms\n"; if (params.stats) { - std::cout << " min: " << runner.min_time_ns() << " ns\n"; - std::cout << " max: " << runner.max_time_ns() << " ns\n"; - std::cout << " median: " << runner.median_time_ns() << " ns\n"; - std::cout << " std dev: " << runner.standard_deviation_time_ns() << " ns\n"; + std::cout << " min: " << runner.get_min() << " ns\n"; + std::cout << " max: " << runner.get_max() << " ns\n"; + std::cout << " median: " << runner.get_median() << " ns\n"; + std::cout << " std dev: " << runner.get_stddev() << " ns\n"; } } else @@ -221,24 +189,48 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_ std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:"; } const char *unit = "ms"; - uint64_t scale = 1000000; - int time_per_call = runner.time_per_call(); - if (time_per_call < 30000) { + double scale = 1000000; + uint64_t time_per_call = runner.time_per_call(); + if (time_per_call < 100) { + scale = 1000; time_per_call = runner.time_per_call(1000); #ifdef _WIN32 unit = "\xb5s"; #else unit = "µs"; #endif - scale = 1000; } + const auto quantiles = runner.get_quantiles(10); + double min = runner.get_min(); + double max = runner.get_max(); + double med = runner.get_median(); + double mean = runner.get_mean(); + double stddev = runner.get_stddev(); + double npskew = runner.get_non_parametric_skew(); + + std::vector prev_instances = params.td.get(test_name); + params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles}); + std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : ""); if (params.stats) { - uint64_t min_ns = runner.min_time_ns() / scale; - uint64_t med_ns = runner.median_time_ns() / scale; - uint64_t stddev_ns = runner.standard_deviation_time_ns() / scale; - std::cout << " (min " << min_ns << " " << unit << ", median " << med_ns << " " << unit << ", std dev " << stddev_ns << " " << unit << ")"; + uint64_t mins = min / scale; + uint64_t maxs = max / scale; + uint64_t meds = med / scale; + uint64_t p95s = quantiles[9] / scale; + uint64_t stddevs = stddev / scale; + std::string cmp; + if (!prev_instances.empty()) + { + const TimingsDatabase::instance &prev_instance = prev_instances.back(); + if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev)) + { + double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean); + cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster"); + } +cmp += " -- " + std::to_string(prev_instance.mean); + } + std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp; } std::cout << std::endl; }