From 33523cc6d8fcff0ccbd25721e37d7215946b9a70 Mon Sep 17 00:00:00 2001 From: Jeff Lenamon <85593689+lenamonj@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:33:44 -0400 Subject: [PATCH 1/4] Report the complexity aggregates in the family's time unit ComputeBigO left big_o.time_unit at the Run() default, so the BigO row was rendered in nanoseconds whatever unit the family declared; CSVReporter rendered the RMS row from the raw, pre-divided field, so it was smaller by the unit multiplier. Both now go through the family unit and the GetAdjusted*Time() accessors, with a millisecond family in complexity_test.cc that fails on main for both. --- docs/user_guide.md | 5 +++++ src/complexity.cc | 2 ++ src/csv_reporter.cc | 4 +++- test/complexity_test.cc | 40 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/user_guide.md b/docs/user_guide.md index 3fed9261d7..30f3608dea 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -850,6 +850,11 @@ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) ->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; }); ``` +The `BigO` row reports the coefficient in the same time unit as the rest of the +family, so a family declared with `->Unit(benchmark::kMicrosecond)` reports its +coefficient in microseconds. The `RMS` row is a normalized quantity and is +reported as a percentage, independent of the time unit. + ## Custom Benchmark Name diff --git a/src/complexity.cc b/src/complexity.cc index 8fa3f073af..c1752354d9 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -229,6 +229,8 @@ std::vector ComputeBigO( big_o.cpu_accumulated_time = result_cpu.coef; big_o.report_big_o = true; big_o.complexity = result_cpu.complexity; + // The coefficient is a time; reporters render it in the family's unit. + big_o.time_unit = reports[0].time_unit; // All the time results are reported after being multiplied by the // time unit multiplier. But since RMS is a relative quantity it diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 3e21d11f0a..e29c466922 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -125,7 +125,9 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - if (run.run_type != Run::RT_Aggregate || + // ComputeBigO stores the RMS pre-divided by the time unit multiplier; the + // GetAdjusted*Time() accessors undo that, as the other reporters rely on. + if (run.report_rms || run.run_type != Run::RT_Aggregate || run.aggregate_unit == StatisticUnit::kTime) { Out << run.GetAdjustedRealTime() << ","; Out << run.GetAdjustedCPUTime() << ","; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 64a7e72a6d..15ebba69da 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -20,13 +20,17 @@ namespace { int AddComplexityTest(const std::string& test_name, const std::string& big_o_test_name, const std::string& rms_test_name, - const std::string& big_o, int family_index) { + const std::string& big_o, int family_index, + const std::string& time_unit = "ns", + const std::string& csv_rms = "%float") { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, {"%bigo_str", "[ ]* %float " + big_o}, {"%bigo", big_o}, - {"%rms", "[ ]*[0-9]+ %"}}); + {"%rms", "[ ]*[0-9]+ %"}, + {"%unit", time_unit}, + {"%csvrms", csv_rms}}); AddCases( TC_ConsoleOut, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, @@ -46,7 +50,7 @@ int AddComplexityTest(const std::string& test_name, {"\"cpu_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next}, - {"\"time_unit\": \"ns\"$", MR_Next}, + {"\"time_unit\": \"%unit\"$", MR_Next}, {"}", MR_Next}, {"\"name\": \"%rms_name\",$"}, {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, @@ -61,7 +65,7 @@ int AddComplexityTest(const std::string& test_name, {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, {"^\"%bigo_name\"", MR_Not}, - {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); + {"^\"%rms_name\",,%csvrms,%float,,,,,,$", MR_Next}}); return 0; } @@ -266,6 +270,34 @@ const std::string complexity_capture_name = ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", complexity_capture_name + "_RMS", "N", /*family_index=*/9); +// ========================================================================= // +// ------------------- Testing BigO with a declared unit ------------------- // +// ========================================================================= // + +void BM_Complexity_O_N_ms(benchmark::State& state) { + for (auto _ : state) { + // 1us per iteration per entry, half the sizes 50% slower so the RMS is + // well above zero, reported in milliseconds + const double skew = (state.range(0) & (1 << 11)) ? 1.5 : 1.0; + state.SetIterationTime(static_cast(state.range(0)) * 1e-6 * skew); + } + state.SetComplexityN(state.range(0)); +} +BENCHMARK(BM_Complexity_O_N_ms) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->UseManualTime() + ->Unit(benchmark::kMillisecond) + ->Complexity(benchmark::oN); + +constexpr char n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time"; +constexpr char big_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_BigO"; +constexpr char rms_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_RMS"; + +ADD_COMPLEXITY_CASES(n_ms_test_name, big_o_n_ms_test_name, rms_o_n_ms_test_name, + enum_auto_big_o_n, /*family_index=*/10, "ms", + "0[.][0-9]+"); + } // end namespace // ========================================================================= // From 750d1879bdf637c33e1149822f7db9eff877bbc2 Mon Sep 17 00:00:00 2001 From: Jeff Lenamon <85593689+lenamonj@users.noreply.github.com> Date: Mon, 7 Sep 2026 08:56:49 -0400 Subject: [PATCH 2/4] complexity test: pass the unit and CSV pattern explicitly, drop a comment --- src/complexity.cc | 1 - test/complexity_test.cc | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/complexity.cc b/src/complexity.cc index c1752354d9..c431a3f035 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -229,7 +229,6 @@ std::vector ComputeBigO( big_o.cpu_accumulated_time = result_cpu.coef; big_o.report_big_o = true; big_o.complexity = result_cpu.complexity; - // The coefficient is a time; reporters render it in the family's unit. big_o.time_unit = reports[0].time_unit; // All the time results are reported after being multiplied by the diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 15ebba69da..8bf5790745 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -21,8 +21,8 @@ int AddComplexityTest(const std::string& test_name, const std::string& big_o_test_name, const std::string& rms_test_name, const std::string& big_o, int family_index, - const std::string& time_unit = "ns", - const std::string& csv_rms = "%float") { + const std::string& time_unit, + const std::string& csv_rms) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, @@ -108,15 +108,15 @@ constexpr char lambda_big_o_1[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/0); + enum_auto_big_o_1, /*family_index=*/0, "ns", "%float"); // Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/1); + enum_auto_big_o_1, /*family_index=*/1, "ns", "%float"); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - lambda_big_o_1, /*family_index=*/2); + lambda_big_o_1, /*family_index=*/2, "ns", "%float"); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -165,15 +165,15 @@ constexpr char lambda_big_o_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/3); + enum_auto_big_o_n, /*family_index=*/3, "ns", "%float"); // Add auto tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/4); + enum_auto_big_o_n, /*family_index=*/4, "ns", "%float"); // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/5); + lambda_big_o_n, /*family_index=*/5, "ns", "%float"); // ========================================================================= // // ------------------------- Testing BigO O(NlgN) ------------------------- // @@ -226,17 +226,17 @@ constexpr char lambda_big_o_n_lg_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/6); + /*family_index=*/6, "ns", "%float"); // NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/7); + /*family_index=*/7, "ns", "%float"); //// Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, - /*family_index=*/8); + /*family_index=*/8, "ns", "%float"); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -269,7 +269,7 @@ const std::string complexity_capture_name = ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", complexity_capture_name + "_RMS", "N", - /*family_index=*/9); + /*family_index=*/9, "ns", "%float"); // ========================================================================= // // ------------------- Testing BigO with a declared unit ------------------- // // ========================================================================= // From 9ab01534b47a629e78675297a2858dde8bcf6196 Mon Sep 17 00:00:00 2001 From: lenamonj Date: Mon, 7 Sep 2026 17:17:40 -0400 Subject: [PATCH 3/4] Drop the CSV reporter change The CSV reporter is deprecated, so its RMS row is left as it is and the test keeps the pattern it had. --- src/csv_reporter.cc | 4 +--- test/complexity_test.cc | 31 ++++++++++++++----------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index e29c466922..3e21d11f0a 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -125,9 +125,7 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - // ComputeBigO stores the RMS pre-divided by the time unit multiplier; the - // GetAdjusted*Time() accessors undo that, as the other reporters rely on. - if (run.report_rms || run.run_type != Run::RT_Aggregate || + if (run.run_type != Run::RT_Aggregate || run.aggregate_unit == StatisticUnit::kTime) { Out << run.GetAdjustedRealTime() << ","; Out << run.GetAdjustedCPUTime() << ","; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 8bf5790745..7551abcb39 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -21,16 +21,14 @@ int AddComplexityTest(const std::string& test_name, const std::string& big_o_test_name, const std::string& rms_test_name, const std::string& big_o, int family_index, - const std::string& time_unit, - const std::string& csv_rms) { + const std::string& time_unit) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, {"%bigo_str", "[ ]* %float " + big_o}, {"%bigo", big_o}, {"%rms", "[ ]*[0-9]+ %"}, - {"%unit", time_unit}, - {"%csvrms", csv_rms}}); + {"%unit", time_unit}}); AddCases( TC_ConsoleOut, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, @@ -65,7 +63,7 @@ int AddComplexityTest(const std::string& test_name, {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, {"^\"%bigo_name\"", MR_Not}, - {"^\"%rms_name\",,%csvrms,%float,,,,,,$", MR_Next}}); + {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); return 0; } @@ -108,15 +106,15 @@ constexpr char lambda_big_o_1[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/0, "ns", "%float"); + enum_auto_big_o_1, /*family_index=*/0, "ns"); // Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/1, "ns", "%float"); + enum_auto_big_o_1, /*family_index=*/1, "ns"); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - lambda_big_o_1, /*family_index=*/2, "ns", "%float"); + lambda_big_o_1, /*family_index=*/2, "ns"); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -165,15 +163,15 @@ constexpr char lambda_big_o_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/3, "ns", "%float"); + enum_auto_big_o_n, /*family_index=*/3, "ns"); // Add auto tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/4, "ns", "%float"); + enum_auto_big_o_n, /*family_index=*/4, "ns"); // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/5, "ns", "%float"); + lambda_big_o_n, /*family_index=*/5, "ns"); // ========================================================================= // // ------------------------- Testing BigO O(NlgN) ------------------------- // @@ -226,17 +224,17 @@ constexpr char lambda_big_o_n_lg_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/6, "ns", "%float"); + /*family_index=*/6, "ns"); // NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/7, "ns", "%float"); + /*family_index=*/7, "ns"); //// Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, - /*family_index=*/8, "ns", "%float"); + /*family_index=*/8, "ns"); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -269,7 +267,7 @@ const std::string complexity_capture_name = ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", complexity_capture_name + "_RMS", "N", - /*family_index=*/9, "ns", "%float"); + /*family_index=*/9, "ns"); // ========================================================================= // // ------------------- Testing BigO with a declared unit ------------------- // // ========================================================================= // @@ -295,8 +293,7 @@ constexpr char big_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_BigO"; constexpr char rms_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_RMS"; ADD_COMPLEXITY_CASES(n_ms_test_name, big_o_n_ms_test_name, rms_o_n_ms_test_name, - enum_auto_big_o_n, /*family_index=*/10, "ms", - "0[.][0-9]+"); + enum_auto_big_o_n, /*family_index=*/10, "ms"); } // end namespace From 3aaa6551e663ba1cc1920eb699ac17bbf4fca5b9 Mon Sep 17 00:00:00 2001 From: Jeff Lenamon <85593689+lenamonj@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:07:29 -0400 Subject: [PATCH 4/4] complexity test: check the magnitude of the millisecond BigO coefficient Until now, scaling the JSON BigO coefficients by 1000 left every test passing. Under manual time the millisecond family's real coefficient is about 1.0004e-03, so its BigO row now has to match 1\.[0-9]+e-03, and that scaling fails it. The skew comment is corrected: of the family's sizes, 1024 to 65536, only 2048 has bit 11 set. --- test/complexity_test.cc | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 7551abcb39..49622dc833 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -21,7 +21,8 @@ int AddComplexityTest(const std::string& test_name, const std::string& big_o_test_name, const std::string& rms_test_name, const std::string& big_o, int family_index, - const std::string& time_unit) { + const std::string& time_unit, + const std::string& real_coefficient) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, @@ -46,7 +47,7 @@ int AddComplexityTest(const std::string& test_name, {"\"aggregate_name\": \"BigO\",$", MR_Next}, {"\"aggregate_unit\": \"time\",$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next}, - {"\"real_coefficient\": %float,$", MR_Next}, + {"\"real_coefficient\": " + real_coefficient + ",$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next}, {"\"time_unit\": \"%unit\"$", MR_Next}, {"}", MR_Next}, @@ -106,15 +107,15 @@ constexpr char lambda_big_o_1[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/0, "ns"); + enum_auto_big_o_1, /*family_index=*/0, "ns", "%float"); // Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/1, "ns"); + enum_auto_big_o_1, /*family_index=*/1, "ns", "%float"); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - lambda_big_o_1, /*family_index=*/2, "ns"); + lambda_big_o_1, /*family_index=*/2, "ns", "%float"); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -163,15 +164,15 @@ constexpr char lambda_big_o_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/3, "ns"); + enum_auto_big_o_n, /*family_index=*/3, "ns", "%float"); // Add auto tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/4, "ns"); + enum_auto_big_o_n, /*family_index=*/4, "ns", "%float"); // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/5, "ns"); + lambda_big_o_n, /*family_index=*/5, "ns", "%float"); // ========================================================================= // // ------------------------- Testing BigO O(NlgN) ------------------------- // @@ -224,17 +225,17 @@ constexpr char lambda_big_o_n_lg_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/6, "ns"); + /*family_index=*/6, "ns", "%float"); // NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/7, "ns"); + /*family_index=*/7, "ns", "%float"); //// Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, - /*family_index=*/8, "ns"); + /*family_index=*/8, "ns", "%float"); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -267,15 +268,16 @@ const std::string complexity_capture_name = ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", complexity_capture_name + "_RMS", "N", - /*family_index=*/9, "ns"); + /*family_index=*/9, "ns", "%float"); + // ========================================================================= // // ------------------- Testing BigO with a declared unit ------------------- // // ========================================================================= // void BM_Complexity_O_N_ms(benchmark::State& state) { for (auto _ : state) { - // 1us per iteration per entry, half the sizes 50% slower so the RMS is - // well above zero, reported in milliseconds + // 1us per iteration per entry, the 2048 size 50% slower so the RMS is + // above zero, reported in milliseconds const double skew = (state.range(0) & (1 << 11)) ? 1.5 : 1.0; state.SetIterationTime(static_cast(state.range(0)) * 1e-6 * skew); } @@ -293,7 +295,8 @@ constexpr char big_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_BigO"; constexpr char rms_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_RMS"; ADD_COMPLEXITY_CASES(n_ms_test_name, big_o_n_ms_test_name, rms_o_n_ms_test_name, - enum_auto_big_o_n, /*family_index=*/10, "ms"); + enum_auto_big_o_n, /*family_index=*/10, "ms", + "1\\.[0-9]+e-03"); } // end namespace