Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 115 additions & 7 deletions lib/bencher_adapter/src/adapters/c_sharp/dot_net.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use bencher_json::{BenchmarkName, JsonAny, JsonNewMetric, project::report::JsonA
use rust_decimal::Decimal;
use serde::Deserialize;

use crate::results::adapter_results::DotNetMeasure;
use crate::{
Adaptable, AdapterError, Settings,
adapters::util::{Units, latency_as_nanos},
Expand Down Expand Up @@ -36,6 +37,7 @@ pub struct Benchmark {
pub namespace: Option<BenchmarkName>,
pub method: BenchmarkName,
pub statistics: Statistics,
pub memory: Option<Memory>,
}

#[derive(Debug, Clone, Deserialize)]
Expand All @@ -51,6 +53,16 @@ pub struct Statistics {
pub interquartile_range: Decimal,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "PascalCase")]
pub struct Memory {
pub gen0_collections: u32,
pub gen1_collections: u32,
pub gen2_collections: u32,
pub total_operations: u32,
pub bytes_allocated_per_operation: u32,
}

impl DotNet {
fn convert(self, settings: Settings) -> Result<Option<AdapterResults>, AdapterError> {
let benchmarks = self.benchmarks.0;
Expand All @@ -60,7 +72,9 @@ impl DotNet {
namespace,
method,
statistics,
memory,
} = benchmark;

let Statistics {
mean,
standard_deviation,
Expand All @@ -84,24 +98,82 @@ impl DotNet {
JsonAverage::Mean => (mean, standard_deviation),
JsonAverage::Median => (median, interquartile_range),
};
let value = latency_as_nanos(average, units);
let latency_value = latency_as_nanos(average, units);
let spread = latency_as_nanos(spread, units);
let json_metric = JsonNewMetric {
value,
lower_value: Some(value - spread),
upper_value: Some(value + spread),
let json_latency_metric = JsonNewMetric {
value: latency_value,
lower_value: Some(latency_value - spread),
upper_value: Some(latency_value + spread),
};

benchmark_metrics.push((benchmark_name, json_metric));
let latency_measure = DotNetMeasure::Latency(json_latency_metric);

let mut measures = vec![latency_measure];

if let Some(m) = memory {
let allocated_json = JsonNewMetric {
value: m.bytes_allocated_per_operation.into(),
lower_value: None,
upper_value: None,
};

let allocated_measure = DotNetMeasure::Allocated(allocated_json);

measures.push(allocated_measure);

let gen0_collects_json = JsonNewMetric {
value: m.gen0_collections.into(),
lower_value: None,
upper_value: None,
};

let gen0_measure = DotNetMeasure::Gen0Collects(gen0_collects_json);

measures.push(gen0_measure);

let gen1_collects_json = JsonNewMetric {
value: m.gen1_collections.into(),
lower_value: None,
upper_value: None,
};

let gen1_measure = DotNetMeasure::Gen1Collects(gen1_collects_json);

measures.push(gen1_measure);

let gen2_collects_json = JsonNewMetric {
value: m.gen2_collections.into(),
lower_value: None,
upper_value: None,
};

let gen2_measure = DotNetMeasure::Gen2Collects(gen2_collects_json);

measures.push(gen2_measure);

let total_operations_json = JsonNewMetric {
value: m.total_operations.into(),
lower_value: None,
upper_value: None,
};

let total_operations_measure =
DotNetMeasure::TotalOperations(total_operations_json);

measures.push(total_operations_measure);
}

benchmark_metrics.push((benchmark_name, measures));
}

Ok(AdapterResults::new_latency(benchmark_metrics))
Ok(AdapterResults::new_dotnet(benchmark_metrics))
}
}

#[cfg(test)]
pub(crate) mod test_c_sharp_dot_net {
use bencher_json::project::report::JsonAverage;
use ordered_float::OrderedFloat;
use pretty_assertions::assert_eq;

use crate::{
Expand Down Expand Up @@ -242,6 +314,42 @@ pub(crate) mod test_c_sharp_dot_net {
);
}

#[test]
fn adapter_c_sharp_dot_net_memory() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can the harness collect both memory and wall-clock time measurements at the same time or are they mutually exclusive?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is above my paygrade... I have no idea 😅

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems like it does:

Yes, BenchmarkDotNet collects both simultaneously (well, technically in separate runs to avoid interference, but in the same invocation). When you add [MemoryDiagnoser], the harness:

  1. Runs timing measurements normally
  2. Performs a separate run for memory/GC metrics (so they don't skew latency numbers)
  3. Combines everything into one unified JSON output per benchmark

In the sample file, there are both:

{                                                                                                                                                                                                                                                                                        
    "Method": "Tokenize",                                                                                                                                                                                                                                                                  
    "Parameters": "ExampleFileName=expressions.step",                                                                                                                                                                                                                                      
    "Statistics": {                                                                                                                                                                                                                                                                        
      "Mean": 106.338...,                                                                                                                                                                                                                                                                  
      "Median": 105.607...,
      "StandardDeviation": 3.223...,
      ...
    },
    "Memory": {
      "Gen0Collections": 168,
      "Gen1Collections": 0,
      "Gen2Collections": 0,
      "TotalOperations": 4194304,
      "BytesAllocatedPerOperation": 672
    }
  }

With that being the case, we're going to want to make sure we:

  1. Always collect the wall-clock time measurements
  2. Optionally collect the memory measurements (if they are present)

Right now, it seems like we are only collecting the memory measurements if they are present (skipping 1.).

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I follow... How is the statistics object relevant for the memory allocations? Isn't this used for the latency measurement? That part should mostly be untouched by my changes (latency is still being collected)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great! Then we just need to make sure we are testing that the latency measures are still being collected properly.

That is, this adapter_c_sharp_dot_net_memory test should assert on each expected measure (including Latency).

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, no problem

let results = convert_c_sharp_dot_net("memory");
assert_eq!(results.inner.len(), 2);

let metrics = results
.get("BenchmarkDotNet.Samples.AllocEmptyList")
.unwrap();

let allocated = metrics.get("allocated").unwrap();
assert_eq!(allocated.value, OrderedFloat::from(368));

let gen0collects = metrics.get("gen0-collects").unwrap();
assert_eq!(gen0collects.value, OrderedFloat::from(184));

let gen1collects = metrics.get("gen1-collects").unwrap();
assert_eq!(gen1collects.value, OrderedFloat::from(0));

let gen2collects = metrics.get("gen2-collects").unwrap();
assert_eq!(gen2collects.value, OrderedFloat::from(0));

let total_operations = metrics.get("total-operations").unwrap();
assert_eq!(total_operations.value, OrderedFloat::from(0x0080_0000));

let latency = metrics.get("latency").unwrap();
assert_eq!(latency.value, OrderedFloat::from(77.494_494_120_279_95));
assert_eq!(
latency.lower_value,
Some(OrderedFloat::from(76.318_534_543_028_99))
);
assert_eq!(
latency.upper_value,
Some(OrderedFloat::from(78.670_453_697_530_92))
);
}

#[test]
fn adapter_c_sharp_dot_net_two_more_median() {
let results = convert_c_sharp_dot_net_median("two_more");
Expand Down
48 changes: 48 additions & 0 deletions lib/bencher_adapter/src/results/adapter_results.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,16 @@ pub enum AdapterMeasure {
Throughput(JsonNewMetric),
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DotNetMeasure {
Latency(JsonNewMetric),
Gen0Collects(JsonNewMetric),
Gen1Collects(JsonNewMetric),
Gen2Collects(JsonNewMetric),
TotalOperations(JsonNewMetric),
Allocated(JsonNewMetric),
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IaiMeasure {
Instructions(JsonNewMetric),
Expand Down Expand Up @@ -212,6 +222,44 @@ impl AdapterResults {
Some(results_map.into())
}

pub fn new_dotnet(benchmark_metrics: Vec<(BenchmarkName, Vec<DotNetMeasure>)>) -> Option<Self> {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For consistency, lets move this method above new_iai. Adapters are (mostly) sorted alphabetically across the code base.

if benchmark_metrics.is_empty() {
return None;
}

let mut results_map = HashMap::new();
for (benchmark_name, measure) in benchmark_metrics {
let metrics_value = results_map
.entry(BenchmarkNameId::new_name(benchmark_name))
.or_insert_with(AdapterMetrics::default);
for metric in measure {
let (resource_id, metric) = match metric {
DotNetMeasure::Latency(json_metric) => {
(built_in::default::Latency::name_id(), json_metric)
},
DotNetMeasure::Allocated(json_metric) => {
(built_in::dotnet::Allocated::name_id(), json_metric)
},
DotNetMeasure::Gen0Collects(json_metric) => {
(built_in::dotnet::Gen0Collects::name_id(), json_metric)
},
DotNetMeasure::Gen1Collects(json_metric) => {
(built_in::dotnet::Gen1Collects::name_id(), json_metric)
},
DotNetMeasure::Gen2Collects(json_metric) => {
(built_in::dotnet::Gen2Collects::name_id(), json_metric)
},
DotNetMeasure::TotalOperations(json_metric) => {
(built_in::dotnet::TotalOperations::name_id(), json_metric)
},
};
metrics_value.inner.insert(resource_id, metric);
}
}

Some(results_map.into())
}

#[expect(clippy::too_many_lines)]
pub fn new_gungraun(
benchmark_metrics: Vec<(BenchmarkName, Vec<GungraunMeasure>)>,
Expand Down
Loading