diff --git a/pyproject.toml b/pyproject.toml index 644c871..eefac2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = ["src/nwb_benchmarks"] [project] name = "nwb_benchmarks" -version="0.4.2" +version="0.4.3" authors = [ { name="Cody Baker", email="cody.baker@catalystneuro.com" }, ] diff --git a/src/nwb_benchmarks/database/_processing.py b/src/nwb_benchmarks/database/_processing.py index 5b7111c..dc009e4 100644 --- a/src/nwb_benchmarks/database/_processing.py +++ b/src/nwb_benchmarks/database/_processing.py @@ -39,6 +39,15 @@ "2026-10-07": "42bfe7119727f6ab276361065e27b6e5fa5bbdc9", # Python 3.13 environment, first run on 2026-10-07 } +# Environment IDs that describe the same installed packages as a canonical environment in ENVIRONMENT_TIMEPOINTS. +# Results recorded under an alias ID are relabeled with the canonical ID during preprocessing. +EQUIVALENT_ENVIRONMENT_IDS = { + # The environment file gained a `runtime` section in database version 5.0.0 (nwb_benchmarks #212), which changed + # the environment ID. Its conda package list is identical to 42bfe711, and the runtime versions match it. + # See https://github.com/NeurodataWithoutBorders/nwb-benchmarks-results/pull/14 + "e833ef393e1bef9cf9915915b1604a87c7b004dc": "42bfe7119727f6ab276361065e27b6e5fa5bbdc9", +} + # Individual benchmark samples flagged for removal during review of the raw results. # A sample is one benchmark test case within one results file, identified by the machine ID and timestamp from the # results filename plus the benchmark and parameter case names. All variables recorded for the sample are dropped, @@ -219,6 +228,7 @@ def _preprocess_results(self, df: pl.LazyFrame) -> pl.DataFrame: # Filter for specific machine early to reduce data volume .filter(pl.col("machine_id") == self.machine_id if self.machine_id is not None else pl.lit(True)) .filter(pl.col("machine_id").is_in(self.machine_ids) if self.machine_ids is not None else pl.lit(True)) + .with_columns(pl.col("environment_id").replace(EQUIVALENT_ENVIRONMENT_IDS)) # Filter for timestamp early to reduce data volume .filter( pl.col("timestamp") >= pl.lit(self.exclude_older).str.to_datetime() diff --git a/tests/test_equivalent_environments.py b/tests/test_equivalent_environments.py new file mode 100644 index 0000000..b1aca87 --- /dev/null +++ b/tests/test_equivalent_environments.py @@ -0,0 +1,42 @@ +import datetime + +import pytest + +pl = pytest.importorskip("polars") + +from nwb_benchmarks.database import BenchmarkDatabase +from nwb_benchmarks.database._processing import EQUIVALENT_ENVIRONMENT_IDS + + +def make_row(*, environment_id: str, variable: str) -> dict: + row = dict( + uuid=variable, + version="5.0.0", + commit_hash="abc", + environment_id=environment_id, + machine_id="machine", + timestamp=datetime.datetime(2026, 11, 6), + benchmark_name="time_remote_slicing.HDF5PyNWBRos3ContinuousSliceBenchmark.time_slice", + parameter_case_name="EcephysTestCase", + parameter_case_slice_range=["slice(0, 80, None)"], + value=1.0, + variable=variable, + value_index=0, + ) + return row + + +def test_equivalent_environment_ids_are_relabeled(): + alias_id, canonical_id = next(iter(EQUIVALENT_ENVIRONMENT_IDS.items())) + rows = [ + make_row(environment_id=alias_id, variable="a"), + make_row(environment_id=canonical_id, variable="b"), + make_row(environment_id="other", variable="c"), + ] + lazy_df = pl.DataFrame(rows).lazy() + + database = BenchmarkDatabase(environment_ids=[canonical_id]) + result = database._preprocess_results(lazy_df).collect() + + assert sorted(result["variable"].to_list()) == ["a", "b"] + assert set(result["environment_id"].to_list()) == {canonical_id}