Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ packages = ["src/nwb_benchmarks"]

[project]
name = "nwb_benchmarks"
version="0.4.2"
version="0.4.3"
authors = [
{ name="Cody Baker", email="cody.baker@catalystneuro.com" },
]
Expand Down
10 changes: 10 additions & 0 deletions src/nwb_benchmarks/database/_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,15 @@
"2026-10-07": "42bfe7119727f6ab276361065e27b6e5fa5bbdc9", # Python 3.13 environment, first run on 2026-10-07
}

# Environment IDs that describe the same installed packages as a canonical environment in ENVIRONMENT_TIMEPOINTS.
# Results recorded under an alias ID are relabeled with the canonical ID during preprocessing.
EQUIVALENT_ENVIRONMENT_IDS = {
# The environment file gained a `runtime` section in database version 5.0.0 (nwb_benchmarks #212), which changed
# the environment ID. Its conda package list is identical to 42bfe711, and the runtime versions match it.
# See https://github.com/NeurodataWithoutBorders/nwb-benchmarks-results/pull/14
"e833ef393e1bef9cf9915915b1604a87c7b004dc": "42bfe7119727f6ab276361065e27b6e5fa5bbdc9",
}

# Individual benchmark samples flagged for removal during review of the raw results.
# A sample is one benchmark test case within one results file, identified by the machine ID and timestamp from the
# results filename plus the benchmark and parameter case names. All variables recorded for the sample are dropped,
Expand Down Expand Up @@ -219,6 +228,7 @@ def _preprocess_results(self, df: pl.LazyFrame) -> pl.DataFrame:
# Filter for specific machine early to reduce data volume
.filter(pl.col("machine_id") == self.machine_id if self.machine_id is not None else pl.lit(True))
.filter(pl.col("machine_id").is_in(self.machine_ids) if self.machine_ids is not None else pl.lit(True))
.with_columns(pl.col("environment_id").replace(EQUIVALENT_ENVIRONMENT_IDS))
# Filter for timestamp early to reduce data volume
.filter(
pl.col("timestamp") >= pl.lit(self.exclude_older).str.to_datetime()
Expand Down
42 changes: 42 additions & 0 deletions tests/test_equivalent_environments.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import datetime

import pytest

pl = pytest.importorskip("polars")

from nwb_benchmarks.database import BenchmarkDatabase
from nwb_benchmarks.database._processing import EQUIVALENT_ENVIRONMENT_IDS


def make_row(*, environment_id: str, variable: str) -> dict:
row = dict(
uuid=variable,
version="5.0.0",
commit_hash="abc",
environment_id=environment_id,
machine_id="machine",
timestamp=datetime.datetime(2026, 11, 6),
benchmark_name="time_remote_slicing.HDF5PyNWBRos3ContinuousSliceBenchmark.time_slice",
parameter_case_name="EcephysTestCase",
parameter_case_slice_range=["slice(0, 80, None)"],
value=1.0,
variable=variable,
value_index=0,
)
return row


def test_equivalent_environment_ids_are_relabeled():
alias_id, canonical_id = next(iter(EQUIVALENT_ENVIRONMENT_IDS.items()))
rows = [
make_row(environment_id=alias_id, variable="a"),
make_row(environment_id=canonical_id, variable="b"),
make_row(environment_id="other", variable="c"),
]
lazy_df = pl.DataFrame(rows).lazy()

database = BenchmarkDatabase(environment_ids=[canonical_id])
result = database._preprocess_results(lazy_df).collect()

assert sorted(result["variable"].to_list()) == ["a", "b"]
assert set(result["environment_id"].to_list()) == {canonical_id}
Loading