From b84310f0c12f8aa3a71840106305554603478168 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Mon, 16 Nov 2026 21:17:38 -0800 Subject: [PATCH 1/3] Restrict slicing-over-time plot to the largest slice and add change tables The slicing performance-over-time plot pooled all five slice sizes into each point, mixing runtimes that differ several-fold and making the values incomparable with the method-ranking heatmap. It now uses the largest slice. generate_figures also writes performance_over__changes.csv tables comparing each method and modality between consecutive environment timepoints, with absolute, percent, and fold changes in runtime plus the corresponding downloaded bytes and web packets from the network tracking runs. Rows are sorted by the size of the change, with sub-10 ms cached reads placed last. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- src/nwb_benchmarks/database/_visualization.py | 88 +++++++++++++++++-- 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3a12996..2c42c23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = ["src/nwb_benchmarks"] [project] name = "nwb_benchmarks" -version="0.4.3" +version="0.4.4" authors = [ { name="Cody Baker", email="cody.baker@catalystneuro.com" }, ] diff --git a/src/nwb_benchmarks/database/_visualization.py b/src/nwb_benchmarks/database/_visualization.py index ef84439..f2de21c 100644 --- a/src/nwb_benchmarks/database/_visualization.py +++ b/src/nwb_benchmarks/database/_visualization.py @@ -2521,12 +2521,13 @@ def plot_performance_across_versions( print(f"Plotting performance over time") df = db.filter_results_for_environment_timepoints() - df = ( - df.filter(pl.col("benchmark_name_type") == benchmark_type) - .filter(pl.col("benchmark_name_clean").is_in(self.pynwb_read_order)) - .collect() - .to_pandas() + df = df.filter(pl.col("benchmark_name_type") == benchmark_type).filter( + pl.col("benchmark_name_clean").is_in(self.pynwb_read_order) ) + if benchmark_type == "time_remote_slicing": + # Pooling slice sizes mixes runtimes that differ several-fold; match Figure 3 by using the largest slice + df = df.filter(pl.col("slice_number") == 5) + df = df.collect().to_pandas() if df.empty: self._record_skipped_output( @@ -2534,6 +2535,8 @@ def plot_performance_across_versions( ) return + self._write_performance_change_table(db=db, benchmark_type=benchmark_type) + summary_group_cols = [ "benchmark_name_type", "environment_timepoint", @@ -2583,12 +2586,87 @@ def plot_performance_across_versions( "Note that the 2025-09-01 timepoint is an estimate for approximately when that environment was generated. " "The 2026-10-07 timepoint is the Python 3.13 environment, dated by its first benchmark run. " ) + if benchmark_type == "time_remote_slicing": + caption += "Slicing results show the largest slice. " g.figure.text(0.5, -0.01, caption, ha="center", va="top", fontsize=9, wrap=True, style="italic") sns.despine() plt.savefig(self.output_directory / f"performance_over_{benchmark_type}.pdf", dpi=300, bbox_inches="tight") plt.close() + def _write_performance_change_table(self, *, db: BenchmarkDatabase, benchmark_type: str) -> None: + """Write a table of runtime and network changes between consecutive environment timepoints.""" + if self.summary_tables_directory is None: + return + + network_type = benchmark_type.replace("time_", "network_tracking_") + network_variables = { + "amount_downloaded_in_bytes": "downloaded_bytes", + "total_traffic_in_number_of_web_packets": "web_packets", + } + group_cols = ["environment_timepoint", "benchmark_name_clean", "modality"] + if benchmark_type == "time_remote_slicing": + group_cols.append("is_preloaded") + + df = db.filter_results_for_environment_timepoints().filter( + pl.col("benchmark_name_type").is_in([benchmark_type, network_type]) + & pl.col("benchmark_name_clean").is_in(self.pynwb_read_order) + ) + if benchmark_type == "time_remote_slicing": + df = df.filter(pl.col("slice_number") == 5) + # Joining with environments repeats every sample once per tracked package + df = df.drop(["package_name", "package_version"]).unique().collect().to_pandas() + + runtime = ( + df[df["benchmark_name_type"].eq(benchmark_type)] + .groupby(group_cols, dropna=False)["value"] + .agg(n="count", mean_s="mean") + .reset_index() + ) + network = df[df["benchmark_name_type"].eq(network_type) & df["variable"].isin(network_variables)] + network = ( + network.groupby([*group_cols, "variable"], dropna=False)["value"] + .mean() + .unstack("variable") + .rename(columns=network_variables) + .reset_index() + ) + per_environment = runtime.merge(network, on=group_cols, how="left") + if per_environment.empty: + self._record_skipped_output( + "summary table", f"performance_over_{benchmark_type}_changes.csv", "No data available." + ) + return + + timepoints = sorted(per_environment["environment_timepoint"].unique()) + key_cols = [col for col in group_cols if col != "environment_timepoint"] + value_cols = [col for col in ["n", "mean_s", *network_variables.values()] if col in per_environment] + pairs = [] + for from_timepoint, to_timepoint in zip(timepoints[:-1], timepoints[1:]): + from_df = per_environment[per_environment["environment_timepoint"].eq(from_timepoint)] + to_df = per_environment[per_environment["environment_timepoint"].eq(to_timepoint)] + pair = from_df[[*key_cols, *value_cols]].merge( + to_df[[*key_cols, *value_cols]], on=key_cols, suffixes=("_from", "_to") + ) + pair.insert(0, "to_environment", to_timepoint) + pair.insert(0, "from_environment", from_timepoint) + pairs.append(pair) + changes = pd.concat(pairs, ignore_index=True) + + changes["absolute_change_s"] = changes["mean_s_to"] - changes["mean_s_from"] + changes["percent_change"] = 100 * changes["absolute_change_s"] / changes["mean_s_from"] + changes["fold_change"] = changes["mean_s_to"] / changes["mean_s_from"] + for name in network_variables.values(): + if f"{name}_from" in changes: + percent_change = 100 * (changes[f"{name}_to"] / changes[f"{name}_from"] - 1) + changes[f"{name}_percent_change"] = percent_change.replace([np.inf, -np.inf], np.nan) + + # Sub-10 ms runtimes are reads served from cache, where large relative changes are negligible in practice + changes["below_10ms"] = changes[["mean_s_from", "mean_s_to"]].max(axis=1) < 0.01 + changes["abs_log2_fold_change"] = np.abs(np.log2(changes["fold_change"])) + changes = changes.sort_values(["below_10ms", "abs_log2_fold_change"], ascending=[True, False]) + changes.to_csv(self.summary_tables_directory / f"performance_over_{benchmark_type}_changes.csv", index=False) + def _plot_environment_specific_figures(self, db: BenchmarkDatabase): """Generate all plots that should be scoped to one environment context.""" From c5aba0acbf7d9661b435c14ffc1c859fd054376f Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Mon, 16 Nov 2026 22:03:41 -0800 Subject: [PATCH 2/3] Use readable panel and legend titles in the performance-over-time plots Panel titles showed raw facet variables (e.g., "is_preloaded = False | modality = Ecephys") and the legend title showed the internal column name "benchmark_name_clean". Panels are now titled by modality, with "first read" or "repeated read" for the slicing plot, and the legend is titled "Method". Co-Authored-By: Claude Opus 5.5 --- src/nwb_benchmarks/database/_visualization.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nwb_benchmarks/database/_visualization.py b/src/nwb_benchmarks/database/_visualization.py index f2de21c..7e88a1c 100644 --- a/src/nwb_benchmarks/database/_visualization.py +++ b/src/nwb_benchmarks/database/_visualization.py @@ -2575,6 +2575,13 @@ def plot_performance_across_versions( display_labels = dict(zip(method_order, self._display_method_labels(method_order))) for text in g._legend.texts: text.set_text(display_labels.get(text.get_text(), text.get_text())) + g._legend.set_title("Method") + for facet_key, ax in g.axes_dict.items(): + if benchmark_type == "time_remote_slicing": + is_preloaded, modality = facet_key + ax.set_title(f"{modality}, {'repeated read' if is_preloaded else 'first read'}") + else: + ax.set_title(facet_key) g.set(xlabel="Environment timepoint", ylabel="Time (s)") # Add figure caption From 1ddc1b4d581b796e33ab54aaa1e4a56b2c7665ca Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Mon, 16 Nov 2026 22:13:46 -0800 Subject: [PATCH 3/3] Name slicing panels "first slice read" and "repeated slice read" Co-Authored-By: Claude Opus 5.5 --- src/nwb_benchmarks/database/_visualization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/database/_visualization.py b/src/nwb_benchmarks/database/_visualization.py index 7e88a1c..aae6db9 100644 --- a/src/nwb_benchmarks/database/_visualization.py +++ b/src/nwb_benchmarks/database/_visualization.py @@ -2579,7 +2579,7 @@ def plot_performance_across_versions( for facet_key, ax in g.axes_dict.items(): if benchmark_type == "time_remote_slicing": is_preloaded, modality = facet_key - ax.set_title(f"{modality}, {'repeated read' if is_preloaded else 'first read'}") + ax.set_title(f"{modality}, {'repeated slice read' if is_preloaded else 'first slice read'}") else: ax.set_title(facet_key) g.set(xlabel="Environment timepoint", ylabel="Time (s)")