From e1665122df58cc1b7f6ee053d65d00c842d2525d Mon Sep 17 00:00:00 2001 From: Calin Teodor <45018530+TeoSlayer@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:01:50 -0700 Subject: [PATCH 1/3] fix: save datasets selected by run mode --- .../commands/benchmark/execute.py | 9 +++++++- tests/unit/commands/test_benchmark.py | 21 ++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index ff7f95ade..fb1a93e0b 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -509,7 +509,14 @@ def setup_benchmark( # Report directory report_dir = resolve_report_dir(config) report_dir.mkdir(parents=True, exist_ok=True) - config.to_yaml_file(report_dir / "config.yaml", redact_secrets=True) + datasets = config.datasets + if test_mode == TestMode.PERF: + datasets = [ds for ds in datasets if ds.type == DatasetType.PERFORMANCE] + elif test_mode == TestMode.ACC: + datasets = [ds for ds in datasets if ds.type == DatasetType.ACCURACY] + config.with_updates(datasets=datasets).to_yaml_file( + report_dir / "config.yaml", redact_secrets=True + ) # Tokenizer check (light API call, no download) model_name = config.model_params.name diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index 9b4729a32..fefb34a74 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -3303,9 +3303,16 @@ def test_agentic_trajectory_repeats_are_included_in_total_samples( class TestReportConfigSecretRedaction: @pytest.mark.unit - @pytest.mark.parametrize("test_mode", [TestMode.PERF, TestMode.ACC]) - def test_report_config_redacts_secrets_without_mutating_runtime( - self, tmp_path, test_mode + @pytest.mark.parametrize( + ("test_mode", "expected_types"), + [ + (TestMode.PERF, ["performance"]), + (TestMode.ACC, ["accuracy"]), + (TestMode.BOTH, ["accuracy", "performance"]), + ], + ) + def test_report_config_matches_mode_and_redacts_secrets( + self, tmp_path, test_mode, expected_types ): endpoint_key = "sentinel-endpoint-key" service_token = "sentinel-service-token" @@ -3327,7 +3334,8 @@ def test_report_config_redacts_secrets_without_mutating_runtime( "swebench_service_auth_token": service_token, }, }, - } + }, + {"name": "random", "type": "performance", "path": "unused.jsonl"}, ], report_dir=str(tmp_path), ) @@ -3351,7 +3359,10 @@ def test_report_config_redacts_secrets_without_mutating_runtime( persisted = (tmp_path / "config.yaml").read_text() assert endpoint_key not in persisted assert service_token not in persisted - assert persisted.count("") >= 2 + assert persisted.count("") >= (1 if test_mode == TestMode.PERF else 2) + saved = BenchmarkConfig.from_yaml_file(tmp_path / "config.yaml") + assert [ds.type.value for ds in saved.datasets] == expected_types + assert len(config.datasets) == 2 assert ctx.config.endpoint_config.api_key == endpoint_key assert ( ctx.config.datasets[0].accuracy_config.extras["swebench_service_auth_token"] From 5231e1d7e935276e5897287f1a84b773e507f35d Mon Sep 17 00:00:00 2001 From: Calin Teodor <45018530+TeoSlayer@users.noreply.github.com> Date: Thu, 5 Nov 2026 09:37:37 -0800 Subject: [PATCH 2/3] test: wait for benchmark startup before wedging metrics --- tests/integration/commands/test_sigint.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/integration/commands/test_sigint.py b/tests/integration/commands/test_sigint.py index 1197744dc..d6986981e 100644 --- a/tests/integration/commands/test_sigint.py +++ b/tests/integration/commands/test_sigint.py @@ -276,6 +276,7 @@ def test_sigint_grace_expiry_abandons_wedged_drain(mock_http_echo_server, tmp_pa """ report_dir = tmp_path / "report" config_path = tmp_path / "bench.yaml" + log_file = tmp_path / "run.log" _write_config( report_dir, mock_http_echo_server.url, @@ -286,10 +287,18 @@ def test_sigint_grace_expiry_abandons_wedged_drain(mock_http_echo_server, tmp_pa agg_pid: int | None = None try: with _benchmark_proc( - [_cli(), "benchmark", "from-config", "-c", str(config_path)] + [_cli(), "benchmark", "from-config", "-c", str(config_path)], + log_file, ) as proc: _wait_services_ready(proc, report_dir) - time.sleep(3.0) # comfortably inside the ~120 s performance phase + # Service readiness precedes client startup. Wait for the session + # before allowing live metrics ticks to reach the parent. + deadline = time.monotonic() + 60.0 + while "Starting phase:" not in log_file.read_text(errors="replace"): + assert proc.poll() is None, log_file.read_text(errors="replace") + assert time.monotonic() < deadline, "performance phase never started" + time.sleep(0.05) + time.sleep(3.0) agg_pid = _pid_of_child("metrics_aggregator", str(report_dir)) assert agg_pid is not None, "aggregator child not found" From 1af6c4887c5a0f83963bedd4ee03d0a340cc647e Mon Sep 17 00:00:00 2001 From: Calin Teodor <45018530+TeoSlayer@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:47:54 -0800 Subject: [PATCH 3/3] fix(config): retain endpoint accuracy datasets in saved perf config --- .../commands/benchmark/execute.py | 10 +++++++++- tests/unit/commands/test_benchmark.py | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index fb1a93e0b..e25425147 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -511,7 +511,15 @@ def setup_benchmark( report_dir.mkdir(parents=True, exist_ok=True) datasets = config.datasets if test_mode == TestMode.PERF: - datasets = [ds for ds in datasets if ds.type == DatasetType.PERFORMANCE] + # Match _load_datasets: only external accuracy scorers are skipped. + datasets = [ + ds + for ds in datasets + if ds.type == DatasetType.PERFORMANCE + or not _resolve_accuracy_components(ds.name, ds.accuracy_config)[ + 0 + ].SKIP_ENDPOINT_PHASE + ] elif test_mode == TestMode.ACC: datasets = [ds for ds in datasets if ds.type == DatasetType.ACCURACY] config.with_updates(datasets=datasets).to_yaml_file( diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index fefb34a74..14fdce704 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -3311,9 +3311,12 @@ class TestReportConfigSecretRedaction: (TestMode.BOTH, ["accuracy", "performance"]), ], ) + @pytest.mark.parametrize("external", [True, False]) def test_report_config_matches_mode_and_redacts_secrets( - self, tmp_path, test_mode, expected_types + self, tmp_path, test_mode, expected_types, external ): + if test_mode == TestMode.PERF and not external: + expected_types = ["accuracy", "performance"] endpoint_key = "sentinel-endpoint-key" service_token = "sentinel-service-token" config = OfflineConfig( @@ -3328,7 +3331,11 @@ def test_report_config_matches_mode_and_redacts_secrets( "type": "accuracy", "path": "unused.jsonl", "accuracy_config": { - "eval_method": "swe_bench_scorer", + "eval_method": ( + "swe_bench_scorer" if external else "string_match" + ), + "extractor": None if external else "identity_extractor", + "ground_truth": "answer", "extras": { "swebench_service_url": "http://service:18080", "swebench_service_auth_token": service_token, @@ -3359,7 +3366,9 @@ def test_report_config_matches_mode_and_redacts_secrets( persisted = (tmp_path / "config.yaml").read_text() assert endpoint_key not in persisted assert service_token not in persisted - assert persisted.count("") >= (1 if test_mode == TestMode.PERF else 2) + assert persisted.count("") >= ( + 2 if "accuracy" in expected_types else 1 + ) saved = BenchmarkConfig.from_yaml_file(tmp_path / "config.yaml") assert [ds.type.value for ds in saved.datasets] == expected_types assert len(config.datasets) == 2