refactor(evaluator)!: unify the backend contract on evaluate/evaluate_dataset

docs/evaluator/test_doc_examples.py

@@ -5,7 +5,7 @@
 """Contract checks for the Evaluator SDK patterns used in these docs.
 
 The Evaluator docs are written against the ``nemo_evaluator`` plugin SDK
-(``evaluator.run(...)`` / ``evaluator.submit(...)``), not the old
+(``evaluator.run(...)`` / ``evaluator.run(...)``), not the old
 ``/v2/.../evaluation/metrics/jobs`` REST endpoints. This module validates the
 import paths and call contract that every runnable doc snippet relies on, so the
 docs cannot silently drift from the SDK again.
@@ -92,17 +92,19 @@ def _evaluator() -> Evaluator:
     return client.evaluator
 
 
-def test_packager_param_is_submit_only() -> None:
-    """``submit`` takes ``metric_bundle_packager``; ``run`` (local, in-process) does not."""
+def test_platform_methods_take_a_metric_bundle_packager() -> None:
+    """Both platform paths take ``metric_bundle_packager``, because metrics cross the wire.
+
+    This previously contrasted ``submit`` against a local ``run``. The plugin no longer executes
+    locally, so there is no longer a method that skips packaging.
+    """
     from nemo_evaluator.sdk import Evaluator
 
-    submit_params = inspect.signature(Evaluator.submit).parameters
-    run_params = inspect.signature(Evaluator.run).parameters
-    assert "metric_bundle_packager" in submit_params
-    assert "metric_bundle_packager" not in run_params
+    for method in (Evaluator.evaluate_dataset, Evaluator.evaluate):
+        assert "metric_bundle_packager" in inspect.signature(method).parameters
 
 
-def test_builtin_submit_does_not_require_a_packager() -> None:
+def test_builtin_metric_does_not_require_a_packager() -> None:
     """Built-in metrics bundle inline, so docs omit the packager on ``submit()``.
 
     Packager resolution happens before delegating to the executor, so we stub the
@@ -117,43 +119,20 @@ def test_builtin_submit_does_not_require_a_packager() -> None:
     evaluator = _evaluator()
     metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
     dataset = [{"expected": "Paris", "output": "Paris"}]
-    sentinel = RuntimeError("reached executor.submit (packaging resolved without a packager)")
+    sentinel = RuntimeError("reached executor.evaluate_dataset (packaging resolved without a packager)")
 
-    with patch.object(evaluator._executor, "submit", side_effect=sentinel):
-        with pytest.raises(RuntimeError, match="reached executor.submit"):
-            evaluator.submit(metric=metric, dataset=dataset)
+    with patch.object(evaluator._executor, "evaluate_dataset", side_effect=sentinel):
+        with pytest.raises(RuntimeError, match="reached executor.evaluate_dataset"):
+            evaluator.evaluate_dataset(metrics=[metric], dataset=dataset)
 
 
-def test_custom_submit_requires_an_explicit_packager() -> None:
-    """Custom (non-built-in) metrics still require an explicit packager for durable submit."""
+def test_custom_metric_requires_an_explicit_packager() -> None:
+    """Custom (non-built-in) metrics still require an explicit packager to reach the platform."""
     evaluator = _evaluator()
     dataset = [{"expected": "Paris", "output": "Paris"}]
 
     with pytest.raises(MetricBundlePackagerPolicyError, match="CloudpickleMetricBundlePackager"):
-        evaluator.submit(metric=_CustomMetric(), dataset=dataset)
-
-
-def test_run_does_not_require_metric_bundle_packager() -> None:
-    """``run()`` must not impose the submit-only packager requirement.
-
-    ``run`` executes in-process; reaching the executor (which then needs a live
-    service) proves the packager guard did not fire. We only assert the failure
-    is NOT the packager ValueError.
-    """
-    from nemo_evaluator_sdk import ExactMatchMetric
-
-    evaluator = _evaluator()
-    metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
-    dataset = [{"expected": "Paris", "output": "Paris"}]
-
-    try:
-        evaluator.run(metric=metric, dataset=dataset)
-    except ValueError as error:  # pragma: no cover - defensive
-        assert "metric_bundle_packager is required" not in str(error)
-    except Exception:
-        # Any non-ValueError (e.g. connection error to the local runtime) is fine;
-        # it means we got past argument validation.
-        pass
+        evaluator.evaluate_dataset(metrics=[_CustomMetric()], dataset=dataset)
 
 
 def main() -> None:

e2e/test_evaluator_plugin.py

@@ -40,7 +40,7 @@
 from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric
 from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric
 from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric
-from nemo_evaluator_sdk.values.results import EvaluationResult
+from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult
 from nemo_evaluator_sdk.values.scores import JSONScoreParser, RangeScore
 from nemo_platform import APIConnectionError, APIStatusError, NeMoPlatform
 from nemo_platform.types.inference import ModelProvider
@@ -133,14 +133,14 @@ def _assert_http_status(exc: APIStatusError | httpx.HTTPStatusError, status_code
     assert actual == status_code
 
 
-def _aggregate_score(result: EvaluationResult) -> Any:
+def _aggregate_score(result: BenchmarkEvaluationResult) -> Any:
     for score in result.aggregate_scores.scores:
         if score.name in EXACT_MATCH_AGGREGATE_SCORE_NAMES:
             return score
     raise AssertionError(f"No exact-match aggregate score in {result.aggregate_scores.scores!r}")
 
 
-def _rows_in_index_order(result: EvaluationResult) -> Sequence[Any]:
+def _rows_in_index_order(result: BenchmarkEvaluationResult) -> Sequence[Any]:
     """Order rows by explicit row index, preserving input order when it is absent."""
     return [
         row
@@ -153,7 +153,7 @@ def _rows_in_index_order(result: EvaluationResult) -> Sequence[Any]:
     ]
 
 
-def _row_score_values(result: EvaluationResult) -> list[float]:
+def _row_score_values(result: BenchmarkEvaluationResult) -> list[float]:
     values: list[float] = []
     seen_score_names: list[str] = []
     for row in _rows_in_index_order(result):
@@ -320,7 +320,7 @@ def _submit_input_spec(sdk: NeMoPlatform, spec: EvaluateInputSpec) -> EvaluatorJ
     return sdk.evaluator.get_job_resource(job_name)
 
 
-def _metric_output_values(result: EvaluationResult, name: str) -> list[float]:
+def _metric_output_values(result: BenchmarkEvaluationResult, name: str) -> list[float]:
     values: list[float] = []
     for row in _rows_in_index_order(result):
         for outputs in row.metrics.values():
@@ -352,10 +352,10 @@ def evaluator_sdk(sdk: NeMoPlatform, evaluator_workspace: str) -> Iterator[NeMoP
 
 @pytest.fixture(scope="module")
 def completed_offline_job(evaluator_sdk: NeMoPlatform) -> Iterator[EvaluatorJobResource]:
-    job = evaluator_sdk.evaluator.submit(
-        metric=_exact_match_metric(),
+    job = evaluator_sdk.evaluator.evaluate_dataset(
+        metrics=[_exact_match_metric()],
         dataset=_offline_rows(),
-        config=RunConfig(parallelism=1),
+        params=RunConfig(parallelism=1),
     )
     try:
         _wait_for_evaluator_job(job)
@@ -479,10 +479,10 @@ def test_fileset_fragment_and_glob_datasets(evaluator_sdk: NeMoPlatform) -> None
             "glob": (f"{workspace}/{fileset_name}#part-*.json", [1.0, 0.0, 1.0]),
         }
         for label, (reference, expected_scores) in cases.items():
-            job = evaluator_sdk.evaluator.submit(
-                metric=_exact_match_metric(),
+            job = evaluator_sdk.evaluator.evaluate_dataset(
+                metrics=[_exact_match_metric()],
                 dataset=FilesetRef(root=reference),
-                config=RunConfig(parallelism=1),
+                params=RunConfig(parallelism=1),
             )
             submitted_jobs.append((label, expected_scores, job))
 
@@ -500,10 +500,10 @@ def test_fileset_fragment_and_glob_datasets(evaluator_sdk: NeMoPlatform) -> None
 
 def test_run_config_limits_samples(evaluator_sdk: NeMoPlatform) -> None:
     rows = [{"expected": str(index), "output": str(index)} for index in range(8)]
-    job = evaluator_sdk.evaluator.submit(
-        metric=_exact_match_metric(),
+    job = evaluator_sdk.evaluator.evaluate_dataset(
+        metrics=[_exact_match_metric()],
         dataset=rows,
-        config=RunConfig(limit_samples=3, parallelism=2),
+        params=RunConfig(limit_samples=3, parallelism=2),
     )
     try:
         _wait_for_evaluator_job(job)
@@ -596,10 +596,10 @@ def test_tool_calling_metric_preserves_structured_references(evaluator_sdk: NeMo
             },
         },
     ]
-    job = evaluator_sdk.evaluator.submit(
-        metric=ToolCallingMetric(reference="{{item.expected_tool_calls}}"),
+    job = evaluator_sdk.evaluator.evaluate_dataset(
+        metrics=[ToolCallingMetric(reference="{{item.expected_tool_calls}}")],
         dataset=rows,
-        config=RunConfig(parallelism=2),
+        params=RunConfig(parallelism=2),
     )
     try:
         _wait_for_evaluator_job(job)
@@ -629,10 +629,10 @@ def test_online_evaluate_job_uses_mock_provider(
         format=ModelFormat.OPEN_AI,
     )
 
-    job = evaluator_sdk.evaluator.submit(
-        metric=_exact_match_metric(candidate=None),
+    job = evaluator_sdk.evaluator.evaluate_dataset(
+        metrics=[_exact_match_metric(candidate=None)],
         dataset=[{"question": "What is the capital of France?", "expected": "Paris"}],
-        config=RunConfigOnlineModel(
+        params=RunConfigOnlineModel(
             parallelism=1,
             request_timeout=60,
             max_retries=0,
@@ -683,10 +683,10 @@ def test_llm_judge_metric_resolves_model_ref(
             ]
         },
     )
-    job = evaluator_sdk.evaluator.submit(
-        metric=metric,
+    job = evaluator_sdk.evaluator.evaluate_dataset(
+        metrics=[metric],
         dataset=[{"answer": "Paris"}],
-        config=RunConfig(parallelism=1),
+        params=RunConfig(parallelism=1),
     )
     try:
         _wait_for_evaluator_job(job)
@@ -701,10 +701,10 @@ def _assert_runtime_input_failure(
     metric: StringCheckMetric | ExactMatchMetric,
     dataset: list[dict[str, object]] | FilesetRef,
 ) -> None:
-    job = evaluator_sdk.evaluator.submit(
-        metric=metric,
+    job = evaluator_sdk.evaluator.evaluate_dataset(
+        metrics=[metric],
         dataset=dataset,
-        config=RunConfig(parallelism=1),
+        params=RunConfig(parallelism=1),
     )
     try:
         with pytest.raises(RuntimeError):

packages/nemo_evaluator_sdk/examples/examples.py

@@ -359,8 +359,8 @@ async def run_offline_local_exact_match_example() -> None:
 
     print("Running offline exact match...")
 
-    exact_match_result = await evaluator.run(
-        metrics=exact_match,
+    exact_match_result = await evaluator.run_dataset(
+        metrics=[exact_match],
         dataset=OFFLINE_EXACT_MATCH_DATASET,
         config=RunConfig(parallelism=4),
     )
@@ -381,8 +381,8 @@ async def run_online_local_exact_match_example() -> None:
 
     print("Running local online exact match...")
 
-    exact_match_result = await evaluator.run(
-        metrics=exact_match,
+    exact_match_result = await evaluator.run_dataset(
+        metrics=[exact_match],
         target=model,
         dataset=ONLINE_EXACT_MATCH_DATASET,
         prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE,
@@ -406,7 +406,7 @@ async def run_offline_local_multi_metric_example() -> None:
 
     print("\nRunning local multi-metric evaluation...")
 
-    combined_result = await evaluator.run(
+    combined_result = await evaluator.run_dataset(
         metrics=[exact_match, custom_metric],
         dataset=OFFLINE_EXACT_MATCH_DATASET,
         config=RunConfig(parallelism=4),
@@ -435,7 +435,7 @@ async def run_offline_local_benchmark_example() -> None:
 
     print("\nRunning local benchmark evaluation...")
 
-    benchmark_result = await evaluator.run(
+    benchmark_result = await evaluator.run_dataset(
         metrics=[exact_match, contains_required_phrase],
         dataset=OFFLINE_BENCHMARK_DATASET,
         config=RunConfig(parallelism=4),
@@ -465,7 +465,7 @@ async def run_online_local_benchmark_example() -> None:
 
     print("\nRunning local online benchmark evaluation...")
 
-    benchmark_result = await evaluator.run(
+    benchmark_result = await evaluator.run_dataset(
         metrics=[exact_match, contains_required_phrase],
         target=model,
         dataset=ONLINE_BENCHMARK_DATASET,
@@ -490,7 +490,7 @@ async def run_local_benchmark_with_metric_failure_example() -> None:
     print("\nRunning local benchmark evaluation with one failing metric...")
 
     try:
-        await evaluator.run(
+        await evaluator.run_dataset(
             metrics=[exact_match, failing_metric],
             dataset=OFFLINE_BENCHMARK_DATASET,
             config=RunConfig(parallelism=4),
@@ -522,8 +522,8 @@ async def run_local_metric_with_template_failure_example() -> None:
 
     print("\nRunning local metric evaluation with an invalid metric template...")
     try:
-        await evaluator.run(
-            metrics=invalid_metric,
+        await evaluator.run_dataset(
+            metrics=[invalid_metric],
             dataset=dataset,
             config=RunConfig(parallelism=1),
         )
@@ -552,8 +552,8 @@ async def run_offline_local_llm_judge_example() -> None:
 
     print("\nRunning local LLM judge evaluation...")
 
-    llm_judge_result = await evaluator.run(
-        metrics=llm_judge_metric,
+    llm_judge_result = await evaluator.run_dataset(
+        metrics=[llm_judge_metric],
         dataset=OFFLINE_JUDGE_DATASET,
         config=RunConfig(parallelism=2),
     )
@@ -574,8 +574,8 @@ async def run_online_local_llm_judge_example() -> None:
 
     print("\nRunning local online LLM judge evaluation...")
 
-    llm_judge_result = await evaluator.run(
-        metrics=llm_judge_metric,
+    llm_judge_result = await evaluator.run_dataset(
+        metrics=[llm_judge_metric],
         target=model_with_custom_headers,
         dataset=ONLINE_JUDGE_DATASET,
         prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE,
@@ -592,8 +592,8 @@ def run_sync_example() -> None:
     """
 
     evaluator = Evaluator()
-    result = evaluator.run_sync(
-        metrics=ExactMatchMetric(reference="{{item.reference}}", candidate="{{item.actual}}"),
+    result = evaluator.run_dataset_sync(
+        metrics=[ExactMatchMetric(reference="{{item.reference}}", candidate="{{item.actual}}")],
         dataset=OFFLINE_EXACT_MATCH_DATASET[:1],  # Only run the first sample
         config=RunConfig(parallelism=1),
     )