From 1c5613e94ccc5a6cd7ba105f67ddc18be8010e68 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:02:31 -0600 Subject: [PATCH 01/10] feat(intake): add canonical evaluation context names Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- .../src/nemo_evaluator/intake/mapping.py | 12 ++-- .../tests/intake/test_mapping.py | 6 +- .../tests/intake/test_publish.py | 5 +- .../integration/test_publish_to_intake.py | 5 +- .../experimentalist/atif.py | 8 +-- .../nemo-experimentalist/tests/test_atif.py | 4 +- .../tests/test_experimentalist_backend.py | 4 +- .../nemo-insights/scripts/insights_demo.py | 10 +-- .../spans/seed_experiment_rollup_data.py | 4 +- .../scripts/spans/seed_experiments_demo.py | 4 +- .../nmp/intake/spans/api/traces_schemas.py | 4 +- .../src/nmp/intake/spans/ingest/atif.py | 4 +- .../intake/spans/ingest/evaluation_context.py | 51 +++++++++++++--- .../ingest/evaluation_context_validation.py | 10 +-- .../intake/tests/test_evaluation_context.py | 61 +++++++++++++++++++ .../studio/src/util/intakeTelemetry.test.ts | 30 ++++++++- .../studio/src/util/intakeTelemetry.ts | 16 ++++- 17 files changed, 188 insertions(+), 50 deletions(-) create mode 100644 services/intake/tests/test_evaluation_context.py diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py b/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py index 963543e640..cd9463ff2d 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py @@ -70,14 +70,14 @@ def session_id_for(run_id: str, trial_id: str) -> str: return f"{run_id}:{trial_id}" -def run_task_to_evaluation_context(trial: AgentEvalTrial, *, experiment_id: str) -> EvaluationContextParam: +def run_task_to_evaluation_context(trial: AgentEvalTrial, *, evaluation_name: str) -> EvaluationContextParam: """Build the lean ingest ``evaluation_context`` for a trial. - Only ``evaluation_id`` (the Evaluation's name — ``experiment_id`` holds it) and - ``test_case_id`` live here. Dataset, group, and free-form metadata belong on the - Evaluation entity (created separately via the platform SDK), not on the per-ingest context. + Only ``evaluation_name`` and ``test_case_name`` live here. Dataset, group, and free-form + metadata belong on the Evaluation entity (created separately via the platform SDK), not on + the per-ingest context. """ - return {"evaluation_id": experiment_id, "test_case_id": trial.task_id} + return {"evaluation_name": evaluation_name, "test_case_name": trial.task_id} def trial_to_atif_ingest( @@ -132,7 +132,7 @@ def trial_to_atif_ingest( "session_id": session_id_for(run_id, trial.id), "agent": agent, "steps": [step], - "evaluation_context": run_task_to_evaluation_context(trial, experiment_id=experiment_id), + "evaluation_context": run_task_to_evaluation_context(trial, evaluation_name=experiment_id), } if final_metrics is not None: body["final_metrics"] = final_metrics diff --git a/plugins/nemo-evaluator/tests/intake/test_mapping.py b/plugins/nemo-evaluator/tests/intake/test_mapping.py index 27362cbd8c..258f44d49e 100644 --- a/plugins/nemo-evaluator/tests/intake/test_mapping.py +++ b/plugins/nemo-evaluator/tests/intake/test_mapping.py @@ -79,8 +79,8 @@ def test_session_id_is_stable_per_trial() -> None: def test_evaluation_context_is_lean() -> None: - context = run_task_to_evaluation_context(_trial(task_id="task-42"), experiment_id="bench-x-variant") - assert context == {"evaluation_id": "bench-x-variant", "test_case_id": "task-42"} + context = run_task_to_evaluation_context(_trial(task_id="task-42"), evaluation_name="bench-x-variant") + assert context == {"evaluation_name": "bench-x-variant", "test_case_name": "task-42"} # --- trial_to_atif_ingest --------------------------------------------------- @@ -99,7 +99,7 @@ def test_trial_to_atif_ingest_shape() -> None: assert body["session_id"] == "run-1:t-1" assert body["agent"] == {"name": "my-agent", "version": DEFAULT_AGENT_VERSION, "model_name": "gpt-4o"} assert body["steps"] == [{"source": "agent", "step_id": 1, "message": "final answer", "timestamp": STARTED_AT}] - assert body["evaluation_context"] == {"evaluation_id": "exp-1", "test_case_id": "task-1"} + assert body["evaluation_context"] == {"evaluation_name": "exp-1", "test_case_name": "task-1"} assert "final_metrics" not in body diff --git a/plugins/nemo-evaluator/tests/intake/test_publish.py b/plugins/nemo-evaluator/tests/intake/test_publish.py index c79f70bade..b94640c25e 100644 --- a/plugins/nemo-evaluator/tests/intake/test_publish.py +++ b/plugins/nemo-evaluator/tests/intake/test_publish.py @@ -160,7 +160,10 @@ async def test_publishes_trajectory_and_scores() -> None: assert len(client.atif_calls) == 1 assert client.atif_calls[0]["session_id"] == "run-1:t-1" - assert client.atif_calls[0]["evaluation_context"] == {"evaluation_id": "exp-1", "test_case_id": "task-1"} + assert client.atif_calls[0]["evaluation_context"] == { + "evaluation_name": "exp-1", + "test_case_name": "task-1", + } # 3 metric outputs across the two score records -> 3 evaluator-result rows. assert len(client.eval_calls) == 3 assert {call["name"] for call in client.eval_calls} == {"accuracy.score", "accuracy.passed", "latency.p50"} diff --git a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py index bbad2ca1e7..8d49f4d8ff 100644 --- a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py +++ b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py @@ -292,8 +292,9 @@ async def test_publish_to_intake_round_trip(platform_base_url: str) -> None: assert trace.session_id == t1.session_id assert trace.root_span_id == t1.span_id assert trace.evaluation_context is not None - assert trace.evaluation_context.evaluation_id == EXPERIMENT_NAME - assert trace.evaluation_context.test_case_id == "task-1" + evaluation_context = trace.evaluation_context.to_dict() + assert evaluation_context["evaluation_name"] == EXPERIMENT_NAME + assert evaluation_context["test_case_name"] == "task-1" # --- trial-1 scores: every field, every data_type coercion. rows = await client.intake.spans.evaluator_results.list(t1.span_id, workspace=WORKSPACE) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/atif.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/atif.py index dd2c4e75ce..05a6ce5a3a 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/atif.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/atif.py @@ -66,8 +66,8 @@ def build_ingest_payload( Args: ref(ResourceRef): Resource reference to the ATIF trajectory file. - evaluation_name(str): Evaluation name, used as ``evaluation_context.evaluation_id``. - task_id(str): Test case id, used as ``evaluation_context.test_case_id``. + evaluation_name(str): Evaluation name, used as ``evaluation_context.evaluation_name``. + task_id(str): Test case name, used as ``evaluation_context.test_case_name``. agent_attrs(dict[str, str]): OTLP-style agent attributes used as fallbacks. Returns: @@ -75,8 +75,8 @@ def build_ingest_payload( """ trajectory = _load(ref) trajectory["evaluation_context"] = { - "evaluation_id": evaluation_name, - "test_case_id": task_id, + "evaluation_name": evaluation_name, + "test_case_name": task_id, } agent = trajectory.get("agent") if not isinstance(agent, dict): diff --git a/plugins/nemo-experimentalist/tests/test_atif.py b/plugins/nemo-experimentalist/tests/test_atif.py index 0c64f073ae..ce841d60e0 100644 --- a/plugins/nemo-experimentalist/tests/test_atif.py +++ b/plugins/nemo-experimentalist/tests/test_atif.py @@ -75,8 +75,8 @@ def test_build_ingest_payload_stamps_evaluation_context(tmp_path): agent_attrs={}, ) assert payload["evaluation_context"] == { - "evaluation_id": "exp-1", - "test_case_id": "tau3-airline/case-a", + "evaluation_name": "exp-1", + "test_case_name": "tau3-airline/case-a", } diff --git a/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py b/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py index 22624b8390..c0cec95f35 100644 --- a/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py +++ b/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py @@ -435,8 +435,8 @@ async def test_upload_trace_atif_posts_to_the_atif_ingest_endpoint(tmp_path): post = client.posts[0] assert post["url"] == "/apis/intake/v2/workspaces/ws-1/ingest/atif" assert post["body"]["evaluation_context"] == { - "evaluation_id": "exp-1", - "test_case_id": "case-a", + "evaluation_name": "exp-1", + "test_case_name": "case-a", } assert post["body"]["agent"]["model_name"] == "gpt-5-mini" diff --git a/plugins/nemo-insights/scripts/insights_demo.py b/plugins/nemo-insights/scripts/insights_demo.py index c9d9247025..85dcd1084c 100755 --- a/plugins/nemo-insights/scripts/insights_demo.py +++ b/plugins/nemo-insights/scripts/insights_demo.py @@ -58,7 +58,7 @@ class DemoError(RuntimeError): @dataclass(frozen=True) class SessionSpec: session_id: str - test_case_id: str + test_case_name: str started_at: datetime latency_ms: int cost_usd: float @@ -118,7 +118,7 @@ def _evaluation( sessions=tuple( SessionSpec( session_id=f"insights-demo-{name}-{index + 1:02d}", - test_case_id=f"case-{index + 1:02d}", + test_case_name=f"case-{index + 1:02d}", started_at=_BASE_TIME + timedelta(minutes=7 * (start_index + index)), latency_ms=latency_ms + index * 125, cost_usd=round(cost_usd + index * 0.002, 3), @@ -378,8 +378,8 @@ def ingest_session(self, evaluation: EvaluationSpec, session: SessionSpec) -> No "schema_version": "ATIF-v1.7", "session_id": session.session_id, "evaluation_context": { - "evaluation_id": evaluation.name, - "test_case_id": session.test_case_id, + "evaluation_name": evaluation.name, + "test_case_name": session.test_case_name, }, "extra": { "verifier": { @@ -403,7 +403,7 @@ def ingest_session(self, evaluation: EvaluationSpec, session: SessionSpec) -> No "step_id": 1, "timestamp": _iso(session.started_at), "source": "user", - "message": f"Investigate support request {session.test_case_id}.", + "message": f"Investigate support request {session.test_case_name}.", }, { "step_id": 2, diff --git a/services/intake/scripts/spans/seed_experiment_rollup_data.py b/services/intake/scripts/spans/seed_experiment_rollup_data.py index 9dba4b9cea..8bc4e5a4b6 100644 --- a/services/intake/scripts/spans/seed_experiment_rollup_data.py +++ b/services/intake/scripts/spans/seed_experiment_rollup_data.py @@ -175,8 +175,8 @@ def _atif_body( "schema_version": "ATIF-v1.7", "session_id": session_id, "evaluation_context": { - "evaluation_id": evaluation_id, - "test_case_id": test_case_id, + "evaluation_name": evaluation_id, + "test_case_name": test_case_id, }, "extra": { "task_id": test_case_id, diff --git a/services/intake/scripts/spans/seed_experiments_demo.py b/services/intake/scripts/spans/seed_experiments_demo.py index 9e8afe92fe..447b208a79 100644 --- a/services/intake/scripts/spans/seed_experiments_demo.py +++ b/services/intake/scripts/spans/seed_experiments_demo.py @@ -593,8 +593,8 @@ def _demo_atif_body( "schema_version": "ATIF-v1.7", "session_id": session_id, "evaluation_context": { - "evaluation_id": evaluation_id, - "test_case_id": test_case_id, + "evaluation_name": evaluation_id, + "test_case_name": test_case_id, }, "extra": { "task_id": test_case_id, diff --git a/services/intake/src/nmp/intake/spans/api/traces_schemas.py b/services/intake/src/nmp/intake/spans/api/traces_schemas.py index e902b99bb8..9f343a41f7 100644 --- a/services/intake/src/nmp/intake/spans/api/traces_schemas.py +++ b/services/intake/src/nmp/intake/spans/api/traces_schemas.py @@ -104,6 +104,6 @@ def _evaluation_context(trace: IntakeTrace) -> EvaluationContext | None: if trace.evaluation_id is None: return None return EvaluationContext( - evaluation_id=trace.evaluation_id, - test_case_id=trace.test_case_id, + evaluation_name=trace.evaluation_id, + test_case_name=trace.test_case_id, ) diff --git a/services/intake/src/nmp/intake/spans/ingest/atif.py b/services/intake/src/nmp/intake/spans/ingest/atif.py index 72e7d369b7..75631bcf2b 100644 --- a/services/intake/src/nmp/intake/spans/ingest/atif.py +++ b/services/intake/src/nmp/intake/spans/ingest/atif.py @@ -137,6 +137,6 @@ async def ingest_atif( ) await service.ingest_batch(TraceBatch(spans=spans, evaluator_results=evaluator_results)) context = body.evaluation_context - if denormalizer is not None and context is not None and context.evaluation_id: - denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_id) + if denormalizer is not None and context is not None and context.evaluation_name: + denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_name) return Response(status_code=status.HTTP_201_CREATED) diff --git a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py index c29d08a34a..b7d19d9ce6 100644 --- a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py +++ b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py @@ -5,17 +5,50 @@ from __future__ import annotations -from pydantic import BaseModel, ConfigDict, Field +from typing import Any +from pydantic import BaseModel, ConfigDict, Field, model_validator -class EvaluationContext(BaseModel): - """Evaluation context accepted by ingest endpoints (the canonical shape). - - ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, - metadata) keeps ingesting without error rather than being rejected. - """ - evaluation_id: str | None = Field(default=None, description="Name of an existing Evaluation.") - test_case_id: str | None = Field(default=None, description="Optional producer-supplied test case id.") +class EvaluationContext(BaseModel): + """Identifies the Evaluation and optional test case associated with ingested telemetry.""" + + evaluation_name: str | None = Field(default=None, description="Name of an existing Evaluation.") + test_case_name: str | None = Field(default=None, description="Optional producer-supplied test case name.") + evaluation_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for evaluation_name. Use evaluation_name instead.", + ) + test_case_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for test_case_name. Use test_case_name instead.", + ) model_config = ConfigDict(extra="ignore") + + @model_validator(mode="before") + @classmethod + def normalize_deprecated_fields(cls, data: Any) -> Any: + """Accept either spelling and keep both response fields consistent.""" + if not isinstance(data, dict): + return data + normalized = dict(data) + cls._normalize_field_pair(normalized, canonical="evaluation_name", deprecated="evaluation_id") + cls._normalize_field_pair(normalized, canonical="test_case_name", deprecated="test_case_id") + return normalized + + @staticmethod + def _normalize_field_pair(data: dict[str, Any], *, canonical: str, deprecated: str) -> None: + canonical_value = data.get(canonical) + deprecated_value = data.get(deprecated) + if canonical_value is not None and deprecated_value is not None and canonical_value != deprecated_value: + raise ValueError(f"{canonical} and deprecated {deprecated} must match when both are provided") + value = canonical_value if canonical_value is not None else deprecated_value + if value is not None: + data[canonical] = value + data[deprecated] = value + + def has_values(self) -> bool: + return self.evaluation_name is not None or self.test_case_name is not None diff --git a/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py b/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py index 9cba54dfbf..e56c265476 100644 --- a/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py +++ b/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py @@ -17,18 +17,18 @@ async def validate_evaluation_context( ) -> None: if context is None: return - experiment_id = context.evaluation_id - if not experiment_id: + evaluation_name = context.evaluation_name + if not evaluation_name: return try: - experiment = await entity_client.get(Experiment, name=experiment_id, workspace=workspace) + experiment = await entity_client.get(Experiment, name=evaluation_name, workspace=workspace) except EntityNotFoundError as exc: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Evaluation '{experiment_id}' must be created before it can be logged.", + detail=f"Evaluation '{evaluation_name}' must be created before it can be logged.", ) from exc if experiment.is_deleted: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Evaluation '{experiment_id}' has been deleted and cannot accept new sessions.", + detail=f"Evaluation '{evaluation_name}' has been deleted and cannot accept new sessions.", ) diff --git a/services/intake/tests/test_evaluation_context.py b/services/intake/tests/test_evaluation_context.py new file mode 100644 index 0000000000..15c7d6d02d --- /dev/null +++ b/services/intake/tests/test_evaluation_context.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compatibility tests for evaluation-context identifier renames.""" + +import pytest +from nmp.intake.spans.ingest.evaluation_context import EvaluationContext +from pydantic import ValidationError + +EXPECTED_CONTEXT = { + "evaluation_name": "eval-a", + "test_case_name": "case-a", + "evaluation_id": "eval-a", + "test_case_id": "case-a", +} + + +def test_evaluation_context_accepts_canonical_fields() -> None: + context = EvaluationContext.model_validate({"evaluation_name": "eval-a", "test_case_name": "case-a"}) + + assert context.model_dump() == EXPECTED_CONTEXT + + +@pytest.mark.parametrize( + "payload", + [ + {"evaluation_id": "eval-a", "test_case_id": "case-a"}, + {"evaluation_name": "eval-a", "test_case_id": "case-a"}, + { + "evaluation_name": "eval-a", + "evaluation_id": "eval-a", + "test_case_name": "case-a", + "test_case_id": "case-a", + }, + ], +) +def test_evaluation_context_normalizes_deprecated_and_mixed_fields(payload: dict[str, str]) -> None: + context = EvaluationContext.model_validate(payload) + + assert context.model_dump() == EXPECTED_CONTEXT + + +@pytest.mark.parametrize( + ("canonical", "deprecated"), + [ + ("evaluation_name", "evaluation_id"), + ("test_case_name", "test_case_id"), + ], +) +def test_evaluation_context_rejects_conflicting_names(canonical: str, deprecated: str) -> None: + with pytest.raises(ValidationError, match=f"{canonical} and deprecated {deprecated} must match"): + EvaluationContext.model_validate({canonical: "new-value", deprecated: "old-value"}) + + +def test_evaluation_context_schema_marks_old_fields_deprecated() -> None: + properties = EvaluationContext.model_json_schema()["properties"] + + assert properties["evaluation_name"].get("deprecated") is not True + assert properties["test_case_name"].get("deprecated") is not True + assert properties["evaluation_id"]["deprecated"] is True + assert properties["test_case_id"]["deprecated"] is True diff --git a/web/packages/studio/src/util/intakeTelemetry.test.ts b/web/packages/studio/src/util/intakeTelemetry.test.ts index 2e9985f2e9..7d991e535f 100644 --- a/web/packages/studio/src/util/intakeTelemetry.test.ts +++ b/web/packages/studio/src/util/intakeTelemetry.test.ts @@ -1,13 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { SpanKind, SpanStatus, type Span } from '@nemo/sdk/generated/platform/schema'; +import { + SpanKind, + SpanStatus, + type Span, + type SpanEvaluationContext, +} from '@nemo/sdk/generated/platform/schema'; import { buildSpanHierarchyRows, buildSpanTree, compareSpansByStartedAt, formatCost, + getEvaluationContextSummary, getSpansDurationMs, + hasEvaluationContext, type SpanTreeNode, } from '@studio/util/intakeTelemetry'; @@ -21,6 +28,27 @@ const makeSpan = (span: Partial & Pick): S ...span, }); +describe('evaluation context helpers', () => { + it('prefers canonical names', () => { + const context: SpanEvaluationContext = { + evaluation_name: 'new-evaluation', + test_case_name: 'new-test-case', + evaluation_id: 'deprecated-evaluation', + test_case_id: 'deprecated-test-case', + }; + + expect(getEvaluationContextSummary(context)).toBe('new-evaluation'); + expect(hasEvaluationContext(context)).toBe(true); + }); + + it('continues to display deprecated fields during the compatibility window', () => { + const context: SpanEvaluationContext = { evaluation_id: 'legacy-evaluation' }; + + expect(getEvaluationContextSummary(context)).toBe('legacy-evaluation'); + expect(hasEvaluationContext(context)).toBe(true); + }); +}); + describe('intakeTelemetry span hierarchy helpers', () => { it('formats sub-cent costs without trailing zero padding', () => { expect(formatCost(0.0032)).toBe('$0.0032'); diff --git a/web/packages/studio/src/util/intakeTelemetry.ts b/web/packages/studio/src/util/intakeTelemetry.ts index adafbed4f6..84503ce87d 100644 --- a/web/packages/studio/src/util/intakeTelemetry.ts +++ b/web/packages/studio/src/util/intakeTelemetry.ts @@ -82,11 +82,23 @@ export const getEvaluationContextSummary = ( context: SpanEvaluationContext | null | undefined ): string => { if (!context) return EMPTY_VALUE; - return context.evaluation_id || context.test_case_id || EMPTY_VALUE; + return ( + context.evaluation_name || + context.test_case_name || + context.evaluation_id || + context.test_case_id || + EMPTY_VALUE + ); }; export const hasEvaluationContext = (context: SpanEvaluationContext | null | undefined): boolean => - Boolean(context && (context.evaluation_id || context.test_case_id)); + Boolean( + context && + (context.evaluation_name || + context.test_case_name || + context.evaluation_id || + context.test_case_id) + ); export const compareSpansByStartedAt = (a: Span, b: Span): number => { const aStartedAt = Date.parse(a.started_at); From 54ef115a0f91f58dd019834f7fb0b77da47dc461 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:04:58 -0600 Subject: [PATCH 02/10] feat(intake): rename span identifier fields Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- .../intake/src/nmp/intake/spans/api/spans.py | 33 ++++++++------ .../src/nmp/intake/spans/api/spans_schemas.py | 35 +++++++-------- .../nmp/intake/spans/clickhouse_migrations.py | 2 +- .../nmp/intake/spans/ingest/atif_mapping.py | 4 +- .../intake/spans/ingest/chat_completions.py | 8 ++-- .../intake/spans/span_attribute_catalog.py | 6 +-- .../intake/spans/span_semantic_attributes.py | 4 +- .../integration/spans/test_atif_ingest.py | 44 +++++++++---------- .../spans/test_chat_completions_ingest.py | 39 ++++++++-------- .../spans/test_experiment_metric_sort.py | 2 +- .../spans/test_experiment_rollups.py | 6 +-- .../spans/test_experiment_sessions.py | 4 +- .../tests/test_spans_clickhouse_migrations.py | 2 +- .../tests/test_spans_filter_contract.py | 21 ++++++++- services/intake/tests/test_spans_schemas.py | 10 ++++- 15 files changed, 123 insertions(+), 97 deletions(-) diff --git a/services/intake/src/nmp/intake/spans/api/spans.py b/services/intake/src/nmp/intake/spans/api/spans.py index 662e1cbe82..ce75a3e7cc 100644 --- a/services/intake/src/nmp/intake/spans/api/spans.py +++ b/services/intake/src/nmp/intake/spans/api/spans.py @@ -42,18 +42,19 @@ # rather than a rejected request. test_spans_filter_contract.py holds that line: it # asserts every name below has a catalog entry, and that no SpanFilter field is # published without a way to serve it. -ATTRIBUTE_EQ_FILTER_FIELDS = frozenset( - { - "agent_id", - "agent_name", - "evaluation_id", - "model", - "project", - "provider", - "test_case_id", - "tool_name", - } -) +ATTRIBUTE_EQ_FILTER_FIELD_MAP = { + "agent_id": "agent_id", + "agent_name": "agent_name", + "evaluation_name": "evaluation_name", + "evaluation_id": "evaluation_name", + "model": "model", + "project": "project", + "provider": "provider", + "test_case_name": "test_case_name", + "test_case_id": "test_case_name", + "tool_name": "tool_name", +} +ATTRIBUTE_EQ_FILTER_FIELDS = frozenset(ATTRIBUTE_EQ_FILTER_FIELD_MAP) @router.get( @@ -64,7 +65,7 @@ openapi_extra=generate_openapi_extra_params( filter_schema=SpanFilter, filter_description=( - "Filter spans by session_id, trace_id, parent_span_id, project, evaluation_id, test_case_id, " + "Filter spans by session_id, trace_id, parent_span_id, project, evaluation_name, test_case_name, " "source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. " "Every field takes one exact value, except started_at, which takes gte and lte." ), @@ -223,7 +224,11 @@ def _span_filter(workspace: str, parsed: ParsedFilter) -> SpanListFilter: elif comparison.field == "parent_span_id": filters.external_parent_span_id = require_string_value(comparison) elif comparison.field in ATTRIBUTE_EQ_FILTER_FIELDS: - _add_attribute_eq_filter(filters, comparison.field, require_string_value(comparison)) + _add_attribute_eq_filter( + filters, + ATTRIBUTE_EQ_FILTER_FIELD_MAP[comparison.field], + require_string_value(comparison), + ) elif comparison.field == "source": filters.source_format = require_string_value(comparison) elif comparison.field == "kind": diff --git a/services/intake/src/nmp/intake/spans/api/spans_schemas.py b/services/intake/src/nmp/intake/spans/api/spans_schemas.py index a52e809a39..9dd349baeb 100644 --- a/services/intake/src/nmp/intake/spans/api/spans_schemas.py +++ b/services/intake/src/nmp/intake/spans/api/spans_schemas.py @@ -20,6 +20,7 @@ SpanStatus, ) from nmp.intake.spans.domain import SpanGroup as IntakeSpanGroup +from nmp.intake.spans.ingest.evaluation_context import EvaluationContext from nmp.intake.spans.span_attribute_bags import SpanAttributeBags from nmp.intake.spans.span_semantic_attributes import SpanSemanticAttributes from nmp.intake.spans.storage import text_for_mode @@ -51,8 +52,18 @@ class SpanFilter(BaseModel): session_id: str | None = Field(default=None, description="Filter by span session id.") trace_id: str | None = Field(default=None, description="Filter by canonical trace id.") project: str | None = Field(default=None, description="Filter by project name.") - evaluation_id: str | None = Field(default=None, description="Filter by evaluation id.") - test_case_id: str | None = Field(default=None, description="Filter by dataset test case id.") + evaluation_name: str | None = Field(default=None, description="Filter by Evaluation name.") + test_case_name: str | None = Field(default=None, description="Filter by test case name.") + evaluation_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for evaluation_name. Use evaluation_name instead.", + ) + test_case_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for test_case_name. Use test_case_name instead.", + ) source: str | None = Field( default=None, description="Filter by ingest source (e.g. 'otel', 'atif', 'chat_completions')." ) @@ -71,32 +82,20 @@ class SpanFilter(BaseModel): started_at: DatetimeFilter | None = Field(default=None, description="Filter by span start timestamp.") -class SpanEvaluationContext(BaseModel): +class SpanEvaluationContext(EvaluationContext): # Read model for span evaluation context, aligned with the ingest EvaluationContext. model_config = ConfigDict(extra="forbid") - evaluation_id: str | None = None - test_case_id: str | None = None - @classmethod def from_semantic_attributes(cls, attributes: SpanSemanticAttributes) -> Self | None: context = cls( - evaluation_id=attributes.evaluation_id, - test_case_id=attributes.test_case_id, + evaluation_name=attributes.evaluation_name, + test_case_name=attributes.test_case_name, ) - if not context.has_scalar_values(): + if not context.has_values(): return None return context - def has_scalar_values(self) -> bool: - return any( - value is not None - for value in ( - self.evaluation_id, - self.test_case_id, - ) - ) - class Span(BaseModel): span_id: str diff --git a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py index 02b704d54b..fd498e7f16 100644 --- a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py +++ b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py @@ -253,7 +253,7 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> client.command(f"DROP TABLE IF EXISTS {table}") project_key = spec_for_field(SpanAttributeField.PROJECT).bag_key - test_case_key = spec_for_field(SpanAttributeField.TEST_CASE_ID).bag_key + test_case_key = spec_for_field(SpanAttributeField.TEST_CASE_NAME).bag_key # Resolve evaluation_id by coalescing the canonical bag key with any legacy aliases. Ingest always # re-keys new spans to the canonical key, so this only matters for the backfill INSERT: spans stored diff --git a/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py b/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py index d0d89a936d..2a442fb43a 100644 --- a/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py +++ b/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py @@ -616,8 +616,8 @@ def _span_attributes( model=model, agent_name=agent_name, agent_version=agent_version, - evaluation_id=evaluation_context.evaluation_id if evaluation_context is not None else None, - test_case_id=evaluation_context.test_case_id if evaluation_context is not None else None, + evaluation_name=evaluation_context.evaluation_name if evaluation_context is not None else None, + test_case_name=evaluation_context.test_case_name if evaluation_context is not None else None, tool_name=tool_name, error_message=error_message, input_tokens=input_tokens, diff --git a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py index e461310647..1b6313b958 100644 --- a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py +++ b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py @@ -162,8 +162,8 @@ async def ingest_chat_completion( span = _chat_completion_to_span(workspace=workspace, body=body, ingested_at=ingested_at) await service.ingest_batch(TraceBatch(spans=[span])) context = body.evaluation_context - if denormalizer is not None and context is not None and context.evaluation_id: - denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_id) + if denormalizer is not None and context is not None and context.evaluation_name: + denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_name) return ChatCompletionsIngestResponse( session_id=span.session_id, span_id=span.external_span_id, @@ -235,8 +235,8 @@ def _build_attribute_bags( semantic = SpanSemanticAttributes( model=_as_str(response.get("model")) or _as_str(request.get("model")), provider=body.provider or _infer_provider(response), - evaluation_id=evaluation_context.evaluation_id if evaluation_context is not None else None, - test_case_id=evaluation_context.test_case_id if evaluation_context is not None else None, + evaluation_name=evaluation_context.evaluation_name if evaluation_context is not None else None, + test_case_name=evaluation_context.test_case_name if evaluation_context is not None else None, error_type=error_type, error_message=error_message, input_tokens=input_tokens, diff --git a/services/intake/src/nmp/intake/spans/span_attribute_catalog.py b/services/intake/src/nmp/intake/spans/span_attribute_catalog.py index 1c0bb3a4e3..4ed918b5c9 100644 --- a/services/intake/src/nmp/intake/spans/span_attribute_catalog.py +++ b/services/intake/src/nmp/intake/spans/span_attribute_catalog.py @@ -36,8 +36,8 @@ class SpanAttributeField(StrEnum): AGENT_VERSION = "agent_version" TOOL_NAME = "tool_name" PROJECT = "project" - EVALUATION_NAME = "evaluation_id" - TEST_CASE_ID = "test_case_id" + EVALUATION_NAME = "evaluation_name" + TEST_CASE_NAME = "test_case_name" ERROR_TYPE = "error_type" ERROR_MESSAGE = "error_message" INPUT_TOKENS = "input_tokens" @@ -158,7 +158,7 @@ class AttributeSpec: source_keys=("nemo.evaluation.name", "nemo.experiment.id"), ), AttributeSpec( - field=SpanAttributeField.TEST_CASE_ID, + field=SpanAttributeField.TEST_CASE_NAME, bag=AttributeBag.STRING, bag_key="nemo.test_case.id", source_keys=("nemo.test_case.id",), diff --git a/services/intake/src/nmp/intake/spans/span_semantic_attributes.py b/services/intake/src/nmp/intake/spans/span_semantic_attributes.py index 5a14a0d461..7d51350ff7 100644 --- a/services/intake/src/nmp/intake/spans/span_semantic_attributes.py +++ b/services/intake/src/nmp/intake/spans/span_semantic_attributes.py @@ -30,8 +30,8 @@ class SpanSemanticAttributes(BaseModel): agent_version: str | None = None tool_name: str | None = None project: str | None = None - evaluation_id: str | None = None - test_case_id: str | None = None + evaluation_name: str | None = None + test_case_name: str | None = None error_type: str | None = None error_message: str | None = None input_tokens: int | None = Field(default=None, ge=0) diff --git a/services/intake/tests/integration/spans/test_atif_ingest.py b/services/intake/tests/integration/spans/test_atif_ingest.py index 51c756834a..f4ffc33bcf 100644 --- a/services/intake/tests/integration/spans/test_atif_ingest.py +++ b/services/intake/tests/integration/spans/test_atif_ingest.py @@ -334,15 +334,11 @@ def test_atif_ingest_rejects_unknown_schema_version(client: TestClient): def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data(client: TestClient): - evaluation_run_id = "evalrun-01JZ8Q7K6V7R3X9N2M4P5A6B7C" evaluation_context = { - "evaluation_id": "eval-sample-agent-baseline", - "evaluation_sha": "abc132901", - "evaluation_run_id": evaluation_run_id, - "test_case_id": "sample-test-case-a", - "metadata": {"trial": "sample-test-case-a__trial-a"}, + "evaluation_name": "eval-sample-agent-baseline", + "test_case_name": "sample-test-case-a", } - _create_experiment(client, evaluation_context["evaluation_id"]) + _create_experiment(client, evaluation_context["evaluation_name"]) base_time = _BASE_TIME user_step_time = base_time agent_step_2_time = base_time + timedelta(seconds=5, milliseconds=636) @@ -529,11 +525,12 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert "cached_tokens" not in trajectory assert "total_tokens" not in trajectory assert "cost_total_usd" not in trajectory - # The retired keys (evaluation_sha, evaluation_run_id, metadata) are accepted but ignored, so only - # evaluation_id and test_case_id survive onto the root trajectory span. + # Deprecated identifiers remain present alongside the canonical names in responses. assert trajectory["evaluation_context"] == { - "evaluation_id": evaluation_context["evaluation_id"], - "test_case_id": evaluation_context["test_case_id"], + "evaluation_name": evaluation_context["evaluation_name"], + "test_case_name": evaluation_context["test_case_name"], + "evaluation_id": evaluation_context["evaluation_name"], + "test_case_id": evaluation_context["test_case_name"], } assert "attributes_string" not in trajectory trajectory_raw = json.loads(trajectory["raw_attributes"]) @@ -552,7 +549,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( evaluation_response = client.get( "/apis/intake/v2/workspaces/default/spans", params={ - "filter[evaluation_id]": evaluation_context["evaluation_id"], + "filter[evaluation_name]": evaluation_context["evaluation_name"], "filter[started_at][gte]": _HISTORICAL_GTE, "page_size": 10, }, @@ -562,14 +559,14 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert len(evaluation_spans) == 1 assert evaluation_spans[0]["name"] == "sample-agent" - for field, value in { - "evaluation_id": evaluation_context["evaluation_id"], - "test_case_id": evaluation_context["test_case_id"], - }.items(): + for filter_field, context_field, value in ( + ("evaluation_name", "evaluation_name", evaluation_context["evaluation_name"]), + ("test_case_name", "test_case_name", evaluation_context["test_case_name"]), + ): filtered = client.get( "/apis/intake/v2/workspaces/default/spans", params={ - f"filter[{field}]": value, + f"filter[{filter_field}]": value, "filter[started_at][gte]": _HISTORICAL_GTE, "page_size": 10, }, @@ -578,7 +575,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( filtered_spans = filtered.json()["data"] assert len(filtered_spans) == 1 assert filtered_spans[0]["name"] == "sample-agent" - assert filtered_spans[0]["evaluation_context"][field] == value + assert filtered_spans[0]["evaluation_context"][context_field] == value evaluator_span = spans_by_name["harbor.verifier"] assert evaluator_span["kind"] == "EVALUATOR" @@ -660,8 +657,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( "session_id": "441e9149-e4e6-41c0-82b0-a36802f83d3a", "evaluation_context": { **evaluation_context, - "test_case_id": "sample-test-case-b", - "metadata": {"trial": "sample-test-case-b__trial-b"}, + "test_case_name": "sample-test-case-b", }, "extra": { "task_name": "sample-dataset/sample-test-case-b", @@ -684,7 +680,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( evaluation_roots_response = client.get( "/apis/intake/v2/workspaces/default/spans", params={ - "filter[evaluation_id]": evaluation_context["evaluation_id"], + "filter[evaluation_name]": evaluation_context["evaluation_name"], "filter[started_at][gte]": _HISTORICAL_GTE, "page_size": 20, "sort": "started_at", @@ -694,15 +690,15 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( evaluation_roots = evaluation_roots_response.json()["data"] assert len(evaluation_roots) == 2 assert {span["name"] for span in evaluation_roots} == {"sample-agent"} - assert {span["evaluation_context"]["evaluation_id"] for span in evaluation_roots} == { - evaluation_context["evaluation_id"] + assert {span["evaluation_context"]["evaluation_name"] for span in evaluation_roots} == { + evaluation_context["evaluation_name"] } assert {span["session_id"] for span in evaluation_roots} == { "d074dfb7-3691-443c-b137-720d75e40afa", "441e9149-e4e6-41c0-82b0-a36802f83d3a", } - # Re-ingesting into the same session keeps span ids stable; evaluation_run_id is ignored on ingest. + # Re-ingesting into the same session keeps span ids stable. same_session_body = { "schema_version": "ATIF-v1.7", "session_id": body["session_id"], diff --git a/services/intake/tests/integration/spans/test_chat_completions_ingest.py b/services/intake/tests/integration/spans/test_chat_completions_ingest.py index a6012497f1..714574f52e 100644 --- a/services/intake/tests/integration/spans/test_chat_completions_ingest.py +++ b/services/intake/tests/integration/spans/test_chat_completions_ingest.py @@ -16,15 +16,14 @@ SPANS_URL = "/apis/intake/v2/workspaces/default/spans" TRACES_URL = "/apis/intake/v2/workspaces/default/traces" EVALUATION_CONTEXT: dict[str, Any] = { - "evaluation_id": "chat-eval", - "evaluation_sha": "chat-eval-sha", - "evaluation_run_id": "evalrun-chat-001", - "test_case_id": "chat-case-001", - "metadata": {"source": "chat-completions-test"}, + "evaluation_name": "chat-eval", + "test_case_name": "chat-case-001", } EXPECTED_EVALUATION_CONTEXT: dict[str, Any] = { - "evaluation_id": EVALUATION_CONTEXT["evaluation_id"], - "test_case_id": EVALUATION_CONTEXT["test_case_id"], + "evaluation_name": EVALUATION_CONTEXT["evaluation_name"], + "test_case_name": EVALUATION_CONTEXT["test_case_name"], + "evaluation_id": EVALUATION_CONTEXT["evaluation_name"], + "test_case_id": EVALUATION_CONTEXT["test_case_name"], } @@ -74,12 +73,12 @@ def _openai_response(**overrides: Any) -> dict[str, Any]: def test_chat_completions_ingest_happy_path(client: TestClient): - evaluation_id = _create_experiment(client, EVALUATION_CONTEXT["evaluation_id"]) + evaluation_name = _create_experiment(client, EVALUATION_CONTEXT["evaluation_name"]) body = { "request": _openai_request(), "response": _openai_response(), "session_id": "session-happy", - "evaluation_context": {**EVALUATION_CONTEXT, "evaluation_id": evaluation_id}, + "evaluation_context": {**EVALUATION_CONTEXT, "evaluation_name": evaluation_name}, "provider": "openai", } response = client.post(INGEST_URL, json=body) @@ -129,7 +128,7 @@ def test_chat_completions_ingest_happy_path(client: TestClient): filtered = client.get( SPANS_URL, - params={"filter[evaluation_id]": EVALUATION_CONTEXT["evaluation_id"], "page_size": 10}, + params={"filter[evaluation_name]": EVALUATION_CONTEXT["evaluation_name"], "page_size": 10}, ) assert filtered.status_code == 200, filtered.text filtered_spans = filtered.json()["data"] @@ -238,12 +237,19 @@ def test_chat_completions_ingest_handles_missing_usage(client: TestClient): def test_chat_completions_ingest_accepts_deprecated_evaluation_context(client: TestClient): - _create_experiment(client, EVALUATION_CONTEXT["evaluation_id"]) + _create_experiment(client, EVALUATION_CONTEXT["evaluation_name"]) + deprecated_context = { + "evaluation_id": EVALUATION_CONTEXT["evaluation_name"], + "test_case_id": EVALUATION_CONTEXT["test_case_name"], + "evaluation_sha": "chat-eval-sha", + "evaluation_run_id": "evalrun-chat-001", + "metadata": {"source": "chat-completions-test"}, + } body = { "request": _openai_request(), "response": _openai_response(id="chatcmpl-run-id-only"), "session_id": "session-run-id-only", - "evaluation_context": EVALUATION_CONTEXT, + "evaluation_context": deprecated_context, } response = client.post(INGEST_URL, json=body) assert response.status_code == 201, response.text @@ -251,12 +257,9 @@ def test_chat_completions_ingest_accepts_deprecated_evaluation_context(client: T listed = client.get(SPANS_URL, params={"filter[session_id]": "session-run-id-only"}) assert listed.status_code == 200, listed.text span = listed.json()["data"][0] - # The retired keys (evaluation_sha, evaluation_run_id, metadata) are accepted but ignored, so only - # evaluation_id and test_case_id survive onto the span. - assert span["evaluation_context"] == { - "evaluation_id": EVALUATION_CONTEXT["evaluation_id"], - "test_case_id": EVALUATION_CONTEXT["test_case_id"], - } + # Deprecated identifier fields remain accepted and are returned alongside their canonical names; + # retired fields (evaluation_sha, evaluation_run_id, metadata) are still ignored. + assert span["evaluation_context"] == EXPECTED_EVALUATION_CONTEXT def test_chat_completions_ingest_rejects_unknown_deprecated_evaluation_context(client: TestClient): diff --git a/services/intake/tests/integration/spans/test_experiment_metric_sort.py b/services/intake/tests/integration/spans/test_experiment_metric_sort.py index c07476726b..524d419720 100644 --- a/services/intake/tests/integration/spans/test_experiment_metric_sort.py +++ b/services/intake/tests/integration/spans/test_experiment_metric_sort.py @@ -31,7 +31,7 @@ def _atif_body(*, started_at: datetime, evaluation_id: str, cost_usd: float, off return { "schema_version": "ATIF-v1.7", "session_id": f"{evaluation_id}-session", - "evaluation_context": {"evaluation_id": evaluation_id, "test_case_id": "case-1"}, + "evaluation_context": {"evaluation_name": evaluation_id, "test_case_name": "case-1"}, "extra": {"task_name": "case-1", "verifier_result": {"rewards": {"reward": 1.0}}}, "agent": {"name": "sample-agent", "version": "1.0.0", "model_name": "provider/sample-model"}, "steps": [ diff --git a/services/intake/tests/integration/spans/test_experiment_rollups.py b/services/intake/tests/integration/spans/test_experiment_rollups.py index 8aeec7f918..ebf04a8b4c 100644 --- a/services/intake/tests/integration/spans/test_experiment_rollups.py +++ b/services/intake/tests/integration/spans/test_experiment_rollups.py @@ -292,7 +292,7 @@ def test_deprecated_evaluation_context_hydrates_evaluation_rollups(client: TestC latency_ms=100, offset_seconds=0, ), - "evaluation_context": {"evaluation_id": evaluation_id, "test_case_id": "case-1"}, + "evaluation_context": {"evaluation_name": evaluation_id, "test_case_name": "case-1"}, }, ) @@ -405,8 +405,8 @@ def _atif_body( "schema_version": "ATIF-v1.7", "session_id": session_id, "evaluation_context": { - "evaluation_id": evaluation_id, - "test_case_id": test_case_id, + "evaluation_name": evaluation_id, + "test_case_name": test_case_id, }, "extra": extra, "agent": { diff --git a/services/intake/tests/integration/spans/test_experiment_sessions.py b/services/intake/tests/integration/spans/test_experiment_sessions.py index c824846cd1..b85b663194 100644 --- a/services/intake/tests/integration/spans/test_experiment_sessions.py +++ b/services/intake/tests/integration/spans/test_experiment_sessions.py @@ -271,8 +271,8 @@ def _atif_body( "schema_version": "ATIF-v1.7", "session_id": session_id, "evaluation_context": { - "evaluation_id": evaluation_name, - "test_case_id": test_case_id, + "evaluation_name": evaluation_name, + "test_case_name": test_case_id, }, "extra": { "task_id": test_case_id, diff --git a/services/intake/tests/test_spans_clickhouse_migrations.py b/services/intake/tests/test_spans_clickhouse_migrations.py index 4a7847e417..6c58e4e68a 100644 --- a/services/intake/tests/test_spans_clickhouse_migrations.py +++ b/services/intake/tests/test_spans_clickhouse_migrations.py @@ -79,4 +79,4 @@ def test_trace_index_mv_keys_match_attribute_catalog(): assert evaluation_spec.bag_key == "nemo.evaluation.name" # The legacy key is still accepted on ingest so pre-rename producers keep associating. assert "nemo.experiment.id" in evaluation_spec.source_keys - assert spec_for_field(SpanAttributeField.TEST_CASE_ID).bag_key == "nemo.test_case.id" + assert spec_for_field(SpanAttributeField.TEST_CASE_NAME).bag_key == "nemo.test_case.id" diff --git a/services/intake/tests/test_spans_filter_contract.py b/services/intake/tests/test_spans_filter_contract.py index 6b1f52b2f0..fa5f91d5fc 100644 --- a/services/intake/tests/test_spans_filter_contract.py +++ b/services/intake/tests/test_spans_filter_contract.py @@ -18,7 +18,7 @@ from nmp.common.api.filter import parse_json_filter from nmp.common.api.parsed_filter import ParsedFilter from nmp.intake.repository.clickhouse.span import _span_where -from nmp.intake.spans.api.spans import ATTRIBUTE_EQ_FILTER_FIELDS, _span_filter +from nmp.intake.spans.api.spans import ATTRIBUTE_EQ_FILTER_FIELD_MAP, ATTRIBUTE_EQ_FILTER_FIELDS, _span_filter from nmp.intake.spans.api.spans_schemas import SpanFilter from nmp.intake.spans.span_attribute_catalog import spec_for_field @@ -63,7 +63,24 @@ def test_every_published_filter_reaches_sql(field: str) -> None: def test_every_attribute_filter_has_a_catalog_entry(field: str) -> None: # The repository resolves attribute filters through the catalog, so a field routed # there without an entry raises rather than returning a clean rejection. - assert spec_for_field(field) is not None + assert spec_for_field(ATTRIBUTE_EQ_FILTER_FIELD_MAP[field]) is not None + + +@pytest.mark.parametrize( + ("canonical", "deprecated"), + [("evaluation_name", "evaluation_id"), ("test_case_name", "test_case_id")], +) +def test_deprecated_identifier_filters_build_the_same_query(canonical: str, deprecated: str) -> None: + assert build_where(canonical) == build_where(deprecated) + + +def test_identifier_filter_schema_marks_only_old_fields_deprecated() -> None: + properties = SpanFilter.model_json_schema()["properties"] + + assert properties["evaluation_name"].get("deprecated") is not True + assert properties["test_case_name"].get("deprecated") is not True + assert properties["evaluation_id"]["deprecated"] is True + assert properties["test_case_id"]["deprecated"] is True def test_no_filter_field_is_published_without_a_way_to_serve_it() -> None: diff --git a/services/intake/tests/test_spans_schemas.py b/services/intake/tests/test_spans_schemas.py index 7c1751c997..0cead077ef 100644 --- a/services/intake/tests/test_spans_schemas.py +++ b/services/intake/tests/test_spans_schemas.py @@ -221,8 +221,14 @@ def test_trace_response_maps_core_trace_fields(): assert response.span_count == 2 assert response.error_count == 1 assert response.evaluation_context is not None - assert response.evaluation_context.evaluation_id == "experiment-a" - assert response.evaluation_context.test_case_id == "case-a" + assert response.evaluation_context.evaluation_name == "experiment-a" + assert response.evaluation_context.test_case_name == "case-a" + assert response.evaluation_context.model_dump() == { + "evaluation_name": "experiment-a", + "test_case_name": "case-a", + "evaluation_id": "experiment-a", + "test_case_id": "case-a", + } def test_trace_response_applies_payload_mode_at_api_boundary(): From a014ab03448a117e602c72442bcefd60bf71a551 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:06:14 -0600 Subject: [PATCH 03/10] feat(intake): use canonical test-case OTLP attribute Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- docs/evaluator/experiments.mdx | 6 +-- .../skills/nemo-experiments-upload/SKILL.md | 20 +++++----- .../references/harbor-quickstart.md | 14 +++---- .../references/troubleshooting.md | 7 ++-- .../nemo-intake/references/ingest-formats.md | 23 ++++++------ .../smoke-agent/scripts/record_traces.py | 2 +- .../record_tau_airline_traces.py | 2 +- .../components/trace_explorer.py | 1 + .../experimentalist_backend.py | 6 +-- .../nemo-experimentalist/tests/test_otlp.py | 4 +- .../nemo-insights/evaluation/otlp_build.py | 6 +-- plugins/nemo-insights/evaluation/reingest.py | 11 +++--- .../tests/evaluation/test_otlp_build.py | 4 +- .../tests/evaluation/test_reingest.py | 24 ++++++++---- services/intake/README.md | 8 ++-- .../scripts/spans/seed_span_type_showcase.py | 2 +- .../nmp/intake/spans/clickhouse_migrations.py | 29 +++++++-------- .../nmp/intake/spans/span_attribute_bags.py | 7 +++- .../intake/spans/span_attribute_catalog.py | 32 +++++++++++----- .../integration/spans/test_traces_read.py | 13 ++++--- services/intake/tests/test_atif_v17.py | 30 ++++++++------- .../tests/test_spans_clickhouse_migrations.py | 13 ++++--- .../test_spans_span_attribute_catalog.py | 37 +++++++++++++++++++ 23 files changed, 186 insertions(+), 115 deletions(-) diff --git a/docs/evaluator/experiments.mdx b/docs/evaluator/experiments.mdx index a043b8ed58..1b66935b92 100644 --- a/docs/evaluator/experiments.mdx +++ b/docs/evaluator/experiments.mdx @@ -235,7 +235,7 @@ session with the Evaluation's identity: `evaluation_context` object to the ingest payload carrying `evaluation_id` (the Evaluation's **name**) and `test_case_id`. - For OpenTelemetry Protocol (OTLP), set the `nemo.evaluation.name` and - `nemo.test_case.id` root-span attributes. + `nemo.test_case.name` root-span attributes. The per-evaluator scores on the leaderboard come from **evaluator results** captured on those sessions, either automatically from ATIF verifier rewards or explicitly through the evaluator-results @@ -246,10 +246,10 @@ The `nemo-experiments-upload` skill walks this through end to end. -**`test_case_id` is required for a populated leaderboard.** A session tagged with only `evaluation_id` +**`test_case_name` is required for a populated leaderboard.** A session tagged with only `evaluation_name` still ingests and appears in the Evaluation's session list, but it does not count toward `test_case_count` or any rollup. The row therefore reads as all zeros, with tokens, model, and agent -blank too. Always send `test_case_id` alongside `evaluation_id`. +blank too. Always send `test_case_name` alongside `evaluation_name`. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md index bccd0fa072..de05ee545d 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md @@ -111,7 +111,7 @@ instead of masking it. The `id` is then read with a GET, so this works on both f ### 2. Create an Evaluation -One Evaluation = one agent/config run against a dataset (one leaderboard row). Its **`name`** is what you reference later in `evaluation_context.evaluation_id`. +Create one Evaluation for each agent/config run against a dataset (one leaderboard row). ```bash curl -sf -X POST \ @@ -126,15 +126,15 @@ curl -sf -X POST \ - `experiment_ids` is a list holding the Experiment's **`id`** (from step 1). - `metadata` values must be **strings** (`dict[str, str]`). -- **You must create the Evaluation before you can log to it** — ingesting with an unknown `evaluation_id` returns `400 "…must be created before it can be logged."` +- **You must create the Evaluation before you can log to it** — ingesting with an unknown `evaluation_name` returns `400 "…must be created before it can be logged."` ### 3. Log traces + evaluator results Pick the ingest endpoint that matches your producer. **Read `../nemo-intake/references/ingest-formats.md` for the full schema and a copy-pasteable example for each.** How you attach evaluation identity depends on the endpoint: -- **ATIF and chat-completions** (JSON body) — add an `evaluation_context = {evaluation_id: "", test_case_id: ""}` object to the payload. -- **OTLP** — there is no body field; set `nemo.evaluation.name` (the Evaluation **name**) and - `nemo.test_case.id` (the task ID) as **attributes on the root span**. Spans missing these still +- **ATIF and chat-completions** (JSON body) — add an `evaluation_context = {evaluation_name: "", test_case_name: ""}` object to the payload. +- **OTLP** — there is no body field; set `nemo.evaluation.name` and `nemo.test_case.name` as + **attributes on the root span**. Spans missing these still ingest but won't associate to an Evaluation. | Producer | Endpoint | Read | @@ -179,23 +179,23 @@ You succeeded when `GET .../evaluations/my-eval-baseline` shows: - `run_count` ≥ 1 (each ingested session counts as one run), and - non-empty `evaluator_names` / `aggregate_scores` if you logged rewards, and/or `cost_usd` if your spans carried cost. -If `run_count` is 0 after ingesting, the traces didn't associate — almost always a wrong evaluation identity: `evaluation_context.evaluation_id` for ATIF/chat-completions, or the `nemo.evaluation.name` root-span attribute for OTLP (see Gotchas). +If `run_count` is 0 after ingesting, the traces didn't associate — almost always a wrong evaluation identity: `evaluation_context.evaluation_name` for ATIF/chat-completions, or the `nemo.evaluation.name` root-span attribute for OTLP (see Gotchas). ## If verification fails | Symptom | Cause | Recovery | |---|---|---| -| `400 "…must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_id` doesn't match | Create the Evaluation (step 2); ensure `evaluation_context.evaluation_id` equals its **name** | +| `400 "…must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_name` doesn't match | Create the Evaluation (step 2); ensure `evaluation_context.evaluation_name` identifies it | | `422 Unprocessable` on ingest | Unknown/typo'd top-level key (ATIF/chat-completions are `extra="forbid"`) or bad `schema_version` | Check the exact schema in `../nemo-intake/references/ingest-formats.md`; remove stray keys | -| Ingest 2xx but `run_count` stays 0 | Evaluation identity missing/wrong — `evaluation_context.evaluation_id` (ATIF/chat-completions) or the `nemo.evaluation.name` root-span attribute (OTLP) ≠ the Evaluation's name | Attach the identity for your endpoint; use the Evaluation **name**, not its id | +| Ingest 2xx but `run_count` stays 0 | Evaluation context is missing or doesn't match the target Evaluation | Attach the correct `evaluation_context.evaluation_name` (ATIF/chat-completions) or `nemo.evaluation.name` root-span attribute (OTLP) | | `503` on GET evaluation / sessions | ClickHouse (telemetry store) not running | Start ClickHouse; rollups and sessions require it | | Scores don't show up | Rewards not under `extra.verifier_result.rewards`, or wrong `data_type` on `/evaluator-results` | See `references/troubleshooting.md` | ## Gotchas - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`. -- **`evaluation_id` is the Evaluation's `name`, not its entity id.** But **`experiment_ids` holds the Experiment's `id`.** Different identifiers; easy to swap. -- **OTLP uses the attribute key `nemo.evaluation.name`** (and `nemo.test_case.id`) — set it to the Evaluation's **name** on your root span, matching the `evaluation_id` field the JSON `evaluation_context` carries on the other endpoints. +- **Evaluation and Experiment references use different fields.** `evaluation_name` associates telemetry with an Evaluation, while `experiment_ids` assigns that Evaluation to its parent Experiments. +- **OTLP evaluation context:** set `nemo.evaluation.name` and, when applicable, `nemo.test_case.name` as root-span attributes. These correspond to `evaluation_name` and `test_case_name` in the JSON `evaluation_context` used by other ingest endpoints. - **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`. - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`. - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`). diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md index d7d3b082c6..cbbb820daa 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md @@ -12,8 +12,8 @@ each task runs several times (trials). You upload **one ATIF payload per trial** | Harbor concept | NeMo entity | How | |---|---|---| | A benchmark / sweep | **Experiment** | `POST /experiments` once | -| One agent+config on that benchmark | **Evaluation** | `POST /evaluations` once (its `name` is your `evaluation_id`) | -| A task / test case | `test_case_id` | field inside `evaluation_context` | +| One agent+config on that benchmark | **Evaluation** | `POST /evaluations` once (its `name` is your `evaluation_name`) | +| A task / test case | `test_case_name` | field inside `evaluation_context` | | One trial (attempt) of a task | one ingested **session** | one `POST /ingest/atif` | ## Mapping: Harbor trial files → ATIF payload @@ -25,8 +25,8 @@ Per trial, Harbor writes result files (typically `result.json` and `agent/trajec | `agent.name` / `agent.version` / `agent.model_name` | the agent under test | | `steps[]` | the trajectory steps (`agent`/`user`/`system`, with `metrics.{prompt_tokens, completion_tokens, cost_usd}`) | | `final_metrics.{total_prompt_tokens, total_completion_tokens, total_cost_usd, total_steps}` | trajectory `final_metrics` or the trial's `agent_result` (`n_input_tokens`, `n_output_tokens`, `cost_usd`) | -| `evaluation_context.evaluation_id` | your Evaluation **name** (constant across the whole run) | -| `evaluation_context.test_case_id` | the task id (constant across that task's trials) | +| `evaluation_context.evaluation_name` | the target Evaluation (constant across the whole run) | +| `evaluation_context.test_case_name` | the task name (constant across that task's trials) | | `extra.verifier_result.rewards` | the verifier's per-criterion scores → one evaluator score row each | **Cost/tokens are pass-through.** NeMo does not recompute cost — it sums the per-call `cost_usd` / @@ -39,7 +39,7 @@ token values Harbor recorded. If Harbor didn't record a cost for a run, that run { "schema_version": "ATIF-v1.5", "session_id": "", - "evaluation_context": { "evaluation_id": "my-eval-baseline", "test_case_id": "tau-bench/airline-042" }, + "evaluation_context": { "evaluation_name": "my-eval-baseline", "test_case_name": "tau-bench/airline-042" }, "agent": { "name": "my-agent", "version": "1.0.0", "model_name": "provider/model" }, "final_metrics": { "total_prompt_tokens": 51701, "total_completion_tokens": 255, "total_cost_usd": 0.264, "total_steps": 3 }, "extra": { @@ -55,8 +55,8 @@ token values Harbor recorded. If Harbor didn't record a cost for a run, that run ## Consistency rules (so rollups aggregate correctly) -- **`evaluation_id` is identical** for every trial of the run (it's the Evaluation name). -- **`test_case_id` is identical** across all trials of the same task, and **differs** between tasks — +- **`evaluation_name` is identical** for every trial of the run. +- **`test_case_name` is identical** across all trials of the same task, and **differs** between tasks — this is what lets the platform group a task's k attempts. - Give each trial a distinct `session_id` (one session = one run in the rollup's `run_count`). diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md index 9816bd521a..bf139d6269 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md @@ -19,7 +19,7 @@ string — read it first. | Status | Meaning | Fix | |---|---|---| -| `400 "Evaluation '…' must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_id` typo | Create the Evaluation first; set `evaluation_context.evaluation_id` to its **name** | +| `400 "Evaluation '…' must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_name` typo | Create the Evaluation first; ensure `evaluation_context.evaluation_name` matches it | | `400 "Evaluation '…' has been deleted…"` | The referenced Evaluation is soft-deleted | Recreate it or target a live one | | `422` on ATIF/chat-completions | Unknown top-level key (both are `extra="forbid"`) | Remove stray keys; check the schema in `../../nemo-intake/references/ingest-formats.md` | | `422` bad `schema_version` (ATIF) | Not one of `ATIF-v1.0` … `ATIF-v1.7` | Use a supported literal | @@ -32,7 +32,7 @@ string — read it first. | Symptom | Cause | Fix | |---|---|---| -| Ingest returned 2xx but `run_count` stays 0 | `evaluation_context` missing, or `evaluation_id` ≠ the Evaluation's name | Attach `evaluation_context`; use the Evaluation **name**. For OTLP, set the span attribute `nemo.evaluation.name` on the root span | +| Ingest returned 2xx but `run_count` stays 0 | Evaluation context is missing or doesn't match the target Evaluation | Set `evaluation_context.evaluation_name`; for OTLP, set `nemo.evaluation.name` on the root span | | No scores on the evaluation | Rewards not under `extra.verifier_result.rewards` (ATIF), or wrong `data_type` (`/evaluator-results`) | ATIF: `extra.verifier_result.rewards = {criterion: value}`. Explicit: `NUMERIC`/`BOOLEAN` need `value`, `CATEGORICAL`/`TEXT` need `string_value` | | No cost on the rollup | The producer never emitted cost | Cost is pass-through — set `cost_usd` (chat-completions / ATIF step `metrics`) or `llm.cost.total` / `gen_ai.usage.cost` (OTLP) | | `503` on `GET .../evaluations/{name}` or `/sessions` | ClickHouse (telemetry store) not running | Start ClickHouse; rollups, sessions, and metric sorts/filters all need it | @@ -40,7 +40,6 @@ string — read it first. ## Identifier cheat-sheet (the #1 source of bugs) -- `evaluation_context.evaluation_id` → the Evaluation's **`name`**. - `experiment_ids` (on create evaluation) → a list with the Experiment's **`id`**. -- OTLP evaluation attribute key → **`nemo.evaluation.name`** (test case → `nemo.test_case.id`). +- OTLP evaluation attribute key → **`nemo.evaluation.name`** (test case → `nemo.test_case.name`). - Parent → **`/experiments`** (`/experiment-groups` is a deprecated hidden alias); evaluations → **`/evaluations`**. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md index 59b0b14457..c58facd9cf 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md @@ -12,18 +12,17 @@ Full request schemas for the three intake ingest endpoints. All are under ```json { "evaluation_context": { - "evaluation_id": "my-eval-baseline", - "test_case_id": "dataset/case-001" + "evaluation_name": "my-eval-baseline", + "test_case_name": "dataset/case-001" } } ``` -- `evaluation_id` is the Evaluation's **name** (not its entity id); `test_case_id` is optional (which - task/test case this run covers). +- `test_case_name` is optional and identifies which task or test case the run covers. - The referenced Evaluation **must already exist** (create it first) or the request is rejected with `400 "…must be created before it can be logged."` - The model is lenient (`extra="ignore"`): retired keys (`evaluation_sha`, `evaluation_run_id`, - `metadata`) are accepted but dropped — only `evaluation_id` and `test_case_id` survive. + `metadata`) are accepted but dropped. - A deprecated `experiment_context` `{experiment_id, test_case_id}` shape is still accepted; `evaluation_context` wins if both are present. Use `evaluation_context`. @@ -46,7 +45,7 @@ automatically; you don't call `/evaluator-results` separately for Harbor runs. { "schema_version": "ATIF-v1.5", "session_id": "d074dfb7-3691-443c-b137-720d75e40afa", - "evaluation_context": { "evaluation_id": "my-eval-baseline", "test_case_id": "my-dataset/case-a" }, + "evaluation_context": { "evaluation_name": "my-eval-baseline", "test_case_name": "my-dataset/case-a" }, "agent": { "name": "my-agent", "version": "1.0.0", "model_name": "provider/model" }, "final_metrics": { "total_prompt_tokens": 51701, "total_completion_tokens": 255, @@ -107,7 +106,7 @@ to this shape and publishing it as an Evaluation. "session_id": "session-001", "provider": "openai", "cost_usd": 0.0001, - "evaluation_context": { "evaluation_id": "my-eval-baseline", "test_case_id": "case-001" } + "evaluation_context": { "evaluation_name": "my-eval-baseline", "test_case_name": "case-001" } } ``` @@ -131,11 +130,11 @@ the root span: | Meaning | Span attribute key | |---|---| -| Evaluation (by name) | **`nemo.evaluation.name`** | -| Test case | **`nemo.test_case.id`** | +| Evaluation | **`nemo.evaluation.name`** | +| Test case | **`nemo.test_case.name`** | -> Set `nemo.evaluation.name` to the Evaluation's **name** (not its id), matching the `evaluation_id` -> field used by the JSON `evaluation_context` on the other endpoints. +These attributes correspond to `evaluation_name` and `test_case_name` in the JSON `evaluation_context` +used by the other ingest endpoints. Cost/token/model attributes are read from standard GenAI / OpenInference keys (first match wins): @@ -161,7 +160,7 @@ export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${NMP_BASE_URL}/apis/intake/v2/worksp export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" ``` -Then set `nemo.evaluation.name` (+ `nemo.test_case.id`) on the root span of each run. +Then set `nemo.evaluation.name` (+ `nemo.test_case.name`) on the root span of each run. --- diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py index 355d453fe7..fc5022ab61 100644 --- a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py @@ -55,7 +55,7 @@ async def _upload_trials( attrs = { "nemo.experiment.id": group, - "nemo.test_case.id": trial.task_id, + "nemo.test_case.name": trial.task_id, "nemo.trial.id": trial.id, "gen_ai.agent.name": AGENT_NAME, "gen_ai.agent.version": AGENT_VERSION, diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py index e153b2f8fe..c9b8cf4c2f 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py @@ -150,7 +150,7 @@ async def _upload_trials( attrs = { "nemo.evaluation.name": evaluation_name, - "nemo.test_case.id": trial.task_id, + "nemo.test_case.name": trial.task_id, "nemo.trial.id": trial.id, "gen_ai.agent.name": agent_name, "gen_ai.agent.version": agent_version, diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_explorer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_explorer.py index aabce9e924..977c8c7c27 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_explorer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_explorer.py @@ -1468,6 +1468,7 @@ def eval_result(self) -> EvalContextData | None: def task_name(self) -> str | None: """Task identifier recorded by trace or evaluator metadata.""" attribute_names = ( + "nemo.test_case.name", "nemo.test_case.id", "test_case.id", "task.name", diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py index 115b7f9388..d897337da4 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py @@ -70,7 +70,7 @@ async def _upload_trace_otlp( path = Path(urlparse(ref.uri).path) attrs: dict[str, str] = { "nemo.evaluation.name": evaluation_name, - "nemo.test_case.id": task_id, + "nemo.test_case.name": task_id, "nemo.trial.id": trial_id, **(extra_attrs or {}), } @@ -727,7 +727,7 @@ async def _persist_trial( trace_id = uri.removeprefix("intake://traces/") trace = await self._retrieve_trace_with_retry(trace_id, workspace=workspace) ctx = getattr(trace, "evaluation_context", None) - if ctx is None or getattr(ctx, "evaluation_id", None) != evaluation_name: + if ctx is None or getattr(ctx, "evaluation_name", None) != evaluation_name: rows: list[dict] = [] async for span in self.client.intake.spans.list( workspace=workspace, @@ -738,7 +738,7 @@ async def _persist_trial( rows.append(span.model_dump(mode="json", exclude_none=True)) attrs = { "nemo.evaluation.name": evaluation_name, - "nemo.test_case.id": trial.task_id, + "nemo.test_case.name": trial.task_id, "nemo.trial.id": trial.id, **agent_attrs, } diff --git a/plugins/nemo-experimentalist/tests/test_otlp.py b/plugins/nemo-experimentalist/tests/test_otlp.py index d4b14cc5c8..d6b22cc58d 100644 --- a/plugins/nemo-experimentalist/tests/test_otlp.py +++ b/plugins/nemo-experimentalist/tests/test_otlp.py @@ -206,10 +206,10 @@ def test_spans_builds_valid_protobuf_with_ids_and_name(): def test_spans_injects_resource_attrs(): - req = _merge(spans_to_protobuf([_span_row()], {"nemo.evaluation.name": "exp-42", "nemo.test_case.id": "task1"})) + req = _merge(spans_to_protobuf([_span_row()], {"nemo.evaluation.name": "exp-42", "nemo.test_case.name": "task1"})) attrs = _resource_attrs(req.resource_spans[0]) assert attrs["nemo.evaluation.name"].string_value == "exp-42" - assert attrs["nemo.test_case.id"].string_value == "task1" + assert attrs["nemo.test_case.name"].string_value == "task1" def test_spans_preserves_raw_attributes_from_json_string(): diff --git a/plugins/nemo-insights/evaluation/otlp_build.py b/plugins/nemo-insights/evaluation/otlp_build.py index 68404edd3b..ba2df020b5 100644 --- a/plugins/nemo-insights/evaluation/otlp_build.py +++ b/plugins/nemo-insights/evaluation/otlp_build.py @@ -96,8 +96,8 @@ def sim_to_spans( withholds them so the eval is unaided). ``evaluation_name`` is stamped on every span as ``nemo.evaluation.name`` (with - the sim's task id as ``nemo.test_case.id``) so a run's spans are queryable - back via the spans filter ``{"evaluation_id": evaluation_name}`` — the per-run + the sim's task name as ``nemo.test_case.name``) so a run's spans are queryable + back via the spans filter ``{"evaluation_name": evaluation_name}`` — the per-run scope that lets many runs share one workspace. ``base_ns`` seeds the per-span monotonic clock; it must be near ingest time @@ -114,7 +114,7 @@ def _common() -> dict[str, Any]: "gen_ai.conversation.id": session_id, "session.id": session_id, "nemo.evaluation.name": evaluation_name, - "nemo.test_case.id": test_case_id, + "nemo.test_case.name": test_case_id, } def _add(*, name: str, kind: str, parent: str | None, attributes: dict[str, Any]) -> dict[str, Any]: diff --git a/plugins/nemo-insights/evaluation/reingest.py b/plugins/nemo-insights/evaluation/reingest.py index 0c12bae187..0fff3da622 100644 --- a/plugins/nemo-insights/evaluation/reingest.py +++ b/plugins/nemo-insights/evaluation/reingest.py @@ -109,8 +109,6 @@ # The platform's entity-name rule (nmp.common NAME_PATTERN) — target workspaces must satisfy it. _WS_OK = re.compile(r"^[a-z](?!.*--)[a-z0-9\-@.+_]{1,62}(? ModuleType: def _doc_value(doc: dict, field: str) -> Any: - """The doc's value for a catalog semantic field (most are flat columns; a few are nested).""" - if field in _EVAL_CONTEXT_FIELDS: - return (doc.get("evaluation_context") or {}).get(field) + """Return a detailed-span document value using canonical response field names.""" + evaluation_context = doc.get("evaluation_context") or {} + if field == "evaluation_name": + return evaluation_context.get("evaluation_name") or evaluation_context.get("evaluation_id") + if field == "test_case_name": + return evaluation_context.get("test_case_name") or evaluation_context.get("test_case_id") if field in _USAGE_DETAIL_FIELDS: return (doc.get("usage_details") or {}).get(_USAGE_DETAIL_FIELDS[field]) if field == "agent_version": diff --git a/plugins/nemo-insights/tests/evaluation/test_otlp_build.py b/plugins/nemo-insights/tests/evaluation/test_otlp_build.py index 432e26a880..10704a86cd 100644 --- a/plugins/nemo-insights/tests/evaluation/test_otlp_build.py +++ b/plugins/nemo-insights/tests/evaluation/test_otlp_build.py @@ -366,7 +366,7 @@ def test_every_span_carries_evaluation_and_test_case_tags(): assert spans # at least the AGENT root + EVALUATOR for s in spans: assert s["attributes"]["nemo.evaluation.name"] == "tau2-airline-20260626-000000-abcd" - assert s["attributes"]["nemo.test_case.id"] == "7" + assert s["attributes"]["nemo.test_case.name"] == "7" def test_tags_identical_across_realistic_and_oracle_twins(): @@ -381,7 +381,7 @@ def test_tags_identical_across_realistic_and_oracle_twins(): oracle = sim_to_spans(sim, include_rewards=True, **common) def tag(s): - return (s["attributes"]["nemo.evaluation.name"], s["attributes"]["nemo.test_case.id"]) + return (s["attributes"]["nemo.evaluation.name"], s["attributes"]["nemo.test_case.name"]) assert {tag(s) for s in realistic} == {("run-1", "7")} assert {tag(s) for s in oracle} == {("run-1", "7")} diff --git a/plugins/nemo-insights/tests/evaluation/test_reingest.py b/plugins/nemo-insights/tests/evaluation/test_reingest.py index 6583ab7956..2ca693de55 100644 --- a/plugins/nemo-insights/tests/evaluation/test_reingest.py +++ b/plugins/nemo-insights/tests/evaluation/test_reingest.py @@ -36,8 +36,8 @@ class _StubCatalog: _Spec(_Field("model"), ("gen_ai.request.model", "gen_ai.response.model", "llm.model_name")), _Spec(_Field("agent_name"), ("gen_ai.agent.name", "llm.agent.name", "agent.name")), _Spec(_Field("agent_version"), ("gen_ai.agent.version", "agent.version")), - _Spec(_Field("evaluation_id"), ("nemo.evaluation.name", "nemo.experiment.id")), - _Spec(_Field("test_case_id"), ("nemo.test_case.id",)), + _Spec(_Field("evaluation_name"), ("nemo.evaluation.name", "nemo.experiment.id")), + _Spec(_Field("test_case_name"), ("nemo.test_case.name", "nemo.test_case.id")), _Spec(_Field("input_tokens"), ("gen_ai.usage.input_tokens", "llm.token_count.prompt")), _Spec(_Field("prompt_cache_write_tokens"), ("llm.token_count.prompt_details.cache_write",)), _Spec(_Field("cost_total_usd"), ("gen_ai.usage.cost", "llm.cost.total")), @@ -61,9 +61,9 @@ class _StubCatalog: "cost_details": {}, "ended_at": "2026-06-26T18:14:41.408179", "evaluation_context": { - "evaluation_id": "smoke-20260626-121437-5559-20260626-121438-a833", + "evaluation_name": "smoke-20260626-121437-5559-20260626-121438-a833", "metadata": {}, - "test_case_id": "1", + "test_case_name": "1", }, "input": "do 1", "name": "smoke-20260626-121437-5559", @@ -92,9 +92,9 @@ class _StubCatalog: "cost_details": {}, "ended_at": "2026-06-26T18:14:41.408179", "evaluation_context": { - "evaluation_id": "smoke-20260626-121437-5559-20260626-121438-a833", + "evaluation_name": "smoke-20260626-121437-5559-20260626-121438-a833", "metadata": {}, - "test_case_id": "1", + "test_case_name": "1", }, "model": "m", "name": "agent-2", @@ -138,7 +138,7 @@ def test_agent_doc_golden(): # catalog inversion: semantic columns re-emitted under their top-precedence source key "gen_ai.agent.name": "smoke-20260626-121437-5559", "nemo.evaluation.name": "smoke-20260626-121437-5559-20260626-121438-a833", - "nemo.test_case.id": "1", + "nemo.test_case.name": "1", } @@ -209,6 +209,16 @@ def test_evaluation_metadata_and_usage_details_invert(): assert attrs["gen_ai.usage.input_tokens"] == 11 +def test_deprecated_evaluation_context_fields_still_invert(): + doc = { + **AGENT_DOC, + "evaluation_context": {"evaluation_id": "legacy-evaluation", "test_case_id": "legacy-case"}, + } + attrs = reingest.doc_to_otlp(doc, CATALOG)["attributes"] + assert attrs["nemo.evaluation.name"] == "legacy-evaluation" + assert attrs["nemo.test_case.name"] == "legacy-case" + + def test_missing_trace_id_is_an_error(): doc = {k: v for k, v in AGENT_DOC.items() if k != "trace_id"} with pytest.raises(ValueError, match="no trace_id"): diff --git a/services/intake/README.md b/services/intake/README.md index 9f96ff536e..1db04ccef7 100644 --- a/services/intake/README.md +++ b/services/intake/README.md @@ -33,12 +33,12 @@ Active v2 workspace endpoints: The Experiments feature captures evaluation runs as leaderboard rows. The flow: 1. Create an **Experiment Group** — `POST /experiment-groups`. -2. Create an **Evaluation** under it — `POST /evaluations`. Its `name` is the `evaluation_id` you - reference when logging. +2. Create an **Evaluation** under it — `POST /evaluations`. Use the Evaluation's `name` when + associating telemetry with it. 3. Log traces + evaluator results to an ingest endpoint. The Evaluation must exist first. Attach evaluation identity per endpoint: **ATIF/Harbor** and **chat-completions** carry it in the JSON body - as `evaluation_context = {evaluation_id, test_case_id}`; **OTLP** carries it as root-span attributes - `nemo.evaluation.name` (the Evaluation name) and `nemo.test_case.id`. + as `evaluation_context = {evaluation_name, test_case_name}`; **OTLP** carries it as root-span attributes + `nemo.evaluation.name` (the Evaluation name) and `nemo.test_case.name`. 4. Read the rollups — `GET /evaluations/{name}` and `.../{name}/sessions` — or view them in Studio (behind the `VITE_FF_EXPERIMENT` flag). diff --git a/services/intake/scripts/spans/seed_span_type_showcase.py b/services/intake/scripts/spans/seed_span_type_showcase.py index 4dd69692d6..82bbc35c22 100644 --- a/services/intake/scripts/spans/seed_span_type_showcase.py +++ b/services/intake/scripts/spans/seed_span_type_showcase.py @@ -572,7 +572,7 @@ def seed_eval_context_trace(seeder: Seeder) -> None: "nemo.evaluation.name": "exp-type-showcase", "nemo.experiment.run_id": "run-01", "nemo.experiment.sha": "a1b2c3d4", - "nemo.test_case.id": "case-0007", + "nemo.test_case.name": "case-0007", "nemo.experiment.metadata": json.dumps({"dataset": "showcase-bench", "split": "test", "seed": 7}), } with seeder.span( diff --git a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py index fd498e7f16..4d951ebf11 100644 --- a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py +++ b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py @@ -10,7 +10,7 @@ from urllib.parse import urlparse from nmp.intake.config import DEFAULT_SPAN_RETENTION_DAYS -from nmp.intake.spans.span_attribute_catalog import SpanAttributeField, spec_for_field +from nmp.intake.spans.span_attribute_catalog import SpanAttributeField, bag_keys, spec_for_field _IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") @@ -253,20 +253,9 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> client.command(f"DROP TABLE IF EXISTS {table}") project_key = spec_for_field(SpanAttributeField.PROJECT).bag_key - test_case_key = spec_for_field(SpanAttributeField.TEST_CASE_NAME).bag_key - - # Resolve evaluation_id by coalescing the canonical bag key with any legacy aliases. Ingest always - # re-keys new spans to the canonical key, so this only matters for the backfill INSERT: spans stored - # under an older key (e.g. the pre-rename ``nemo.experiment.id``) keep their evaluation association - # instead of being dropped when the MV is rebuilt. - evaluation_spec = spec_for_field(SpanAttributeField.EVALUATION_NAME) - evaluation_keys = [ - evaluation_spec.bag_key, - *(k for k in evaluation_spec.source_keys if k != evaluation_spec.bag_key), - ] - evaluation_id_expr = ( - "coalesce(" + ", ".join(f"nullIf(attributes_string['{key}'], '')" for key in evaluation_keys) + ", '')" - ) + # Ingest writes canonical keys, while the backfill must preserve associations on historical rows. + evaluation_id_expr = _coalesced_string_attribute(SpanAttributeField.EVALUATION_NAME) + test_case_id_expr = _coalesced_string_attribute(SpanAttributeField.TEST_CASE_NAME) # Note this is logically a single table. CH requires creating an underlying table and then a view that writes to that table. client.command( @@ -324,7 +313,7 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> output AS root_output, attributes_string['{project_key}'] AS project, {evaluation_id_expr} AS evaluation_id, - attributes_string['{test_case_key}'] AS test_case_id, + {test_case_id_expr} AS test_case_id, start_time AS root_started_at, nullIf(end_time, toDateTime64(0, 6)) AS root_ended_at, if(end_time = toDateTime64(0, 6), NULL, dateDiff('millisecond', start_time, end_time)) AS latency_ms, @@ -373,10 +362,18 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> # the MV now coalesces both keys, so spans already ingested under ``nemo.experiment.id`` keep their # evaluation association while new spans use the canonical key. ("ch_trace_index_0006_nemo_evaluation_name", _create_trace_index_schema), + # The test-case span-attribute bag key was renamed ``nemo.test_case.id`` -> ``nemo.test_case.name``. + # Rebuild the MV so new root spans use the canonical key while the backfill coalesces both keys. + ("ch_trace_index_0007_nemo_test_case_name", _create_trace_index_schema), ] CURRENT_SCHEMA_VERSION = _MIGRATIONS[-1][0] +def _coalesced_string_attribute(field: SpanAttributeField) -> str: + keys = bag_keys(spec_for_field(field)) + return "coalesce(" + ", ".join(f"nullIf(attributes_string['{key}'], '')" for key in keys) + ", '')" + + def _table(settings: ClickHouseMigrationSettings, name: str) -> str: return f"{quote_clickhouse_identifier(settings.database)}.{quote_clickhouse_identifier(name)}" diff --git a/services/intake/src/nmp/intake/spans/span_attribute_bags.py b/services/intake/src/nmp/intake/spans/span_attribute_bags.py index 9eee45f508..6ef30f2c1d 100644 --- a/services/intake/src/nmp/intake/spans/span_attribute_bags.py +++ b/services/intake/src/nmp/intake/spans/span_attribute_bags.py @@ -18,6 +18,7 @@ AttributeBag, AttributeSpec, SpanAttributeField, + bag_keys, from_bag, scaled_decimal_to_int, spec_for_field, @@ -50,7 +51,11 @@ def from_domain_maps( def get_field(self, field: SpanAttributeField | str) -> str | int | float | bool | Decimal | None: spec = spec_for_field(field) bag = self._bag_for_spec(spec) - return from_bag(bag.get(spec.bag_key), spec) + for key in bag_keys(spec): + value = from_bag(bag.get(key), spec) + if value is not None: + return value + return None def put_field(self, field: SpanAttributeField | str, value: Any) -> None: self.put_spec(spec_for_field(field), value) diff --git a/services/intake/src/nmp/intake/spans/span_attribute_catalog.py b/services/intake/src/nmp/intake/spans/span_attribute_catalog.py index 4ed918b5c9..1706c1198a 100644 --- a/services/intake/src/nmp/intake/spans/span_attribute_catalog.py +++ b/services/intake/src/nmp/intake/spans/span_attribute_catalog.py @@ -60,6 +60,7 @@ class AttributeSpec: bag: AttributeBag bag_key: str source_keys: tuple[str, ...] + bag_aliases: tuple[str, ...] = () scale: int | None = None @@ -156,12 +157,14 @@ class AttributeSpec: # their evaluation association. bag_key="nemo.evaluation.name", source_keys=("nemo.evaluation.name", "nemo.experiment.id"), + bag_aliases=("nemo.experiment.id",), ), AttributeSpec( field=SpanAttributeField.TEST_CASE_NAME, bag=AttributeBag.STRING, - bag_key="nemo.test_case.id", - source_keys=("nemo.test_case.id",), + bag_key="nemo.test_case.name", + source_keys=("nemo.test_case.name", "nemo.test_case.id"), + bag_aliases=("nemo.test_case.id",), ), AttributeSpec( field=SpanAttributeField.ERROR_TYPE, @@ -265,7 +268,7 @@ class AttributeSpec: SPECS_BY_FIELD = {spec.field: spec for spec in ATTRIBUTE_SPECS} SPECS_BY_FIELD_VALUE = {spec.field.value: spec for spec in ATTRIBUTE_SPECS} -SPECS_BY_BAG_KEY = {spec.bag_key: spec for spec in ATTRIBUTE_SPECS} +SPECS_BY_BAG_KEY = {key: spec for spec in ATTRIBUTE_SPECS for key in (spec.bag_key, *spec.bag_aliases)} KNOWN_BAG_KEYS = frozenset(SPECS_BY_BAG_KEY) QUERYABLE_FIELDS = frozenset(SPECS_BY_FIELD_VALUE) @@ -315,6 +318,12 @@ def spec_for_field(field: SpanAttributeField | str) -> AttributeSpec: raise ValueError(f"Unsupported span attribute field: {field}") from exc +def bag_keys(spec: AttributeSpec) -> tuple[str, ...]: + """Return the canonical storage key followed by historical storage aliases.""" + + return (spec.bag_key, *spec.bag_aliases) + + def to_bag(typed_value: Any, spec: AttributeSpec) -> str | int | float | bool | None: if typed_value is None: return None @@ -365,18 +374,23 @@ def where_clause( raise ValueError(f"Span attribute filter {field!r} only supports equality comparisons") param_root = param_prefix or field - key_param = f"{param_root}_key" value_param = f"{param_root}_value" parsed_value = _parse_filter_value(value) if spec.bag == AttributeBag.NUMBER else value bag_value = to_bag(parsed_value, spec) if bag_value is None: raise ValueError(f"Span attribute filter {field!r} does not support null values") - sql = ( - f"has(mapKeys({spec.bag.value}), %({key_param})s) " - f"AND {spec.bag.value}[%({key_param})s] {sql_operator} %({value_param})s" - ) - return sql, {key_param: spec.bag_key, value_param: bag_value} + conditions: list[str] = [] + params = {value_param: bag_value} + for index, key in enumerate(bag_keys(spec)): + key_param = f"{param_root}_key" if index == 0 else f"{param_root}_key_alias_{index}" + params[key_param] = key + conditions.append( + f"has(mapKeys({spec.bag.value}), %({key_param})s) " + f"AND {spec.bag.value}[%({key_param})s] {sql_operator} %({value_param})s" + ) + sql = conditions[0] if len(conditions) == 1 else "(" + ") OR (".join(conditions) + ")" + return sql, params def to_semantic_value(value: Any, spec: AttributeSpec) -> str | int | float | bool | Decimal | None: diff --git a/services/intake/tests/integration/spans/test_traces_read.py b/services/intake/tests/integration/spans/test_traces_read.py index b775e5fe25..41c5137f7b 100644 --- a/services/intake/tests/integration/spans/test_traces_read.py +++ b/services/intake/tests/integration/spans/test_traces_read.py @@ -28,7 +28,7 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re # nemo.evaluation.name, so this asserts the dual-read path end to end (a pre-rename # producer still associates its traces to the evaluation). "nemo.experiment.id": "experiment-a", - "nemo.test_case.id": "case-a", + "nemo.test_case.name": "case-a", "deployment.environment.name": "prod", "tag.tags": ["trace-read"], "metadata": {"owner": "trace-test"}, @@ -95,8 +95,12 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re assert Decimal(str(trace["cost_output_usd"])) == Decimal("0.0037") assert trace["span_count"] == 2 assert trace["error_count"] == 0 - assert trace["evaluation_context"]["evaluation_id"] == "experiment-a" - assert trace["evaluation_context"]["test_case_id"] == "case-a" + assert trace["evaluation_context"] == { + "evaluation_name": "experiment-a", + "test_case_name": "case-a", + "evaluation_id": "experiment-a", + "test_case_id": "case-a", + } assert "evaluation_id" not in trace assert "experiment_id" not in trace assert "test_case_id" not in trace @@ -118,8 +122,7 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re summary_trace = summary_response.json()["data"][0] assert summary_trace["id"] == trace["id"] assert summary_trace["status"] == "success" - assert summary_trace["evaluation_context"]["evaluation_id"] == "experiment-a" - assert summary_trace["evaluation_context"]["test_case_id"] == "case-a" + assert summary_trace["evaluation_context"] == trace["evaluation_context"] assert "evaluation_id" not in summary_trace assert "experiment_id" not in summary_trace assert "test_case_id" not in summary_trace diff --git a/services/intake/tests/test_atif_v17.py b/services/intake/tests/test_atif_v17.py index 96e104c908..057ba0e1f3 100644 --- a/services/intake/tests/test_atif_v17.py +++ b/services/intake/tests/test_atif_v17.py @@ -23,11 +23,8 @@ from pydantic import ValidationError EVALUATION_CONTEXT: dict[str, Any] = { - "evaluation_id": "eval-sample-agent-baseline", - "evaluation_sha": "abc132901", - "evaluation_run_id": "evalrun-01JZ8Q7K6V7R3X9N2M4P5A6B7C", - "test_case_id": "sample-test-case", - "metadata": {"attempt": 1}, + "evaluation_name": "eval-sample-agent-baseline", + "test_case_name": "sample-test-case", } @@ -456,7 +453,7 @@ def test_atif_v17_subagent_ref_requires_resolution_key() -> None: assert AtifSubagentTrajectoryRef(trajectory_path="subagents/sub-trajectory.json").trajectory_path is not None -def test_evaluation_context_ignores_retired_fields() -> None: +def test_evaluation_context_normalizes_deprecated_fields_and_ignores_retired_fields() -> None: # Retired keys (evaluation_sha, evaluation_run_id, metadata) are accepted and dropped rather # than rejected, so stale producers keep ingesting without ingest erroring on unknown fields. context = EvaluationContext.model_validate( @@ -468,9 +465,14 @@ def test_evaluation_context_ignores_retired_fields() -> None: "metadata": {"attempt": 1}, } ) - assert context.evaluation_id == "eval-1" - assert context.test_case_id == "case-1" - assert context.model_dump() == {"evaluation_id": "eval-1", "test_case_id": "case-1"} + assert context.evaluation_name == "eval-1" + assert context.test_case_name == "case-1" + assert context.model_dump() == { + "evaluation_name": "eval-1", + "test_case_name": "case-1", + "evaluation_id": "eval-1", + "test_case_id": "case-1", + } def test_atif_ingest_request_rejects_legacy_top_level_project() -> None: @@ -510,19 +512,19 @@ def test_atif_mapping_writes_evaluation_context_only_on_root_span() -> None: root = next(span for span in spans if span.name == "sample-agent") child = next(span for span in spans if span.name == "user-1") - assert root.attributes_string["nemo.evaluation.name"] == EVALUATION_CONTEXT["evaluation_id"] + assert root.attributes_string["nemo.evaluation.name"] == EVALUATION_CONTEXT["evaluation_name"] # sha/run_id/metadata are dropped by the trimmed ingest EvaluationContext, so they never # reach the span from the JSON evaluation_context path. assert "nemo.experiment.sha" not in root.attributes_string assert "nemo.experiment.run_id" not in root.attributes_string assert "evaluation.id" not in root.attributes_string - assert root.attributes_string["nemo.test_case.id"] == EVALUATION_CONTEXT["test_case_id"] + assert root.attributes_string["nemo.test_case.name"] == EVALUATION_CONTEXT["test_case_name"] assert "nemo.experiment.metadata" not in root.attributes_string root_response = Span.from_domain(root) assert root_response.evaluation_context is not None - assert root_response.evaluation_context.evaluation_id == EVALUATION_CONTEXT["evaluation_id"] - assert root_response.evaluation_context.test_case_id == EVALUATION_CONTEXT["test_case_id"] + assert root_response.evaluation_context.evaluation_name == EVALUATION_CONTEXT["evaluation_name"] + assert root_response.evaluation_context.test_case_name == EVALUATION_CONTEXT["test_case_name"] assert root_response.raw_attributes is not None root_raw = json.loads(root_response.raw_attributes) assert "evaluation_context" not in root_raw @@ -533,7 +535,7 @@ def test_atif_mapping_writes_evaluation_context_only_on_root_span() -> None: assert child_response.evaluation_context is None assert "evaluation.id" not in child.attributes_string assert "nemo.evaluation.name" not in child.attributes_string - assert "nemo.test_case.id" not in child.attributes_string + assert "nemo.test_case.name" not in child.attributes_string def test_atif_mapping_uses_root_final_metrics_when_steps_have_no_metrics() -> None: diff --git a/services/intake/tests/test_spans_clickhouse_migrations.py b/services/intake/tests/test_spans_clickhouse_migrations.py index 6c58e4e68a..f63767ed0e 100644 --- a/services/intake/tests/test_spans_clickhouse_migrations.py +++ b/services/intake/tests/test_spans_clickhouse_migrations.py @@ -61,11 +61,12 @@ def test_trace_index_schema_is_root_span_projection(): assert "TO {table}" in ddl assert "INSERT INTO {table}" in source assert "WHERE external_parent_span_id = ''" in ddl - # evaluation_id is resolved via a coalesce expression (canonical key + legacy aliases), not a single - # bag-key lookup, so the backfill keeps spans stored under an older key associated. + # Both identifiers resolve via canonical keys plus historical aliases, so the backfill keeps older + # spans associated. assert "{evaluation_id_expr} AS evaluation_id" in ddl - assert "coalesce(" in ddl - assert "attributes_string['{test_case_key}'] AS test_case_id" in ddl + assert "{test_case_id_expr} AS test_case_id" in ddl + assert "_coalesced_string_attribute(SpanAttributeField.EVALUATION_NAME)" in ddl + assert "_coalesced_string_attribute(SpanAttributeField.TEST_CASE_NAME)" in ddl assert "root_status LowCardinality(String)" in ddl assert "root_input String" in ddl assert "PRIMARY KEY (workspace, root_started_at)" in ddl @@ -79,4 +80,6 @@ def test_trace_index_mv_keys_match_attribute_catalog(): assert evaluation_spec.bag_key == "nemo.evaluation.name" # The legacy key is still accepted on ingest so pre-rename producers keep associating. assert "nemo.experiment.id" in evaluation_spec.source_keys - assert spec_for_field(SpanAttributeField.TEST_CASE_NAME).bag_key == "nemo.test_case.id" + test_case_spec = spec_for_field(SpanAttributeField.TEST_CASE_NAME) + assert test_case_spec.bag_key == "nemo.test_case.name" + assert test_case_spec.bag_aliases == ("nemo.test_case.id",) diff --git a/services/intake/tests/test_spans_span_attribute_catalog.py b/services/intake/tests/test_spans_span_attribute_catalog.py index 62bbfa8c3f..24f3303476 100644 --- a/services/intake/tests/test_spans_span_attribute_catalog.py +++ b/services/intake/tests/test_spans_span_attribute_catalog.py @@ -60,6 +60,34 @@ def test_span_attribute_catalog_extracts_source_aliases_and_consumed_keys(): } +def test_span_semantic_attributes_use_name_based_evaluation_fields(): + semantic, consumed_keys = SpanSemanticAttributes.from_source_attributes( + { + "nemo.evaluation.name": "evaluation-a", + "nemo.test_case.name": "case-a", + } + ) + + assert semantic.evaluation_name == "evaluation-a" + assert semantic.test_case_name == "case-a" + assert "evaluation_id" not in SpanSemanticAttributes.model_fields + assert "test_case_id" not in SpanSemanticAttributes.model_fields + assert consumed_keys == {"nemo.evaluation.name", "nemo.test_case.name"} + + bags = semantic.to_bags() + assert bags.string["nemo.test_case.name"] == "case-a" + assert "nemo.test_case.id" not in bags.string + + +def test_span_semantic_attributes_read_deprecated_test_case_bag_key(): + semantic, consumed_keys = SpanSemanticAttributes.from_source_attributes({"nemo.test_case.id": "case-a"}) + restored = SpanSemanticAttributes.from_bags(SpanAttributeBags(string={"nemo.test_case.id": "case-a"})) + + assert semantic.test_case_name == "case-a" + assert restored.test_case_name == "case-a" + assert consumed_keys == {"nemo.test_case.id"} + + def test_span_semantic_attributes_normalizes_source_layers_with_span_precedence(): normalized = SpanSemanticAttributes.from_source_attribute_layers( resource_attributes={ @@ -171,6 +199,15 @@ def test_span_attribute_catalog_rejects_ordering_on_string_fields(): where_clause("model", ">", "gpt-4") +def test_test_case_filter_queries_canonical_and_deprecated_bag_keys(): + sql, params = where_clause("test_case_name", "=", "case-a") + + assert " OR " in sql + assert params["test_case_name_key"] == "nemo.test_case.name" + assert params["test_case_name_key_alias_1"] == "nemo.test_case.id" + assert params["test_case_name_value"] == "case-a" + + def test_span_attribute_catalog_rejects_non_numeric_numeric_filters(): with pytest.raises(ValueError): where_clause("total_tokens", ">", "not-a-number") From 4195153aa8457fa62207be65d0826ca538d339bf Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:06:38 -0600 Subject: [PATCH 04/10] chore(sdk): regenerate intake API clients Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- docs/cli/reference.mdx | 10 +++- openapi/ga/individual/platform.openapi.yaml | 56 ++++++++++++++----- openapi/ga/openapi.yaml | 56 ++++++++++++++----- openapi/openapi.yaml | 56 ++++++++++++++----- .../cli/commands/api/intake/ingest/atif.py | 2 +- .../api/intake/ingest/chat_completions.py | 2 +- .../cli/commands/api/intake/spans/__init__.py | 10 +++- .../cli/commands/api/intake/spans/groups.py | 8 +++ .../nemo-platform/.nmpcontext/openapi.yaml | 56 ++++++++++++++----- .../cli/commands/api/intake/ingest/atif.py | 2 +- .../api/intake/ingest/chat_completions.py | 2 +- .../cli/commands/api/intake/spans/__init__.py | 10 +++- .../cli/commands/api/intake/spans/groups.py | 8 +++ .../resources/intake/ingest/atif.py | 14 ++--- .../intake/ingest/chat_completions.py | 14 ++--- .../resources/intake/spans/spans.py | 8 +-- .../skills/nemo-experiments-upload/SKILL.md | 20 +++---- .../references/harbor-quickstart.md | 14 ++--- .../references/troubleshooting.md | 7 +-- .../nemo-intake/references/ingest-formats.md | 23 ++++---- .../types/intake/evaluation_context.py | 14 +++-- .../types/intake/evaluation_context_param.py | 14 +++-- .../types/intake/ingest/atif_create_params.py | 8 +-- .../intake/ingest/atif_trajectory_param.py | 8 +-- .../ingest/chat_completion_create_params.py | 8 +-- .../types/intake/span_evaluation_context.py | 8 +++ .../types/intake/span_filter_param.py | 10 +++- .../types/intake/span_list_params.py | 4 +- .../src/nemo_platform/types/intake/trace.py | 8 +-- .../api_resources/intake/ingest/test_atif.py | 8 +++ .../intake/ingest/test_chat_completions.py | 4 ++ .../api_resources/intake/spans/test_groups.py | 4 ++ .../tests/api_resources/intake/test_spans.py | 4 ++ 33 files changed, 320 insertions(+), 160 deletions(-) diff --git a/docs/cli/reference.mdx b/docs/cli/reference.mdx index 284f45f4ee..c830d17b62 100644 --- a/docs/cli/reference.mdx +++ b/docs/cli/reference.mdx @@ -6599,7 +6599,7 @@ nemo intake ingest atif create [OPTIONS] * `--agent`: JSON string * `--schema-version `: (required) [possible values: ATIF-v1.0, ATIF-v1.1, ATIF-v1.2, ATIF-v1.3, ATIF-v1.4, ATIF-v1.5, ATIF-v1.6, ATIF-v1.7] * `--continued-trajectory-ref` -* `--evaluation-context`: Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string) +* `--evaluation-context`: Identifies the Evaluation and optional test case associated with ingested telemetry. (JSON string) * `--extra`: JSON string * `--final-metrics`: JSON string * `--notes` @@ -6669,7 +6669,7 @@ nemo intake ingest chat-completions create [OPTIONS] * `--cost-input-usd `: Estimated input-token cost of this model call in USD. * `--cost-output-usd `: Estimated output-token cost of this model call in USD. * `--cost-usd `: Total estimated cost of this model call in USD. This matches ATIF step metrics; Intake stores it as semantic cost_total_usd on spans. -* `--evaluation-context`: Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string) +* `--evaluation-context`: Identifies the Evaluation and optional test case associated with ingested telemetry. (JSON string) * `--provider` * `--session-id`: Groups related chat-completions calls without forcing them into the same trace. * `--trace-id`: Opt into joining an existing trace built via OTel or ATIF. This is not a grouping mechanism for chat-completions calls; use session_id to group related calls. @@ -6925,10 +6925,11 @@ nemo intake spans list [OPTIONS] JSON-only fields: started_at: \{gte: str, lte: str} -Filter spans by session_id, trace_id, parent_span_id, project, evaluation_id, test_case_id, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte. +Filter spans by session_id, trace_id, parent_span_id, project, evaluation_name, test_case_name, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte. * `--filter.agent-id` * `--filter.agent-name` * `--filter.evaluation-id` +* `--filter.evaluation-name` * `--filter.kind` * `--filter.model` * `--filter.parent-span-id` @@ -6938,6 +6939,7 @@ Filter spans by session_id, trace_id, parent_span_id, project, evaluation_id, te * `--filter.source` * `--filter.status` * `--filter.test-case-id` +* `--filter.test-case-name` * `--filter.tool-name` * `--filter.trace-id` @@ -7072,6 +7074,7 @@ Filter spans by the same fields as the span list endpoint, then group matching s * `--filter.agent-id` * `--filter.agent-name` * `--filter.evaluation-id` +* `--filter.evaluation-name` * `--filter.kind` * `--filter.model` * `--filter.parent-span-id` @@ -7081,6 +7084,7 @@ Filter spans by the same fields as the span list endpoint, then group matching s * `--filter.source` * `--filter.status` * `--filter.test-case-id` +* `--filter.test-case-name` * `--filter.tool-name` * `--filter.trace-id` diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 8185971b76..938c21513d 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -4688,9 +4688,9 @@ paths: schema: $ref: '#/components/schemas/SpanFilter' description: Filter spans by session_id, trace_id, parent_span_id, project, - evaluation_id, test_case_id, source, kind, status, model, tool_name, provider, - agent_id, agent_name, and started_at. Every field takes one exact value, - except started_at, which takes gte and lte. + evaluation_name, test_case_name, source, kind, status, model, tool_name, + provider, agent_id, agent_name, and started_at. Every field takes one exact + value, except started_at, which takes gte and lte. responses: '200': description: Successful Response @@ -11055,24 +11055,28 @@ components: description: Schema for updating an entity. EvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id - description: Name of an existing Evaluation. + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id - description: Optional producer-supplied test case id. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string type: object title: EvaluationContext - description: 'Evaluation context accepted by ingest endpoints (the canonical - shape). - - - ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, - - metadata) keeps ingesting without error rather than being rejected.' + description: Identifies the Evaluation and optional test case associated with + ingested telemetry. EvaluationFilter: additionalProperties: false description: Filter for listing Evaluations. @@ -18365,11 +18369,23 @@ components: title: Span SpanEvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string additionalProperties: false type: object @@ -18388,12 +18404,22 @@ components: description: Filter by project name. title: Project type: string + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by dataset test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string source: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 8185971b76..938c21513d 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -4688,9 +4688,9 @@ paths: schema: $ref: '#/components/schemas/SpanFilter' description: Filter spans by session_id, trace_id, parent_span_id, project, - evaluation_id, test_case_id, source, kind, status, model, tool_name, provider, - agent_id, agent_name, and started_at. Every field takes one exact value, - except started_at, which takes gte and lte. + evaluation_name, test_case_name, source, kind, status, model, tool_name, + provider, agent_id, agent_name, and started_at. Every field takes one exact + value, except started_at, which takes gte and lte. responses: '200': description: Successful Response @@ -11055,24 +11055,28 @@ components: description: Schema for updating an entity. EvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id - description: Name of an existing Evaluation. + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id - description: Optional producer-supplied test case id. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string type: object title: EvaluationContext - description: 'Evaluation context accepted by ingest endpoints (the canonical - shape). - - - ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, - - metadata) keeps ingesting without error rather than being rejected.' + description: Identifies the Evaluation and optional test case associated with + ingested telemetry. EvaluationFilter: additionalProperties: false description: Filter for listing Evaluations. @@ -18365,11 +18369,23 @@ components: title: Span SpanEvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string additionalProperties: false type: object @@ -18388,12 +18404,22 @@ components: description: Filter by project name. title: Project type: string + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by dataset test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string source: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 8185971b76..938c21513d 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4688,9 +4688,9 @@ paths: schema: $ref: '#/components/schemas/SpanFilter' description: Filter spans by session_id, trace_id, parent_span_id, project, - evaluation_id, test_case_id, source, kind, status, model, tool_name, provider, - agent_id, agent_name, and started_at. Every field takes one exact value, - except started_at, which takes gte and lte. + evaluation_name, test_case_name, source, kind, status, model, tool_name, + provider, agent_id, agent_name, and started_at. Every field takes one exact + value, except started_at, which takes gte and lte. responses: '200': description: Successful Response @@ -11055,24 +11055,28 @@ components: description: Schema for updating an entity. EvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id - description: Name of an existing Evaluation. + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id - description: Optional producer-supplied test case id. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string type: object title: EvaluationContext - description: 'Evaluation context accepted by ingest endpoints (the canonical - shape). - - - ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, - - metadata) keeps ingesting without error rather than being rejected.' + description: Identifies the Evaluation and optional test case associated with + ingested telemetry. EvaluationFilter: additionalProperties: false description: Filter for listing Evaluations. @@ -18365,11 +18369,23 @@ components: title: Span SpanEvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string additionalProperties: false type: object @@ -18388,12 +18404,22 @@ components: description: Filter by project name. title: Project type: string + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by dataset test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string source: diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py index 6eb1106246..0ccd087e50 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py @@ -36,7 +36,7 @@ def create_atif( str | None, typer.Option( "--evaluation-context", - help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', + help="Identifies the Evaluation and optional test case associated with ingested telemetry. (JSON string)", ), ] = None, extra: Annotated[str | None, typer.Option("--extra", help="JSON string")] = None, diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py index 95129af9da..e1c8e831d2 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py @@ -54,7 +54,7 @@ def create_chat_completions( str | None, typer.Option( "--evaluation-context", - help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', + help="Identifies the Evaluation and optional test case associated with ingested telemetry. (JSON string)", ), ] = None, provider: Annotated[str | None, typer.Option("--provider")] = None, diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/__init__.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/__init__.py index 922122a757..81f11273d7 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/__init__.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/__init__.py @@ -51,7 +51,7 @@ def list_spans( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter spans by session_id, trace_id, parent_span_id, project, evaluation_id, test_case_id, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter spans by session_id, trace_id, parent_span_id, project, evaluation_name, test_case_name, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte.", rich_help_panel="Filter Options", ), ] = None, @@ -62,6 +62,9 @@ def list_spans( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, + filter_evaluation_name: Annotated[ + str | None, typer.Option("--filter.evaluation-name", rich_help_panel="Filter Options") + ] = None, filter_kind: Annotated[str | None, typer.Option("--filter.kind", rich_help_panel="Filter Options")] = None, filter_model: Annotated[str | None, typer.Option("--filter.model", rich_help_panel="Filter Options")] = None, filter_parent_span_id: Annotated[ @@ -77,6 +80,9 @@ def list_spans( filter_test_case_id: Annotated[ str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") ] = None, + filter_test_case_name: Annotated[ + str | None, typer.Option("--filter.test-case-name", rich_help_panel="Filter Options") + ] = None, filter_tool_name: Annotated[ str | None, typer.Option("--filter.tool-name", rich_help_panel="Filter Options") ] = None, @@ -119,6 +125,7 @@ def list_spans( agent_id=filter_agent_id, agent_name=filter_agent_name, evaluation_id=filter_evaluation_id, + evaluation_name=filter_evaluation_name, kind=filter_kind, model=filter_model, parent_span_id=filter_parent_span_id, @@ -128,6 +135,7 @@ def list_spans( source=filter_source, status=filter_status, test_case_id=filter_test_case_id, + test_case_name=filter_test_case_name, tool_name=filter_tool_name, trace_id=filter_trace_id, ), diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/groups.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/groups.py index c6e9f13a51..062c3c695c 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/groups.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/spans/groups.py @@ -53,6 +53,9 @@ def list_groups( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, + filter_evaluation_name: Annotated[ + str | None, typer.Option("--filter.evaluation-name", rich_help_panel="Filter Options") + ] = None, filter_kind: Annotated[str | None, typer.Option("--filter.kind", rich_help_panel="Filter Options")] = None, filter_model: Annotated[str | None, typer.Option("--filter.model", rich_help_panel="Filter Options")] = None, filter_parent_span_id: Annotated[ @@ -68,6 +71,9 @@ def list_groups( filter_test_case_id: Annotated[ str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") ] = None, + filter_test_case_name: Annotated[ + str | None, typer.Option("--filter.test-case-name", rich_help_panel="Filter Options") + ] = None, filter_tool_name: Annotated[ str | None, typer.Option("--filter.tool-name", rich_help_panel="Filter Options") ] = None, @@ -110,6 +116,7 @@ def list_groups( agent_id=filter_agent_id, agent_name=filter_agent_name, evaluation_id=filter_evaluation_id, + evaluation_name=filter_evaluation_name, kind=filter_kind, model=filter_model, parent_span_id=filter_parent_span_id, @@ -119,6 +126,7 @@ def list_groups( source=filter_source, status=filter_status, test_case_id=filter_test_case_id, + test_case_name=filter_test_case_name, tool_name=filter_tool_name, trace_id=filter_trace_id, ), diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index e53d1e5cca..9370fe681d 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -4691,9 +4691,9 @@ paths: schema: $ref: '#/components/schemas/SpanFilter' description: Filter spans by session_id, trace_id, parent_span_id, project, - evaluation_id, test_case_id, source, kind, status, model, tool_name, provider, - agent_id, agent_name, and started_at. Every field takes one exact value, - except started_at, which takes gte and lte. + evaluation_name, test_case_name, source, kind, status, model, tool_name, + provider, agent_id, agent_name, and started_at. Every field takes one exact + value, except started_at, which takes gte and lte. responses: '200': description: Successful Response @@ -11058,24 +11058,28 @@ components: description: Schema for updating an entity. EvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id - description: Name of an existing Evaluation. + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id - description: Optional producer-supplied test case id. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string type: object title: EvaluationContext - description: 'Evaluation context accepted by ingest endpoints (the canonical - shape). - - - ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, - - metadata) keeps ingesting without error rather than being rejected.' + description: Identifies the Evaluation and optional test case associated with + ingested telemetry. EvaluationFilter: additionalProperties: false description: Filter for listing Evaluations. @@ -18368,11 +18372,23 @@ components: title: Span SpanEvaluationContext: properties: + evaluation_name: + title: Evaluation Name + description: Name of an existing Evaluation. + type: string + test_case_name: + title: Test Case Name + description: Optional producer-supplied test case name. + type: string evaluation_id: title: Evaluation Id + description: Deprecated alias for evaluation_name. Use evaluation_name instead. + deprecated: true type: string test_case_id: title: Test Case Id + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string additionalProperties: false type: object @@ -18391,12 +18407,22 @@ components: description: Filter by project name. title: Project type: string + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by dataset test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string source: diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py index c14a117efb..725d26cca4 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py @@ -36,7 +36,7 @@ def create_atif( str | None, typer.Option( "--evaluation-context", - help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', + help="Identifies the Evaluation and optional test case associated with ingested telemetry. (JSON string)", ), ] = None, extra: Annotated[str | None, typer.Option("--extra", help="JSON string")] = None, diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py index 1dc5f7aaae..2d692b00d9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py @@ -54,7 +54,7 @@ def create_chat_completions( str | None, typer.Option( "--evaluation-context", - help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', + help="Identifies the Evaluation and optional test case associated with ingested telemetry. (JSON string)", ), ] = None, provider: Annotated[str | None, typer.Option("--provider")] = None, diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/__init__.py index 8f3ff24de0..d3726cc423 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/__init__.py @@ -51,7 +51,7 @@ def list_spans( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter spans by session_id, trace_id, parent_span_id, project, evaluation_id, test_case_id, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter spans by session_id, trace_id, parent_span_id, project, evaluation_name, test_case_name, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte.", rich_help_panel="Filter Options", ), ] = None, @@ -62,6 +62,9 @@ def list_spans( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, + filter_evaluation_name: Annotated[ + str | None, typer.Option("--filter.evaluation-name", rich_help_panel="Filter Options") + ] = None, filter_kind: Annotated[str | None, typer.Option("--filter.kind", rich_help_panel="Filter Options")] = None, filter_model: Annotated[str | None, typer.Option("--filter.model", rich_help_panel="Filter Options")] = None, filter_parent_span_id: Annotated[ @@ -77,6 +80,9 @@ def list_spans( filter_test_case_id: Annotated[ str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") ] = None, + filter_test_case_name: Annotated[ + str | None, typer.Option("--filter.test-case-name", rich_help_panel="Filter Options") + ] = None, filter_tool_name: Annotated[ str | None, typer.Option("--filter.tool-name", rich_help_panel="Filter Options") ] = None, @@ -119,6 +125,7 @@ def list_spans( agent_id=filter_agent_id, agent_name=filter_agent_name, evaluation_id=filter_evaluation_id, + evaluation_name=filter_evaluation_name, kind=filter_kind, model=filter_model, parent_span_id=filter_parent_span_id, @@ -128,6 +135,7 @@ def list_spans( source=filter_source, status=filter_status, test_case_id=filter_test_case_id, + test_case_name=filter_test_case_name, tool_name=filter_tool_name, trace_id=filter_trace_id, ), diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/groups.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/groups.py index c09589ebf4..111c48fb42 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/groups.py @@ -53,6 +53,9 @@ def list_groups( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, + filter_evaluation_name: Annotated[ + str | None, typer.Option("--filter.evaluation-name", rich_help_panel="Filter Options") + ] = None, filter_kind: Annotated[str | None, typer.Option("--filter.kind", rich_help_panel="Filter Options")] = None, filter_model: Annotated[str | None, typer.Option("--filter.model", rich_help_panel="Filter Options")] = None, filter_parent_span_id: Annotated[ @@ -68,6 +71,9 @@ def list_groups( filter_test_case_id: Annotated[ str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") ] = None, + filter_test_case_name: Annotated[ + str | None, typer.Option("--filter.test-case-name", rich_help_panel="Filter Options") + ] = None, filter_tool_name: Annotated[ str | None, typer.Option("--filter.tool-name", rich_help_panel="Filter Options") ] = None, @@ -110,6 +116,7 @@ def list_groups( agent_id=filter_agent_id, agent_name=filter_agent_name, evaluation_id=filter_evaluation_id, + evaluation_name=filter_evaluation_name, kind=filter_kind, model=filter_model, parent_span_id=filter_parent_span_id, @@ -119,6 +126,7 @@ def list_groups( source=filter_source, status=filter_status, test_case_id=filter_test_case_id, + test_case_name=filter_test_case_name, tool_name=filter_tool_name, trace_id=filter_trace_id, ), diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py index 2ac8104188..2aa1455203 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py @@ -91,11 +91,8 @@ def create( Ingest Atif Args: - evaluation_context: Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + evaluation_context: Identifies the Evaluation and optional test case associated with ingested + telemetry. extra_headers: Send extra headers @@ -183,11 +180,8 @@ async def create( Ingest Atif Args: - evaluation_context: Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + evaluation_context: Identifies the Evaluation and optional test case associated with ingested + telemetry. extra_headers: Send extra headers diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py index 4b455117f4..1963e580f1 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py @@ -101,11 +101,8 @@ def create( cost_usd: Total estimated cost of this model call in USD. This matches ATIF step metrics; Intake stores it as semantic cost_total_usd on spans. - evaluation_context: Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + evaluation_context: Identifies the Evaluation and optional test case associated with ingested + telemetry. session_id: Groups related chat-completions calls without forcing them into the same trace. @@ -207,11 +204,8 @@ async def create( cost_usd: Total estimated cost of this model call in USD. This matches ATIF step metrics; Intake stores it as semantic cost_total_usd on spans. - evaluation_context: Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + evaluation_context: Identifies the Evaluation and optional test case associated with ingested + telemetry. session_id: Groups related chat-completions calls without forcing them into the same trace. diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/spans/spans.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/spans/spans.py index 2ea853c7ed..3f3911b89f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/spans/spans.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/spans/spans.py @@ -145,8 +145,8 @@ def list( List Spans Args: - filter: Filter spans by session_id, trace_id, parent_span_id, project, evaluation_id, - test_case_id, source, kind, status, model, tool_name, provider, agent_id, + filter: Filter spans by session_id, trace_id, parent_span_id, project, evaluation_name, + test_case_name, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte. @@ -281,8 +281,8 @@ def list( List Spans Args: - filter: Filter spans by session_id, trace_id, parent_span_id, project, evaluation_id, - test_case_id, source, kind, status, model, tool_name, provider, agent_id, + filter: Filter spans by session_id, trace_id, parent_span_id, project, evaluation_name, + test_case_name, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte. diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md index bccd0fa072..de05ee545d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md @@ -111,7 +111,7 @@ instead of masking it. The `id` is then read with a GET, so this works on both f ### 2. Create an Evaluation -One Evaluation = one agent/config run against a dataset (one leaderboard row). Its **`name`** is what you reference later in `evaluation_context.evaluation_id`. +Create one Evaluation for each agent/config run against a dataset (one leaderboard row). ```bash curl -sf -X POST \ @@ -126,15 +126,15 @@ curl -sf -X POST \ - `experiment_ids` is a list holding the Experiment's **`id`** (from step 1). - `metadata` values must be **strings** (`dict[str, str]`). -- **You must create the Evaluation before you can log to it** — ingesting with an unknown `evaluation_id` returns `400 "…must be created before it can be logged."` +- **You must create the Evaluation before you can log to it** — ingesting with an unknown `evaluation_name` returns `400 "…must be created before it can be logged."` ### 3. Log traces + evaluator results Pick the ingest endpoint that matches your producer. **Read `../nemo-intake/references/ingest-formats.md` for the full schema and a copy-pasteable example for each.** How you attach evaluation identity depends on the endpoint: -- **ATIF and chat-completions** (JSON body) — add an `evaluation_context = {evaluation_id: "", test_case_id: ""}` object to the payload. -- **OTLP** — there is no body field; set `nemo.evaluation.name` (the Evaluation **name**) and - `nemo.test_case.id` (the task ID) as **attributes on the root span**. Spans missing these still +- **ATIF and chat-completions** (JSON body) — add an `evaluation_context = {evaluation_name: "", test_case_name: ""}` object to the payload. +- **OTLP** — there is no body field; set `nemo.evaluation.name` and `nemo.test_case.name` as + **attributes on the root span**. Spans missing these still ingest but won't associate to an Evaluation. | Producer | Endpoint | Read | @@ -179,23 +179,23 @@ You succeeded when `GET .../evaluations/my-eval-baseline` shows: - `run_count` ≥ 1 (each ingested session counts as one run), and - non-empty `evaluator_names` / `aggregate_scores` if you logged rewards, and/or `cost_usd` if your spans carried cost. -If `run_count` is 0 after ingesting, the traces didn't associate — almost always a wrong evaluation identity: `evaluation_context.evaluation_id` for ATIF/chat-completions, or the `nemo.evaluation.name` root-span attribute for OTLP (see Gotchas). +If `run_count` is 0 after ingesting, the traces didn't associate — almost always a wrong evaluation identity: `evaluation_context.evaluation_name` for ATIF/chat-completions, or the `nemo.evaluation.name` root-span attribute for OTLP (see Gotchas). ## If verification fails | Symptom | Cause | Recovery | |---|---|---| -| `400 "…must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_id` doesn't match | Create the Evaluation (step 2); ensure `evaluation_context.evaluation_id` equals its **name** | +| `400 "…must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_name` doesn't match | Create the Evaluation (step 2); ensure `evaluation_context.evaluation_name` identifies it | | `422 Unprocessable` on ingest | Unknown/typo'd top-level key (ATIF/chat-completions are `extra="forbid"`) or bad `schema_version` | Check the exact schema in `../nemo-intake/references/ingest-formats.md`; remove stray keys | -| Ingest 2xx but `run_count` stays 0 | Evaluation identity missing/wrong — `evaluation_context.evaluation_id` (ATIF/chat-completions) or the `nemo.evaluation.name` root-span attribute (OTLP) ≠ the Evaluation's name | Attach the identity for your endpoint; use the Evaluation **name**, not its id | +| Ingest 2xx but `run_count` stays 0 | Evaluation context is missing or doesn't match the target Evaluation | Attach the correct `evaluation_context.evaluation_name` (ATIF/chat-completions) or `nemo.evaluation.name` root-span attribute (OTLP) | | `503` on GET evaluation / sessions | ClickHouse (telemetry store) not running | Start ClickHouse; rollups and sessions require it | | Scores don't show up | Rewards not under `extra.verifier_result.rewards`, or wrong `data_type` on `/evaluator-results` | See `references/troubleshooting.md` | ## Gotchas - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`. -- **`evaluation_id` is the Evaluation's `name`, not its entity id.** But **`experiment_ids` holds the Experiment's `id`.** Different identifiers; easy to swap. -- **OTLP uses the attribute key `nemo.evaluation.name`** (and `nemo.test_case.id`) — set it to the Evaluation's **name** on your root span, matching the `evaluation_id` field the JSON `evaluation_context` carries on the other endpoints. +- **Evaluation and Experiment references use different fields.** `evaluation_name` associates telemetry with an Evaluation, while `experiment_ids` assigns that Evaluation to its parent Experiments. +- **OTLP evaluation context:** set `nemo.evaluation.name` and, when applicable, `nemo.test_case.name` as root-span attributes. These correspond to `evaluation_name` and `test_case_name` in the JSON `evaluation_context` used by other ingest endpoints. - **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`. - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`. - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`). diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md index d7d3b082c6..cbbb820daa 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md @@ -12,8 +12,8 @@ each task runs several times (trials). You upload **one ATIF payload per trial** | Harbor concept | NeMo entity | How | |---|---|---| | A benchmark / sweep | **Experiment** | `POST /experiments` once | -| One agent+config on that benchmark | **Evaluation** | `POST /evaluations` once (its `name` is your `evaluation_id`) | -| A task / test case | `test_case_id` | field inside `evaluation_context` | +| One agent+config on that benchmark | **Evaluation** | `POST /evaluations` once (its `name` is your `evaluation_name`) | +| A task / test case | `test_case_name` | field inside `evaluation_context` | | One trial (attempt) of a task | one ingested **session** | one `POST /ingest/atif` | ## Mapping: Harbor trial files → ATIF payload @@ -25,8 +25,8 @@ Per trial, Harbor writes result files (typically `result.json` and `agent/trajec | `agent.name` / `agent.version` / `agent.model_name` | the agent under test | | `steps[]` | the trajectory steps (`agent`/`user`/`system`, with `metrics.{prompt_tokens, completion_tokens, cost_usd}`) | | `final_metrics.{total_prompt_tokens, total_completion_tokens, total_cost_usd, total_steps}` | trajectory `final_metrics` or the trial's `agent_result` (`n_input_tokens`, `n_output_tokens`, `cost_usd`) | -| `evaluation_context.evaluation_id` | your Evaluation **name** (constant across the whole run) | -| `evaluation_context.test_case_id` | the task id (constant across that task's trials) | +| `evaluation_context.evaluation_name` | the target Evaluation (constant across the whole run) | +| `evaluation_context.test_case_name` | the task name (constant across that task's trials) | | `extra.verifier_result.rewards` | the verifier's per-criterion scores → one evaluator score row each | **Cost/tokens are pass-through.** NeMo does not recompute cost — it sums the per-call `cost_usd` / @@ -39,7 +39,7 @@ token values Harbor recorded. If Harbor didn't record a cost for a run, that run { "schema_version": "ATIF-v1.5", "session_id": "", - "evaluation_context": { "evaluation_id": "my-eval-baseline", "test_case_id": "tau-bench/airline-042" }, + "evaluation_context": { "evaluation_name": "my-eval-baseline", "test_case_name": "tau-bench/airline-042" }, "agent": { "name": "my-agent", "version": "1.0.0", "model_name": "provider/model" }, "final_metrics": { "total_prompt_tokens": 51701, "total_completion_tokens": 255, "total_cost_usd": 0.264, "total_steps": 3 }, "extra": { @@ -55,8 +55,8 @@ token values Harbor recorded. If Harbor didn't record a cost for a run, that run ## Consistency rules (so rollups aggregate correctly) -- **`evaluation_id` is identical** for every trial of the run (it's the Evaluation name). -- **`test_case_id` is identical** across all trials of the same task, and **differs** between tasks — +- **`evaluation_name` is identical** for every trial of the run. +- **`test_case_name` is identical** across all trials of the same task, and **differs** between tasks — this is what lets the platform group a task's k attempts. - Give each trial a distinct `session_id` (one session = one run in the rollup's `run_count`). diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md index 9816bd521a..bf139d6269 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md @@ -19,7 +19,7 @@ string — read it first. | Status | Meaning | Fix | |---|---|---| -| `400 "Evaluation '…' must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_id` typo | Create the Evaluation first; set `evaluation_context.evaluation_id` to its **name** | +| `400 "Evaluation '…' must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_name` typo | Create the Evaluation first; ensure `evaluation_context.evaluation_name` matches it | | `400 "Evaluation '…' has been deleted…"` | The referenced Evaluation is soft-deleted | Recreate it or target a live one | | `422` on ATIF/chat-completions | Unknown top-level key (both are `extra="forbid"`) | Remove stray keys; check the schema in `../../nemo-intake/references/ingest-formats.md` | | `422` bad `schema_version` (ATIF) | Not one of `ATIF-v1.0` … `ATIF-v1.7` | Use a supported literal | @@ -32,7 +32,7 @@ string — read it first. | Symptom | Cause | Fix | |---|---|---| -| Ingest returned 2xx but `run_count` stays 0 | `evaluation_context` missing, or `evaluation_id` ≠ the Evaluation's name | Attach `evaluation_context`; use the Evaluation **name**. For OTLP, set the span attribute `nemo.evaluation.name` on the root span | +| Ingest returned 2xx but `run_count` stays 0 | Evaluation context is missing or doesn't match the target Evaluation | Set `evaluation_context.evaluation_name`; for OTLP, set `nemo.evaluation.name` on the root span | | No scores on the evaluation | Rewards not under `extra.verifier_result.rewards` (ATIF), or wrong `data_type` (`/evaluator-results`) | ATIF: `extra.verifier_result.rewards = {criterion: value}`. Explicit: `NUMERIC`/`BOOLEAN` need `value`, `CATEGORICAL`/`TEXT` need `string_value` | | No cost on the rollup | The producer never emitted cost | Cost is pass-through — set `cost_usd` (chat-completions / ATIF step `metrics`) or `llm.cost.total` / `gen_ai.usage.cost` (OTLP) | | `503` on `GET .../evaluations/{name}` or `/sessions` | ClickHouse (telemetry store) not running | Start ClickHouse; rollups, sessions, and metric sorts/filters all need it | @@ -40,7 +40,6 @@ string — read it first. ## Identifier cheat-sheet (the #1 source of bugs) -- `evaluation_context.evaluation_id` → the Evaluation's **`name`**. - `experiment_ids` (on create evaluation) → a list with the Experiment's **`id`**. -- OTLP evaluation attribute key → **`nemo.evaluation.name`** (test case → `nemo.test_case.id`). +- OTLP evaluation attribute key → **`nemo.evaluation.name`** (test case → `nemo.test_case.name`). - Parent → **`/experiments`** (`/experiment-groups` is a deprecated hidden alias); evaluations → **`/evaluations`**. diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md index 59b0b14457..c58facd9cf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md @@ -12,18 +12,17 @@ Full request schemas for the three intake ingest endpoints. All are under ```json { "evaluation_context": { - "evaluation_id": "my-eval-baseline", - "test_case_id": "dataset/case-001" + "evaluation_name": "my-eval-baseline", + "test_case_name": "dataset/case-001" } } ``` -- `evaluation_id` is the Evaluation's **name** (not its entity id); `test_case_id` is optional (which - task/test case this run covers). +- `test_case_name` is optional and identifies which task or test case the run covers. - The referenced Evaluation **must already exist** (create it first) or the request is rejected with `400 "…must be created before it can be logged."` - The model is lenient (`extra="ignore"`): retired keys (`evaluation_sha`, `evaluation_run_id`, - `metadata`) are accepted but dropped — only `evaluation_id` and `test_case_id` survive. + `metadata`) are accepted but dropped. - A deprecated `experiment_context` `{experiment_id, test_case_id}` shape is still accepted; `evaluation_context` wins if both are present. Use `evaluation_context`. @@ -46,7 +45,7 @@ automatically; you don't call `/evaluator-results` separately for Harbor runs. { "schema_version": "ATIF-v1.5", "session_id": "d074dfb7-3691-443c-b137-720d75e40afa", - "evaluation_context": { "evaluation_id": "my-eval-baseline", "test_case_id": "my-dataset/case-a" }, + "evaluation_context": { "evaluation_name": "my-eval-baseline", "test_case_name": "my-dataset/case-a" }, "agent": { "name": "my-agent", "version": "1.0.0", "model_name": "provider/model" }, "final_metrics": { "total_prompt_tokens": 51701, "total_completion_tokens": 255, @@ -107,7 +106,7 @@ to this shape and publishing it as an Evaluation. "session_id": "session-001", "provider": "openai", "cost_usd": 0.0001, - "evaluation_context": { "evaluation_id": "my-eval-baseline", "test_case_id": "case-001" } + "evaluation_context": { "evaluation_name": "my-eval-baseline", "test_case_name": "case-001" } } ``` @@ -131,11 +130,11 @@ the root span: | Meaning | Span attribute key | |---|---| -| Evaluation (by name) | **`nemo.evaluation.name`** | -| Test case | **`nemo.test_case.id`** | +| Evaluation | **`nemo.evaluation.name`** | +| Test case | **`nemo.test_case.name`** | -> Set `nemo.evaluation.name` to the Evaluation's **name** (not its id), matching the `evaluation_id` -> field used by the JSON `evaluation_context` on the other endpoints. +These attributes correspond to `evaluation_name` and `test_case_name` in the JSON `evaluation_context` +used by the other ingest endpoints. Cost/token/model attributes are read from standard GenAI / OpenInference keys (first match wins): @@ -161,7 +160,7 @@ export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${NMP_BASE_URL}/apis/intake/v2/worksp export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" ``` -Then set `nemo.evaluation.name` (+ `nemo.test_case.id`) on the root span of each run. +Then set `nemo.evaluation.name` (+ `nemo.test_case.name`) on the root span of each run. --- diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context.py index 20f7977103..6330875d31 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context.py @@ -23,14 +23,18 @@ class EvaluationContext(BaseModel): - """Evaluation context accepted by ingest endpoints (the canonical shape). - - ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, - metadata) keeps ingesting without error rather than being rejected. + """ + Identifies the Evaluation and optional test case associated with ingested telemetry. """ evaluation_id: Optional[str] = None + """Deprecated alias for evaluation_name. Use evaluation_name instead.""" + + evaluation_name: Optional[str] = None """Name of an existing Evaluation.""" test_case_id: Optional[str] = None - """Optional producer-supplied test case id.""" + """Deprecated alias for test_case_name. Use test_case_name instead.""" + + test_case_name: Optional[str] = None + """Optional producer-supplied test case name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py index 3609d259b8..48af75eb6f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py @@ -23,14 +23,18 @@ class EvaluationContextParam(TypedDict, total=False): - """Evaluation context accepted by ingest endpoints (the canonical shape). - - ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, - metadata) keeps ingesting without error rather than being rejected. + """ + Identifies the Evaluation and optional test case associated with ingested telemetry. """ evaluation_id: str + """Deprecated alias for evaluation_name. Use evaluation_name instead.""" + + evaluation_name: str """Name of an existing Evaluation.""" test_case_id: str - """Optional producer-supplied test case id.""" + """Deprecated alias for test_case_name. Use test_case_name instead.""" + + test_case_name: str + """Optional producer-supplied test case name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py index 59dd0083a9..e2384d00ef 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py @@ -40,11 +40,9 @@ class AtifCreateParams(TypedDict, total=False): continued_trajectory_ref: str evaluation_context: EvaluationContextParam - """Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + """ + Identifies the Evaluation and optional test case associated with ingested + telemetry. """ extra: Dict[str, object] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_trajectory_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_trajectory_param.py index ae369f9661..77bc730f88 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_trajectory_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_trajectory_param.py @@ -34,11 +34,9 @@ class AtifTrajectoryParam(TypedDict, total=False): continued_trajectory_ref: str evaluation_context: EvaluationContextParam - """Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + """ + Identifies the Evaluation and optional test case associated with ingested + telemetry. """ extra: Dict[str, object] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py index 3537849b81..a0b4f709b0 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py @@ -53,11 +53,9 @@ class ChatCompletionCreateParams(TypedDict, total=False): """ evaluation_context: EvaluationContextParam - """Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + """ + Identifies the Evaluation and optional test case associated with ingested + telemetry. """ provider: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py index e6d83338ec..6201be9498 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py @@ -24,5 +24,13 @@ class SpanEvaluationContext(BaseModel): evaluation_id: Optional[str] = None + """Deprecated alias for evaluation_name. Use evaluation_name instead.""" + + evaluation_name: Optional[str] = None + """Name of an existing Evaluation.""" test_case_id: Optional[str] = None + """Deprecated alias for test_case_name. Use test_case_name instead.""" + + test_case_name: Optional[str] = None + """Optional producer-supplied test case name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_filter_param.py index 658c03d57b..5233259765 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_filter_param.py @@ -34,7 +34,10 @@ class SpanFilterParam(TypedDict, total=False): """Filter by agent application name (e.g. 'claude-code', 'codex').""" evaluation_id: str - """Filter by evaluation id.""" + """Deprecated alias for evaluation_name. Use evaluation_name instead.""" + + evaluation_name: str + """Filter by Evaluation name.""" kind: SpanKind """Filter by normalized span kind.""" @@ -64,7 +67,10 @@ class SpanFilterParam(TypedDict, total=False): """Filter by normalized span status.""" test_case_id: str - """Filter by dataset test case id.""" + """Deprecated alias for test_case_name. Use test_case_name instead.""" + + test_case_name: str + """Filter by test case name.""" tool_name: str """Filter by tool name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_list_params.py index bdda4fe5af..94f82d9540 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_list_params.py @@ -30,8 +30,8 @@ class SpanListParams(TypedDict, total=False): filter: SpanFilterParam """ - Filter spans by session_id, trace_id, parent_span_id, project, evaluation_id, - test_case_id, source, kind, status, model, tool_name, provider, agent_id, + Filter spans by session_id, trace_id, parent_span_id, project, evaluation_name, + test_case_name, source, kind, status, model, tool_name, provider, agent_id, agent_name, and started_at. Every field takes one exact value, except started_at, which takes gte and lte. """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py index e5ababe62f..d8bfddf159 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py @@ -51,11 +51,9 @@ class Trace(BaseModel): error_count: Optional[int] = None evaluation_context: Optional[EvaluationContext] = None - """Evaluation context accepted by ingest endpoints (the canonical shape). - - `extra="ignore"` so a producer still sending retired keys (evaluation_sha, - evaluation_run_id, metadata) keeps ingesting without error rather than being - rejected. + """ + Identifies the Evaluation and optional test case associated with ingested + telemetry. """ input: Optional[str] = None diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py index bcad47fa9c..cdc9a50eb3 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py @@ -60,7 +60,9 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: continued_trajectory_ref="continued_trajectory_ref", evaluation_context={ "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, extra={"foo": "bar"}, final_metrics={ @@ -113,7 +115,9 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: "continued_trajectory_ref": "continued_trajectory_ref", "evaluation_context": { "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, "extra": {"foo": "bar"}, "final_metrics": { @@ -247,7 +251,9 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo continued_trajectory_ref="continued_trajectory_ref", evaluation_context={ "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, extra={"foo": "bar"}, final_metrics={ @@ -300,7 +306,9 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo "continued_trajectory_ref": "continued_trajectory_ref", "evaluation_context": { "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, "extra": {"foo": "bar"}, "final_metrics": { diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py index 4c651481d6..73e4e50486 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py @@ -66,7 +66,9 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: cost_usd=0, evaluation_context={ "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, provider="provider", session_id="session_id", @@ -161,7 +163,9 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo cost_usd=0, evaluation_context={ "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, provider="provider", session_id="session_id", diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/spans/test_groups.py b/sdk/python/nemo-platform/tests/api_resources/intake/spans/test_groups.py index 4358af3ab8..4dd9cae8a0 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/spans/test_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/spans/test_groups.py @@ -53,6 +53,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: "agent_id": "agent_id", "agent_name": "agent_name", "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "kind": "LLM", "model": "model", "parent_span_id": "parent_span_id", @@ -66,6 +67,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: }, "status": "success", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", "tool_name": "tool_name", "trace_id": "trace_id", }, @@ -137,6 +139,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform "agent_id": "agent_id", "agent_name": "agent_name", "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "kind": "LLM", "model": "model", "parent_span_id": "parent_span_id", @@ -150,6 +153,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform }, "status": "success", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", "tool_name": "tool_name", "trace_id": "trace_id", }, diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/test_spans.py b/sdk/python/nemo-platform/tests/api_resources/intake/test_spans.py index bc1ee452d5..8c94e71c7d 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/test_spans.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/test_spans.py @@ -103,6 +103,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: "agent_id": "agent_id", "agent_name": "agent_name", "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "kind": "LLM", "model": "model", "parent_span_id": "parent_span_id", @@ -116,6 +117,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: }, "status": "success", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", "tool_name": "tool_name", "trace_id": "trace_id", }, @@ -235,6 +237,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform "agent_id": "agent_id", "agent_name": "agent_name", "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "kind": "LLM", "model": "model", "parent_span_id": "parent_span_id", @@ -248,6 +251,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform }, "status": "success", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", "tool_name": "tool_name", "trace_id": "trace_id", }, From 8fc568b6b4d37e05d7a15c0fb39cdfc57759aa96 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:06:39 -0600 Subject: [PATCH 05/10] coderabbit Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- docs/evaluator/experiments.mdx | 6 +-- openapi/ga/individual/platform.openapi.yaml | 16 +++++-- openapi/ga/openapi.yaml | 16 +++++-- openapi/openapi.yaml | 16 +++++-- .../intake/src/nmp/intake/spans/api/traces.py | 6 ++- .../nmp/intake/spans/api/traces_schemas.py | 14 +++++- .../spans/test_experiment_rollups.py | 2 +- .../integration/spans/test_traces_read.py | 2 +- services/intake/tests/test_traces_api.py | 45 +++++++++++++++---- 9 files changed, 97 insertions(+), 26 deletions(-) diff --git a/docs/evaluator/experiments.mdx b/docs/evaluator/experiments.mdx index 1b66935b92..31e539a41d 100644 --- a/docs/evaluator/experiments.mdx +++ b/docs/evaluator/experiments.mdx @@ -87,7 +87,7 @@ At read time, each Evaluation is enriched with rollups derived from its sessions | Rollup | Meaning | |--------|---------| -| `test_case_count` | Number of distinct test cases (distinct non-empty `test_case_id` values). Sessions with no `test_case_id` do not count toward it or the rollups. | +| `test_case_count` | Number of distinct test cases (distinct non-empty `test_case_name` values). Sessions with no `test_case_name` do not count toward it or the rollups. | | `cost_usd` | Cost aggregate across the Evaluation's sessions. | | `latency_ms` | Latency aggregate across the Evaluation's sessions. | | `tokens` | Average total tokens (input + output) per test case. | @@ -233,7 +233,7 @@ session with the Evaluation's identity: - For Agent Trajectory Interchange Format (ATIF) and chat-completions, add a **top-level** `evaluation_context` object to the ingest payload - carrying `evaluation_id` (the Evaluation's **name**) and `test_case_id`. + carrying `evaluation_name` and `test_case_name`. - For OpenTelemetry Protocol (OTLP), set the `nemo.evaluation.name` and `nemo.test_case.name` root-span attributes. @@ -501,7 +501,7 @@ window. | Symptom | Cause and fix | |---------|---------------| -| **Rows show zero metrics** | Three causes: no sessions have been ingested for that Evaluation yet; the sessions were ingested without `test_case_id`, so they do not count toward `test_case_count` or the rollups; or ClickHouse is unreachable. Confirm ingestion in Intake, confirm that sessions carry `test_case_id`, and run the explicit metric-sort health check. | +| **Rows show zero metrics** | Three causes: no sessions have been ingested for that Evaluation yet; the sessions were ingested without `test_case_name`, so they do not count toward `test_case_count` or the rollups; or ClickHouse is unreachable. Confirm ingestion in Intake, confirm that sessions carry `test_case_name`, and run the explicit metric-sort health check. | | **A metric sort or filter returns `503`** | Rollups cannot be computed because ClickHouse is down. Retry after the read path is healthy, or use an entity-column sort. | | **A list returns `413`** | The Experiment selected more than 1,000 Evaluations for an in-memory sort. Add filters to narrow the set. | | **An Evaluation is not in the Experiment** | Confirm that you created it with the correct Experiment `id` in `experiment_ids` and that you are querying the correct workspace (`filter[experiment_id]=`). | diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 938c21513d..0fffa7679a 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -4930,7 +4930,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id, and test_case_id. + root span started_at, evaluation_name, and test_case_name. responses: '200': description: Successful Response @@ -18999,12 +18999,22 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by root-span evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by root-span evaluation test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string title: TraceFilter diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 938c21513d..0fffa7679a 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -4930,7 +4930,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id, and test_case_id. + root span started_at, evaluation_name, and test_case_name. responses: '200': description: Successful Response @@ -18999,12 +18999,22 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by root-span evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by root-span evaluation test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string title: TraceFilter diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 938c21513d..0fffa7679a 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4930,7 +4930,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id, and test_case_id. + root span started_at, evaluation_name, and test_case_name. responses: '200': description: Successful Response @@ -18999,12 +18999,22 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by root-span evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by root-span evaluation test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string title: TraceFilter diff --git a/services/intake/src/nmp/intake/spans/api/traces.py b/services/intake/src/nmp/intake/spans/api/traces.py index 928d77ee86..79ff937f9a 100644 --- a/services/intake/src/nmp/intake/spans/api/traces.py +++ b/services/intake/src/nmp/intake/spans/api/traces.py @@ -26,12 +26,16 @@ API_TAG = "Traces" TRACE_INDEX_FILTER_FIELDS = frozenset( { + "evaluation_name", + "test_case_name", "evaluation_id", "test_case_id", } ) TRACE_INDEX_FILTER_ALIASES = { + "evaluation_name": "evaluation_id", "evaluation_id": "evaluation_id", + "test_case_name": "test_case_id", "test_case_id": "test_case_id", } @@ -45,7 +49,7 @@ filter_schema=TraceFilter, filter_description=( "Filter root-span-backed traces by id, session_id, root status, root span started_at, " - "evaluation_id, and test_case_id." + "evaluation_name, and test_case_name." ), ), ) diff --git a/services/intake/src/nmp/intake/spans/api/traces_schemas.py b/services/intake/src/nmp/intake/spans/api/traces_schemas.py index 9f343a41f7..cf56cb5b50 100644 --- a/services/intake/src/nmp/intake/spans/api/traces_schemas.py +++ b/services/intake/src/nmp/intake/spans/api/traces_schemas.py @@ -34,8 +34,18 @@ class TraceFilter(BaseModel): session_id: str | None = Field(default=None, description="Filter by session id.") status: SpanStatus | None = Field(default=None, description="Filter by root span status.") started_at: DatetimeFilter | None = Field(default=None, description="Filter by root span start timestamp.") - evaluation_id: str | None = Field(default=None, description="Filter by root-span evaluation id.") - test_case_id: str | None = Field(default=None, description="Filter by root-span evaluation test case id.") + evaluation_name: str | None = Field(default=None, description="Filter by Evaluation name.") + test_case_name: str | None = Field(default=None, description="Filter by test case name.") + evaluation_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for evaluation_name. Use evaluation_name instead.", + ) + test_case_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for test_case_name. Use test_case_name instead.", + ) class Trace(BaseModel): diff --git a/services/intake/tests/integration/spans/test_experiment_rollups.py b/services/intake/tests/integration/spans/test_experiment_rollups.py index ebf04a8b4c..61620f86a3 100644 --- a/services/intake/tests/integration/spans/test_experiment_rollups.py +++ b/services/intake/tests/integration/spans/test_experiment_rollups.py @@ -292,7 +292,7 @@ def test_deprecated_evaluation_context_hydrates_evaluation_rollups(client: TestC latency_ms=100, offset_seconds=0, ), - "evaluation_context": {"evaluation_name": evaluation_id, "test_case_name": "case-1"}, + "evaluation_context": {"evaluation_id": evaluation_id, "test_case_id": "case-1"}, }, ) diff --git a/services/intake/tests/integration/spans/test_traces_read.py b/services/intake/tests/integration/spans/test_traces_read.py index 41c5137f7b..e5426d89d0 100644 --- a/services/intake/tests/integration/spans/test_traces_read.py +++ b/services/intake/tests/integration/spans/test_traces_read.py @@ -70,7 +70,7 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re "/apis/intake/v2/workspaces/default/traces", params={ "filter[session_id]": "trace-session", - "filter[evaluation_id]": "experiment-a", + "filter[evaluation_name]": "experiment-a", "page_size": 20, }, ) diff --git a/services/intake/tests/test_traces_api.py b/services/intake/tests/test_traces_api.py index b8739204a4..c74bd4abdb 100644 --- a/services/intake/tests/test_traces_api.py +++ b/services/intake/tests/test_traces_api.py @@ -6,6 +6,8 @@ import json from datetime import datetime, timezone +import pytest +from fastapi import HTTPException from nmp.common.api.filter import parse_json_filter from nmp.common.api.parsed_filter import ParsedFilter from nmp.intake.spans.api.traces import _trace_filter @@ -23,8 +25,8 @@ def test_trace_filter_maps_public_fields_to_repository_filter(): "session_id": "session-a", "status": "error", "started_at": {"$gte": started_at.isoformat()}, - "evaluation_id": "experiment-a", - "test_case_id": "case-a", + "evaluation_name": "experiment-a", + "test_case_name": "case-a", } ), ) @@ -38,23 +40,48 @@ def test_trace_filter_maps_public_fields_to_repository_filter(): assert filters.test_case_id == "case-a" -def test_trace_filter_accepts_evaluation_id(): +def test_trace_filter_accepts_deprecated_identifier_aliases(): filters = _trace_filter( "workspace-a", - _parsed_filter({"evaluation_id": "experiment-a"}), + _parsed_filter( + { + "evaluation_name": "experiment-a", + "evaluation_id": "experiment-a", + "test_case_name": "case-a", + "test_case_id": "case-a", + } + ), ) assert filters.evaluation_id == "experiment-a" + assert filters.test_case_id == "case-a" + + +def test_trace_filter_rejects_conflicting_identifier_aliases(): + with pytest.raises(HTTPException) as exc_info: + _trace_filter( + "workspace-a", + _parsed_filter({"evaluation_name": "experiment-a", "evaluation_id": "experiment-b"}), + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == "Conflicting trace filters for evaluation_id" -def test_trace_filter_schema_exposes_evaluation_id(): + +def test_trace_filter_schema_exposes_canonical_names_and_deprecated_aliases(): properties = TraceFilter.model_json_schema()["properties"] - assert properties["evaluation_id"]["description"] == "Filter by root-span evaluation id." - assert "deprecated" not in properties["evaluation_id"] + assert properties["evaluation_name"]["description"] == "Filter by Evaluation name." + assert properties["test_case_name"]["description"] == "Filter by test case name." + assert properties["evaluation_id"]["deprecated"] is True + assert properties["evaluation_id"]["description"] == ( + "Deprecated alias for evaluation_name. Use evaluation_name instead." + ) + assert properties["test_case_id"]["deprecated"] is True + assert properties["test_case_id"]["description"] == ( + "Deprecated alias for test_case_name. Use test_case_name instead." + ) assert "experiment_id" not in properties - assert properties["test_case_id"]["description"] == "Filter by root-span evaluation test case id." - assert "deprecated" not in properties["test_case_id"] def test_trace_filter_applies_no_implicit_time_bound(): From f453d9148941fda889fb2d5bf9cd69aaaf6c9f4c Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:48:56 -0600 Subject: [PATCH 06/10] misc rename continuations Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- .../skills/nemo-experiments-upload/SKILL.md | 2 - .../nemo-intake/references/ingest-formats.md | 4 -- .../src/nemo_evaluator/intake/mapping.py | 4 +- .../src/nemo_evaluator/intake/publish.py | 2 +- .../tests/intake/test_mapping.py | 16 +++++--- .../skills/nemo-experiments-upload/SKILL.md | 2 - .../nemo-intake/references/ingest-formats.md | 4 -- .../scripts/spans/seed_experiments_demo.py | 24 ++++++------ .../intake/api/v2/experiments/endpoints.py | 2 +- .../nmp/intake/experiments/denormalizer.py | 38 +++++++++---------- .../src/nmp/intake/spans/ingest/atif.py | 2 +- .../intake/spans/ingest/chat_completions.py | 2 +- .../src/nmp/intake/spans/ingest/otlp.py | 2 +- .../src/nmp/intake/spans/ingest/spans.py | 4 +- .../tests/test_evaluation_denormalizer.py | 20 +++++----- .../test_evaluation_denormalizer_self_heal.py | 4 +- 16 files changed, 62 insertions(+), 70 deletions(-) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md index de05ee545d..fa0898339f 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md @@ -194,8 +194,6 @@ If `run_count` is 0 after ingesting, the traces didn't associate — almost alwa ## Gotchas - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`. -- **Evaluation and Experiment references use different fields.** `evaluation_name` associates telemetry with an Evaluation, while `experiment_ids` assigns that Evaluation to its parent Experiments. -- **OTLP evaluation context:** set `nemo.evaluation.name` and, when applicable, `nemo.test_case.name` as root-span attributes. These correspond to `evaluation_name` and `test_case_name` in the JSON `evaluation_context` used by other ingest endpoints. - **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`. - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`. - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`). diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md index c58facd9cf..3b7fc9ec97 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md @@ -21,10 +21,6 @@ Full request schemas for the three intake ingest endpoints. All are under - `test_case_name` is optional and identifies which task or test case the run covers. - The referenced Evaluation **must already exist** (create it first) or the request is rejected with `400 "…must be created before it can be logged."` -- The model is lenient (`extra="ignore"`): retired keys (`evaluation_sha`, `evaluation_run_id`, - `metadata`) are accepted but dropped. -- A deprecated `experiment_context` `{experiment_id, test_case_id}` shape is still accepted; - `evaluation_context` wins if both are present. Use `evaluation_context`. --- diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py b/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py index cd9463ff2d..ad73b6da12 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py @@ -84,7 +84,7 @@ def trial_to_atif_ingest( trial: AgentEvalTrial, *, run_id: str, - experiment_id: str, + evaluation_name: str, agent_name: str, started_at: datetime, agent_version: str = DEFAULT_AGENT_VERSION, @@ -132,7 +132,7 @@ def trial_to_atif_ingest( "session_id": session_id_for(run_id, trial.id), "agent": agent, "steps": [step], - "evaluation_context": run_task_to_evaluation_context(trial, evaluation_name=experiment_id), + "evaluation_context": run_task_to_evaluation_context(trial, evaluation_name=evaluation_name), } if final_metrics is not None: body["final_metrics"] = final_metrics diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py b/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py index 9c4dc39695..5071e1c1d5 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py @@ -186,7 +186,7 @@ async def _publish_trial(trial: AgentEvalTrial) -> PublishedTrial: body = mapping.trial_to_atif_ingest( trial, run_id=result.run_id, - experiment_id=experiment_id, + evaluation_name=experiment_id, agent_name=agent_name, started_at=started_at, agent_version=agent_version, diff --git a/plugins/nemo-evaluator/tests/intake/test_mapping.py b/plugins/nemo-evaluator/tests/intake/test_mapping.py index 258f44d49e..d00fe47837 100644 --- a/plugins/nemo-evaluator/tests/intake/test_mapping.py +++ b/plugins/nemo-evaluator/tests/intake/test_mapping.py @@ -90,7 +90,7 @@ def test_trial_to_atif_ingest_shape() -> None: body = trial_to_atif_ingest( _trial(trial_id="t-1", task_id="task-1", output_text="final answer"), run_id="run-1", - experiment_id="exp-1", + evaluation_name="exp-1", agent_name="my-agent", started_at=STARTED_AT, model_name="gpt-4o", @@ -104,14 +104,16 @@ def test_trial_to_atif_ingest_shape() -> None: def test_trial_to_atif_ingest_defaults_version_and_omits_model_name() -> None: - body = trial_to_atif_ingest(_trial(), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT) + body = trial_to_atif_ingest( + _trial(), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT + ) assert body["agent"] == {"name": "a", "version": "unknown"} assert "model_name" not in body["agent"] def test_trial_to_atif_ingest_handles_missing_output() -> None: body = trial_to_atif_ingest( - _trial(output_text=None), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT + _trial(output_text=None), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT ) assert body["steps"] == [{"source": "agent", "step_id": 1, "message": "", "timestamp": STARTED_AT}] @@ -120,7 +122,7 @@ def test_trial_to_atif_ingest_includes_final_metrics_when_given() -> None: body = trial_to_atif_ingest( _trial(), run_id="run-1", - experiment_id="exp-1", + evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT, final_metrics={"total_prompt_tokens": 10}, @@ -133,7 +135,7 @@ def test_trial_to_atif_ingest_adds_invocation_window_when_ended_at_given() -> No # root-span latency is the trial's runtime instead of 0. ended = STARTED_AT + timedelta(seconds=12.5) body = trial_to_atif_ingest( - _trial(), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT, ended_at=ended + _trial(), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT, ended_at=ended ) (step,) = body["steps"] assert step["extra"] == { @@ -142,7 +144,9 @@ def test_trial_to_atif_ingest_adds_invocation_window_when_ended_at_given() -> No def test_trial_to_atif_ingest_omits_invocation_window_without_ended_at() -> None: - body = trial_to_atif_ingest(_trial(), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT) + body = trial_to_atif_ingest( + _trial(), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT + ) assert "extra" not in body["steps"][0] diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md index de05ee545d..fa0898339f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md @@ -194,8 +194,6 @@ If `run_count` is 0 after ingesting, the traces didn't associate — almost alwa ## Gotchas - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`. -- **Evaluation and Experiment references use different fields.** `evaluation_name` associates telemetry with an Evaluation, while `experiment_ids` assigns that Evaluation to its parent Experiments. -- **OTLP evaluation context:** set `nemo.evaluation.name` and, when applicable, `nemo.test_case.name` as root-span attributes. These correspond to `evaluation_name` and `test_case_name` in the JSON `evaluation_context` used by other ingest endpoints. - **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`. - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`. - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`). diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md index c58facd9cf..3b7fc9ec97 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md @@ -21,10 +21,6 @@ Full request schemas for the three intake ingest endpoints. All are under - `test_case_name` is optional and identifies which task or test case the run covers. - The referenced Evaluation **must already exist** (create it first) or the request is rejected with `400 "…must be created before it can be logged."` -- The model is lenient (`extra="ignore"`): retired keys (`evaluation_sha`, `evaluation_run_id`, - `metadata`) are accepted but dropped. -- A deprecated `experiment_context` `{experiment_id, test_case_id}` shape is still accepted; - `evaluation_context` wins if both are present. Use `evaluation_context`. --- diff --git a/services/intake/scripts/spans/seed_experiments_demo.py b/services/intake/scripts/spans/seed_experiments_demo.py index 447b208a79..705e396bcf 100644 --- a/services/intake/scripts/spans/seed_experiments_demo.py +++ b/services/intake/scripts/spans/seed_experiments_demo.py @@ -516,7 +516,7 @@ def _seed_sessions( prompt_tokens = max(10, int(rng.gauss(spec.prompt_tokens_mean, spec.prompt_tokens_mean * 0.25))) completion_tokens = max(5, int(rng.gauss(spec.completion_tokens_mean, spec.completion_tokens_mean * 0.3))) - test_case_id = f"case-{i:04d}" + test_case_name = f"case-{i:04d}" run_id = f"run-{i // 25:02d}" # Spread sessions across the ~5.5h prior to "now" so the Studio timeline looks varied. offset_seconds = (i / max(1, spec.n_sessions)) * 5.5 * 3600 @@ -531,9 +531,9 @@ def _seed_sessions( ) atif_body = _demo_atif_body( base_started_at=base_started_at, - evaluation_id=spec.name, + evaluation_name=spec.name, run_id=run_id, - test_case_id=test_case_id, + test_case_name=test_case_name, cost_usd=cost_usd, latency_ms=latency_ms, offset_seconds=offset_seconds, @@ -571,9 +571,9 @@ def _seed_sessions( def _demo_atif_body( *, base_started_at: datetime, - evaluation_id: str, + evaluation_name: str, run_id: str, - test_case_id: str, + test_case_name: str, cost_usd: float, latency_ms: int, offset_seconds: float, @@ -585,7 +585,7 @@ def _demo_atif_body( ) -> dict[str, Any]: session_started_at = base_started_at + timedelta(seconds=offset_seconds) finished_at = session_started_at + timedelta(milliseconds=latency_ms) - session_id = f"{evaluation_id}-{run_id}-{test_case_id}" + session_id = f"{evaluation_name}-{run_id}-{test_case_name}" # `extra.verifier` carries the timing block (used by the rollup for session latency). # We omit `extra.verifier_result` so ATIF ingest doesn't auto-create a `harbor.verifier` # evaluator alongside our cleanly-named ones from POST /evaluator-results. @@ -593,12 +593,12 @@ def _demo_atif_body( "schema_version": "ATIF-v1.7", "session_id": session_id, "evaluation_context": { - "evaluation_name": evaluation_id, - "test_case_name": test_case_id, + "evaluation_name": evaluation_name, + "test_case_name": test_case_name, }, "extra": { - "task_id": test_case_id, - "task_name": test_case_id, + "task_id": test_case_name, + "task_name": test_case_name, "verifier": { "started_at": _iso(session_started_at), "finished_at": _iso(finished_at), @@ -614,14 +614,14 @@ def _demo_atif_body( "step_id": 1, "timestamp": _iso(session_started_at), "source": "user", - "message": f"test case: {test_case_id}", + "message": f"test case: {test_case_name}", }, { "step_id": 2, "timestamp": _iso(finished_at), "source": "agent", "model_name": model_name, - "message": f"solved {test_case_id}", + "message": f"solved {test_case_name}", "metrics": { "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 1ead0fbfa4..c823f7f556 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -1508,7 +1508,7 @@ def _enqueue_stale_denormalization( or entity.agent_versions != rollup.agent_versions or entity.model_names != rollup.model_names ): - denormalizer.mark_dirty(workspace=workspace, evaluation_id=entity.name) + denormalizer.mark_dirty(workspace=workspace, evaluation_name=entity.name) async def _evaluation_response_with_rollup( diff --git a/services/intake/src/nmp/intake/experiments/denormalizer.py b/services/intake/src/nmp/intake/experiments/denormalizer.py index 88284f714d..2751727f04 100644 --- a/services/intake/src/nmp/intake/experiments/denormalizer.py +++ b/services/intake/src/nmp/intake/experiments/denormalizer.py @@ -3,7 +3,7 @@ """Denormalizes agent/model name fields from ClickHouse onto Evaluation entities. -Ingest marks ``(workspace, evaluation_id)`` dirty; a background loop recomputes each touched +Ingest marks ``(workspace, evaluation_name)`` dirty; a background loop recomputes each touched evaluation's rollup and writes the distinct ``agent_names``/``agent_versions``/``model_names`` sets onto the (system-managed) fields of its Evaluation entity. That lets the Evaluations list filter by agent/model name against the entity store (``$contains``) instead of scanning the ClickHouse session @@ -35,7 +35,7 @@ class EvaluationDenormalizer(BackgroundWorker): - """Coalesces dirty evaluation ids and refreshes their denormalized name fields on a fixed cadence. + """Coalesces dirty evaluation names and refreshes their denormalized name fields on a fixed cadence. A burst of :meth:`mark_dirty` calls for one evaluation within an interval collapses to a single refresh, and marking is a plain, non-blocking set add — so the ingest and read hot paths are never @@ -55,12 +55,12 @@ def __init__( self._interval_seconds = interval_seconds self._dirty: set[tuple[str, str]] = set() - def mark_dirty(self, *, workspace: str, evaluation_id: str) -> None: + def mark_dirty(self, *, workspace: str, evaluation_name: str) -> None: """Queue an evaluation for refresh. Cheap and non-blocking; safe to call from the ingest path.""" - self._dirty.add((workspace, evaluation_id)) + self._dirty.add((workspace, evaluation_name)) def pending(self) -> set[tuple[str, str]]: - """Return a copy of the currently-queued ``(workspace, evaluation_id)`` pairs (observability/tests).""" + """Return a copy of the currently-queued ``(workspace, evaluation_name)`` pairs (observability/tests).""" return set(self._dirty) async def _run(self) -> None: @@ -99,28 +99,28 @@ async def flush(self) -> None: batch = self._dirty self._dirty = set() by_workspace: dict[str, list[str]] = {} - for workspace, evaluation_id in batch: - by_workspace.setdefault(workspace, []).append(evaluation_id) - for workspace, evaluation_ids in by_workspace.items(): + for workspace, evaluation_name in batch: + by_workspace.setdefault(workspace, []).append(evaluation_name) + for workspace, evaluation_names in by_workspace.items(): try: - await self._refresh_workspace(workspace, evaluation_ids) + await self._refresh_workspace(workspace, evaluation_names) except Exception: # Re-queue the whole workspace batch for the next cycle (e.g. ClickHouse unavailable). logger.exception("Failed to refresh evaluation names for workspace %s; re-queuing", workspace) - for evaluation_id in evaluation_ids: - self.mark_dirty(workspace=workspace, evaluation_id=evaluation_id) + for evaluation_name in evaluation_names: + self.mark_dirty(workspace=workspace, evaluation_name=evaluation_name) - async def _refresh_workspace(self, workspace: str, evaluation_ids: list[str]) -> None: - rollups = await self._rollup_repository.get_rollups(workspace=workspace, evaluation_ids=evaluation_ids) - for evaluation_id in evaluation_ids: - rollup = rollups.get(evaluation_id) + async def _refresh_workspace(self, workspace: str, evaluation_names: list[str]) -> None: + rollups = await self._rollup_repository.get_rollups(workspace=workspace, evaluation_ids=evaluation_names) + for evaluation_name in evaluation_names: + rollup = rollups.get(evaluation_name) if rollup is None: continue - await self._write_names(workspace, evaluation_id, rollup) + await self._write_names(workspace, evaluation_name, rollup) - async def _write_names(self, workspace: str, evaluation_id: str, rollup: EvaluationRollup) -> None: + async def _write_names(self, workspace: str, evaluation_name: str, rollup: EvaluationRollup) -> None: try: - evaluation = await self._entity_client.get(Experiment, name=evaluation_id, workspace=workspace) + evaluation = await self._entity_client.get(Experiment, name=evaluation_name, workspace=workspace) except EntityNotFoundError: # Deleted between ingest and refresh; nothing to update. return @@ -139,4 +139,4 @@ async def _write_names(self, workspace: str, evaluation_id: str, rollup: Evaluat await self._entity_client.update(evaluation) except EntityConflictError: # A concurrent user edit won the optimistic lock; re-queue for the next cycle. - self.mark_dirty(workspace=workspace, evaluation_id=evaluation_id) + self.mark_dirty(workspace=workspace, evaluation_name=evaluation_name) diff --git a/services/intake/src/nmp/intake/spans/ingest/atif.py b/services/intake/src/nmp/intake/spans/ingest/atif.py index 75631bcf2b..5013d934d9 100644 --- a/services/intake/src/nmp/intake/spans/ingest/atif.py +++ b/services/intake/src/nmp/intake/spans/ingest/atif.py @@ -138,5 +138,5 @@ async def ingest_atif( await service.ingest_batch(TraceBatch(spans=spans, evaluator_results=evaluator_results)) context = body.evaluation_context if denormalizer is not None and context is not None and context.evaluation_name: - denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_name) + denormalizer.mark_dirty(workspace=workspace, evaluation_name=context.evaluation_name) return Response(status_code=status.HTTP_201_CREATED) diff --git a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py index 1b6313b958..0bd0d019df 100644 --- a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py +++ b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py @@ -163,7 +163,7 @@ async def ingest_chat_completion( await service.ingest_batch(TraceBatch(spans=[span])) context = body.evaluation_context if denormalizer is not None and context is not None and context.evaluation_name: - denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_name) + denormalizer.mark_dirty(workspace=workspace, evaluation_name=context.evaluation_name) return ChatCompletionsIngestResponse( session_id=span.session_id, span_id=span.external_span_id, diff --git a/services/intake/src/nmp/intake/spans/ingest/otlp.py b/services/intake/src/nmp/intake/spans/ingest/otlp.py index 7c21f5ba09..6d31f728a9 100644 --- a/services/intake/src/nmp/intake/spans/ingest/otlp.py +++ b/services/intake/src/nmp/intake/spans/ingest/otlp.py @@ -118,7 +118,7 @@ async def ingest_otlp_traces( # nemo.evaluation.name attribute), so refresh the denormalized name facets for every one it touched. if denormalizer is not None: for evaluation_name in evaluation_names: - denormalizer.mark_dirty(workspace=workspace, evaluation_id=evaluation_name) + denormalizer.mark_dirty(workspace=workspace, evaluation_name=evaluation_name) return IngestResponse(errors=errors) diff --git a/services/intake/src/nmp/intake/spans/ingest/spans.py b/services/intake/src/nmp/intake/spans/ingest/spans.py index fcb8ae42ea..8d67670602 100644 --- a/services/intake/src/nmp/intake/spans/ingest/spans.py +++ b/services/intake/src/nmp/intake/spans/ingest/spans.py @@ -134,9 +134,9 @@ async def ingest_spans( spans = [item for item, _ in converted] await service.ingest_batch(TraceBatch(spans=spans)) if denormalizer is not None: - evaluation_names = {semantic.evaluation_id for _, semantic in converted if semantic.evaluation_id} + evaluation_names = {semantic.evaluation_name for _, semantic in converted if semantic.evaluation_name} for evaluation_name in evaluation_names: - denormalizer.mark_dirty(workspace=workspace, evaluation_id=evaluation_name) + denormalizer.mark_dirty(workspace=workspace, evaluation_name=evaluation_name) return Response(status_code=status.HTTP_201_CREATED) diff --git a/services/intake/tests/test_evaluation_denormalizer.py b/services/intake/tests/test_evaluation_denormalizer.py index 60d62dddc8..54aa9dd028 100644 --- a/services/intake/tests/test_evaluation_denormalizer.py +++ b/services/intake/tests/test_evaluation_denormalizer.py @@ -79,8 +79,8 @@ async def test_flush_writes_denormalized_facets() -> None: entity_client = _FakeEntityClient() refresher = _refresher(repo, entity_client) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") - refresher.mark_dirty(workspace="default", evaluation_id="eval-b") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-b") await refresher.flush() # One batched rollup query for the workspace, covering both dirty evaluations. @@ -112,7 +112,7 @@ async def test_unchanged_facets_skip_write() -> None: ) entity_client = _FakeEntityClient(existing=existing) refresher = _refresher(_FakeRollupRepo(), entity_client) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") await refresher.flush() assert entity_client.updated == [] assert refresher.pending() == set() @@ -120,8 +120,8 @@ async def test_unchanged_facets_skip_write() -> None: def test_mark_dirty_dedupes() -> None: refresher = _refresher(_FakeRollupRepo(), _FakeEntityClient()) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") assert refresher.pending() == {("default", "eval-a")} @@ -136,7 +136,7 @@ async def test_flush_noop_when_clean() -> None: async def test_missing_evaluation_is_skipped() -> None: entity_client = _FakeEntityClient(get_error=EntityNotFoundError("gone")) refresher = _refresher(_FakeRollupRepo(), entity_client) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") await refresher.flush() assert entity_client.updated == [] assert refresher.pending() == set() # not re-queued; the evaluation no longer exists @@ -146,7 +146,7 @@ async def test_missing_evaluation_is_skipped() -> None: async def test_update_conflict_requeues() -> None: entity_client = _FakeEntityClient(update_error=EntityConflictError("version mismatch")) refresher = _refresher(_FakeRollupRepo(), entity_client) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") await refresher.flush() # A concurrent edit won the optimistic lock; the evaluation is re-queued for the next cycle. assert refresher.pending() == {("default", "eval-a")} @@ -156,7 +156,7 @@ async def test_update_conflict_requeues() -> None: async def test_stop_flushes_pending_without_loss() -> None: entity_client = _FakeEntityClient() refresher = _refresher(_FakeRollupRepo(), entity_client) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") refresher.start() # stop() signals the loop to exit and lets it run a final drain — no mid-flush cancellation. await refresher.stop() @@ -168,7 +168,7 @@ async def test_stop_flushes_pending_without_loss() -> None: async def test_rollup_query_failure_requeues() -> None: repo = _FakeRollupRepo(error=RuntimeError("clickhouse down")) refresher = _refresher(repo, _FakeEntityClient()) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") await refresher.flush() assert refresher.pending() == {("default", "eval-a")} @@ -179,7 +179,7 @@ async def test_stop_bounded_drain_does_not_hang_on_persistent_requeue() -> None: # terminate (bounded drain) rather than loop forever, leaving the key queued. entity_client = _FakeEntityClient(update_error=EntityConflictError("version mismatch")) refresher = _refresher(_FakeRollupRepo(), entity_client) - refresher.mark_dirty(workspace="default", evaluation_id="eval-a") + refresher.mark_dirty(workspace="default", evaluation_name="eval-a") refresher.start() # wait_for turns a (regressed) infinite drain into a failure instead of a hung test. await asyncio.wait_for(refresher.stop(), timeout=5) diff --git a/services/intake/tests/test_evaluation_denormalizer_self_heal.py b/services/intake/tests/test_evaluation_denormalizer_self_heal.py index 3be821b954..fe5a2d64db 100644 --- a/services/intake/tests/test_evaluation_denormalizer_self_heal.py +++ b/services/intake/tests/test_evaluation_denormalizer_self_heal.py @@ -23,8 +23,8 @@ class _CapturingRefresher: def __init__(self) -> None: self.marked: list[tuple[str, str]] = [] - def mark_dirty(self, *, workspace: str, evaluation_id: str) -> None: - self.marked.append((workspace, evaluation_id)) + def mark_dirty(self, *, workspace: str, evaluation_name: str) -> None: + self.marked.append((workspace, evaluation_name)) def _entity( From 40cabb661493817a8cf4be8224ad061dea06416d Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:15:20 -0600 Subject: [PATCH 07/10] lint Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- docs/cli/reference.mdx | 4 +++- .../cli/commands/api/intake/traces.py | 10 +++++++++- .../nemo-platform/.nmpcontext/openapi.yaml | 16 +++++++++++++--- .../cli/commands/api/intake/traces.py | 10 +++++++++- .../src/nemo_platform/resources/intake/traces.py | 4 ++-- .../types/intake/trace_filter_param.py | 10 ++++++++-- .../types/intake/trace_list_params.py | 2 +- .../tests/api_resources/intake/test_traces.py | 4 ++++ 8 files changed, 49 insertions(+), 11 deletions(-) diff --git a/docs/cli/reference.mdx b/docs/cli/reference.mdx index c830d17b62..a12d91fadd 100644 --- a/docs/cli/reference.mdx +++ b/docs/cli/reference.mdx @@ -7143,12 +7143,14 @@ nemo intake traces list [OPTIONS] JSON-only fields: started_at: \{gte: str, lte: str} -Filter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id, and test_case_id. +Filter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_name, and test_case_name. * `--filter.id` * `--filter.evaluation-id` +* `--filter.evaluation-name` * `--filter.session-id` * `--filter.status` * `--filter.test-case-id` +* `--filter.test-case-name` **Help:** diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py index 55f6a14220..fc249ff838 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py @@ -42,7 +42,7 @@ def list_traces( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id, and test_case_id.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_name, and test_case_name.", rich_help_panel="Filter Options", ), ] = None, @@ -50,6 +50,9 @@ def list_traces( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, + filter_evaluation_name: Annotated[ + str | None, typer.Option("--filter.evaluation-name", rich_help_panel="Filter Options") + ] = None, filter_session_id: Annotated[ str | None, typer.Option("--filter.session-id", rich_help_panel="Filter Options") ] = None, @@ -57,6 +60,9 @@ def list_traces( filter_test_case_id: Annotated[ str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") ] = None, + filter_test_case_name: Annotated[ + str | None, typer.Option("--filter.test-case-name", rich_help_panel="Filter Options") + ] = None, mode: Annotated[ Literal["summary", "preview", "detailed"] | None, typer.Option( @@ -94,9 +100,11 @@ def list_traces( filter, id=filter_id, evaluation_id=filter_evaluation_id, + evaluation_name=filter_evaluation_name, session_id=filter_session_id, status=filter_status, test_case_id=filter_test_case_id, + test_case_name=filter_test_case_name, ), mode=mode, page=page, diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 9370fe681d..d57b67f9ad 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -4933,7 +4933,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id, and test_case_id. + root span started_at, evaluation_name, and test_case_name. responses: '200': description: Successful Response @@ -19002,12 +19002,22 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_name: + description: Filter by Evaluation name. + title: Evaluation Name + type: string + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string evaluation_id: - description: Filter by root-span evaluation id. + deprecated: true + description: Deprecated alias for evaluation_name. Use evaluation_name instead. title: Evaluation Id type: string test_case_id: - description: Filter by root-span evaluation test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string title: TraceFilter diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py index 11b1a8890a..c7a0c949f3 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py @@ -42,7 +42,7 @@ def list_traces( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id, and test_case_id.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_name, and test_case_name.", rich_help_panel="Filter Options", ), ] = None, @@ -50,6 +50,9 @@ def list_traces( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, + filter_evaluation_name: Annotated[ + str | None, typer.Option("--filter.evaluation-name", rich_help_panel="Filter Options") + ] = None, filter_session_id: Annotated[ str | None, typer.Option("--filter.session-id", rich_help_panel="Filter Options") ] = None, @@ -57,6 +60,9 @@ def list_traces( filter_test_case_id: Annotated[ str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") ] = None, + filter_test_case_name: Annotated[ + str | None, typer.Option("--filter.test-case-name", rich_help_panel="Filter Options") + ] = None, mode: Annotated[ Literal["summary", "preview", "detailed"] | None, typer.Option( @@ -94,9 +100,11 @@ def list_traces( filter, id=filter_id, evaluation_id=filter_evaluation_id, + evaluation_name=filter_evaluation_name, session_id=filter_session_id, status=filter_status, test_case_id=filter_test_case_id, + test_case_name=filter_test_case_name, ), mode=mode, page=page, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py index 45772b94fb..44de226f6a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py @@ -130,7 +130,7 @@ def list( Args: filter: Filter root-span-backed traces by id, session_id, root status, root span - started_at, evaluation_id, and test_case_id. + started_at, evaluation_name, and test_case_name. mode: Response mode. summary returns root-span fields without payloads or rollups; preview adds token, cost, and span-count rollups plus 300-character input/output @@ -264,7 +264,7 @@ def list( Args: filter: Filter root-span-backed traces by id, session_id, root status, root span - started_at, evaluation_id, and test_case_id. + started_at, evaluation_name, and test_case_name. mode: Response mode. summary returns root-span fields without payloads or rollups; preview adds token, cost, and span-count rollups plus 300-character input/output diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py index a18bc2e3b3..c86287ca59 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py @@ -30,7 +30,10 @@ class TraceFilterParam(TypedDict, total=False): """Filter by canonical Intake trace id.""" evaluation_id: str - """Filter by root-span evaluation id.""" + """Deprecated alias for evaluation_name. Use evaluation_name instead.""" + + evaluation_name: str + """Filter by Evaluation name.""" session_id: str """Filter by session id.""" @@ -42,4 +45,7 @@ class TraceFilterParam(TypedDict, total=False): """Filter by root span status.""" test_case_id: str - """Filter by root-span evaluation test case id.""" + """Deprecated alias for test_case_name. Use test_case_name instead.""" + + test_case_name: str + """Filter by test case name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py index 6587d40d33..1f84cc7536 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py @@ -31,7 +31,7 @@ class TraceListParams(TypedDict, total=False): filter: TraceFilterParam """ Filter root-span-backed traces by id, session_id, root status, root span - started_at, evaluation_id, and test_case_id. + started_at, evaluation_name, and test_case_name. """ mode: Literal["summary", "preview", "detailed"] diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py index f77bf6ea7e..fdeb46f10d 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py @@ -112,6 +112,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: filter={ "id": "id", "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "session_id": "session_id", "started_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), @@ -119,6 +120,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: }, "status": "success", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, mode="summary", page=1, @@ -245,6 +247,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform filter={ "id": "id", "evaluation_id": "evaluation_id", + "evaluation_name": "evaluation_name", "session_id": "session_id", "started_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), @@ -252,6 +255,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform }, "status": "success", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, mode="summary", page=1, From cd76f9c5c055d1ae74c19a5686281f18af18de05 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:48:30 -0600 Subject: [PATCH 08/10] fix tests Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- plugins/nemo-evaluator/tests/jobs/test_publication.py | 2 +- .../tests/integration/spans/test_clickhouse_bootstrap.py | 1 + web/packages/studio/src/util/intakeTelemetry.ts | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/plugins/nemo-evaluator/tests/jobs/test_publication.py b/plugins/nemo-evaluator/tests/jobs/test_publication.py index 6c469cc8a9..48d1a4ba2d 100644 --- a/plugins/nemo-evaluator/tests/jobs/test_publication.py +++ b/plugins/nemo-evaluator/tests/jobs/test_publication.py @@ -722,7 +722,7 @@ def test_evaluate_job_uses_the_configured_test_case_id_column(tmp_path: Path, mo ) assert client.atif_calls[0]["session_id"] == "job-1:q-1" - assert client.atif_calls[0]["evaluation_context"]["test_case_id"] == "q-1" + assert client.atif_calls[0]["evaluation_context"]["test_case_name"] == "q-1" def test_evaluate_job_without_a_job_id_cannot_publish(tmp_path: Path, mocker: MockerFixture) -> None: diff --git a/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py b/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py index bef9a86ed7..0ba89cb113 100644 --- a/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py +++ b/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py @@ -42,6 +42,7 @@ def test_clickhouse_bootstrap_is_idempotent(clickhouse_client: ClickHouseSpanCli ("ch_trace_index_0004_nemo_keys",), ("ch_trace_index_0005_evaluation_id",), ("ch_trace_index_0006_nemo_evaluation_name",), + ("ch_trace_index_0007_nemo_test_case_name",), ] expected_ttl = { ClickHouseTable.SPANS: "TTL toDate(start_time) + toIntervalDay(90)", diff --git a/web/packages/studio/src/util/intakeTelemetry.ts b/web/packages/studio/src/util/intakeTelemetry.ts index 84503ce87d..c77de2185a 100644 --- a/web/packages/studio/src/util/intakeTelemetry.ts +++ b/web/packages/studio/src/util/intakeTelemetry.ts @@ -94,10 +94,10 @@ export const getEvaluationContextSummary = ( export const hasEvaluationContext = (context: SpanEvaluationContext | null | undefined): boolean => Boolean( context && - (context.evaluation_name || - context.test_case_name || - context.evaluation_id || - context.test_case_id) + (context.evaluation_name || + context.test_case_name || + context.evaluation_id || + context.test_case_id) ); export const compareSpansByStartedAt = (a: Span, b: Span): number => { From 4526418280e1731bb31ecb8d8724d54bfaf0c1bf Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:03:46 -0600 Subject: [PATCH 09/10] Address evaluation context review feedback Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- .../nmp/intake/repository/clickhouse/trace.py | 20 +++++++++--------- .../intake/src/nmp/intake/spans/api/traces.py | 8 +++---- .../nmp/intake/spans/api/traces_schemas.py | 6 +++--- .../nmp/intake/spans/clickhouse_migrations.py | 21 ++++++++++--------- .../intake/src/nmp/intake/spans/domain.py | 8 +++---- .../integration/spans/test_traces_read.py | 8 +++---- .../tests/test_spans_clickhouse_migrations.py | 7 ++++--- services/intake/tests/test_spans_schemas.py | 4 ++-- services/intake/tests/test_traces_api.py | 10 ++++----- .../test_traces_clickhouse_repository.py | 14 ++++++------- .../IntakeComponents/traceKeyValues.test.tsx | 2 +- .../IntakeComponents/traceKeyValues.tsx | 4 ++-- .../IntakeDetail/SessionDetailView.tsx | 6 +++--- .../IntakeDetail/useSessionTrajectories.ts | 8 +++---- .../studio/src/mocks/intake/telemetry.ts | 4 ++-- .../TestCaseCompare.tsx | 12 +++++------ .../EvaluationSessionDetailRoute/index.tsx | 4 ++-- .../useSessionCompareRuns.ts | 13 ++++++------ 18 files changed, 81 insertions(+), 78 deletions(-) diff --git a/services/intake/src/nmp/intake/repository/clickhouse/trace.py b/services/intake/src/nmp/intake/repository/clickhouse/trace.py index a9d6811923..6f0ee626d1 100644 --- a/services/intake/src/nmp/intake/repository/clickhouse/trace.py +++ b/services/intake/src/nmp/intake/repository/clickhouse/trace.py @@ -44,8 +44,8 @@ "input", "output", "project", - "evaluation_id", - "test_case_id", + "evaluation_name", + "test_case_name", "started_at", "ended_at", "status", @@ -336,8 +336,8 @@ def _trace_select_columns(*, include_aggregates: bool) -> str: "traces.input AS input", "traces.output AS output", "traces.project AS project", - "traces.evaluation_id AS evaluation_id", - "traces.test_case_id AS test_case_id", + "traces.evaluation_name AS evaluation_name", + "traces.test_case_name AS test_case_name", "traces.started_at AS started_at", "traces.ended_at AS ended_at", "traces.status AS status", @@ -381,8 +381,8 @@ def _trace_index_select_columns(*, mode: TraceMode) -> tuple[str, dict[str, Any] "nullIf(trace_roots.root_name, '') AS name", *payload_columns, "nullIf(trace_roots.project, '') AS project", - "nullIf(trace_roots.evaluation_id, '') AS evaluation_id", - "nullIf(trace_roots.test_case_id, '') AS test_case_id", + "nullIf(trace_roots.evaluation_name, '') AS evaluation_name", + "nullIf(trace_roots.test_case_name, '') AS test_case_name", "trace_roots.root_started_at AS started_at", "trace_roots.root_ended_at AS ended_at", "trace_roots.root_status AS status", @@ -462,8 +462,8 @@ def _reconcile_hydrated_page( # Maps API/filter field names to their physical trace_index columns. _TRACE_INDEX_FILTER_COLUMNS = { - "evaluation_id": "evaluation_id", - "test_case_id": "test_case_id", + "evaluation_name": "evaluation_name", + "test_case_name": "test_case_name", } @@ -559,8 +559,8 @@ def _row_to_trace(row: dict[str, Any]) -> IntakeTrace: input=row.get("input") or None, output=row.get("output") or None, project=row.get("project") or None, - evaluation_id=row.get("evaluation_id") or None, - test_case_id=row.get("test_case_id") or None, + evaluation_name=row.get("evaluation_name") or None, + test_case_name=row.get("test_case_name") or None, started_at=row["started_at"], ended_at=ended_at, duration_ms=_duration_ms(row["started_at"], ended_at), diff --git a/services/intake/src/nmp/intake/spans/api/traces.py b/services/intake/src/nmp/intake/spans/api/traces.py index 79ff937f9a..273cecce95 100644 --- a/services/intake/src/nmp/intake/spans/api/traces.py +++ b/services/intake/src/nmp/intake/spans/api/traces.py @@ -33,10 +33,10 @@ } ) TRACE_INDEX_FILTER_ALIASES = { - "evaluation_name": "evaluation_id", - "evaluation_id": "evaluation_id", - "test_case_name": "test_case_id", - "test_case_id": "test_case_id", + "evaluation_name": "evaluation_name", + "evaluation_id": "evaluation_name", + "test_case_name": "test_case_name", + "test_case_id": "test_case_name", } diff --git a/services/intake/src/nmp/intake/spans/api/traces_schemas.py b/services/intake/src/nmp/intake/spans/api/traces_schemas.py index cf56cb5b50..8fe8a2e8cf 100644 --- a/services/intake/src/nmp/intake/spans/api/traces_schemas.py +++ b/services/intake/src/nmp/intake/spans/api/traces_schemas.py @@ -111,9 +111,9 @@ def from_domain(cls, trace: IntakeTrace, *, mode: TraceMode = "detailed") -> Sel def _evaluation_context(trace: IntakeTrace) -> EvaluationContext | None: - if trace.evaluation_id is None: + if trace.evaluation_name is None: return None return EvaluationContext( - evaluation_name=trace.evaluation_id, - test_case_name=trace.test_case_id, + evaluation_name=trace.evaluation_name, + test_case_name=trace.test_case_name, ) diff --git a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py index 4d951ebf11..5515ad77c8 100644 --- a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py +++ b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py @@ -254,8 +254,8 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> project_key = spec_for_field(SpanAttributeField.PROJECT).bag_key # Ingest writes canonical keys, while the backfill must preserve associations on historical rows. - evaluation_id_expr = _coalesced_string_attribute(SpanAttributeField.EVALUATION_NAME) - test_case_id_expr = _coalesced_string_attribute(SpanAttributeField.TEST_CASE_NAME) + evaluation_name_expr = _coalesced_string_attribute(SpanAttributeField.EVALUATION_NAME) + test_case_name_expr = _coalesced_string_attribute(SpanAttributeField.TEST_CASE_NAME) # Note this is logically a single table. CH requires creating an underlying table and then a view that writes to that table. client.command( @@ -273,8 +273,8 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> root_output String CODEC(ZSTD(3)), project String DEFAULT '', - evaluation_id String DEFAULT '', - test_case_id String DEFAULT '', + evaluation_name String DEFAULT '', + test_case_name String DEFAULT '', root_started_at DateTime64(6) CODEC(Delta(8), ZSTD(1)), root_ended_at Nullable(DateTime64(6)) CODEC(Delta(8), ZSTD(1)), @@ -285,8 +285,8 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> INDEX idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_session_id session_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_evaluation_id evaluation_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_test_case_id test_case_id TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_evaluation_name evaluation_name TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_test_case_name test_case_name TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_root_status root_status TYPE set(4) GRANULARITY 4, INDEX idx_source_format source_format TYPE set(8) GRANULARITY 4 ) @@ -312,8 +312,8 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> input AS root_input, output AS root_output, attributes_string['{project_key}'] AS project, - {evaluation_id_expr} AS evaluation_id, - {test_case_id_expr} AS test_case_id, + {evaluation_name_expr} AS evaluation_name, + {test_case_name_expr} AS test_case_name, start_time AS root_started_at, nullIf(end_time, toDateTime64(0, 6)) AS root_ended_at, if(end_time = toDateTime64(0, 6), NULL, dateDiff('millisecond', start_time, end_time)) AS latency_ms, @@ -362,8 +362,9 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> # the MV now coalesces both keys, so spans already ingested under ``nemo.experiment.id`` keep their # evaluation association while new spans use the canonical key. ("ch_trace_index_0006_nemo_evaluation_name", _create_trace_index_schema), - # The test-case span-attribute bag key was renamed ``nemo.test_case.id`` -> ``nemo.test_case.name``. - # Rebuild the MV so new root spans use the canonical key while the backfill coalesces both keys. + # The test-case span-attribute bag key was renamed ``nemo.test_case.id`` -> ``nemo.test_case.name``, + # and the trace_index columns now describe the name values they store. Rebuild the derived table + # and MV with canonical columns while the backfill coalesces canonical and historical bag keys. ("ch_trace_index_0007_nemo_test_case_name", _create_trace_index_schema), ] CURRENT_SCHEMA_VERSION = _MIGRATIONS[-1][0] diff --git a/services/intake/src/nmp/intake/spans/domain.py b/services/intake/src/nmp/intake/spans/domain.py index f13b470abb..35d84b7e5a 100644 --- a/services/intake/src/nmp/intake/spans/domain.py +++ b/services/intake/src/nmp/intake/spans/domain.py @@ -96,8 +96,8 @@ class TraceListFilter(BaseModel): status: SpanStatus | None = None started_at_gte: datetime | None = None started_at_lte: datetime | None = None - evaluation_id: str | None = None - test_case_id: str | None = None + evaluation_name: str | None = None + test_case_name: str | None = None IntakeResponseMode = Literal["summary", "preview", "detailed"] @@ -115,8 +115,8 @@ class IntakeTrace(BaseModel): input: str | None = None output: str | None = None project: str | None = None - evaluation_id: str | None = None - test_case_id: str | None = None + evaluation_name: str | None = None + test_case_name: str | None = None started_at: datetime ended_at: datetime | None = None duration_ms: float | None = None diff --git a/services/intake/tests/integration/spans/test_traces_read.py b/services/intake/tests/integration/spans/test_traces_read.py index e5426d89d0..6b40da141c 100644 --- a/services/intake/tests/integration/spans/test_traces_read.py +++ b/services/intake/tests/integration/spans/test_traces_read.py @@ -24,11 +24,11 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re "openinference.span.kind": "AGENT", "gen_ai.conversation.id": "trace-session", "project": "project-a", - # Emit the legacy evaluation key on purpose: ingest normalizes it to the canonical - # nemo.evaluation.name, so this asserts the dual-read path end to end (a pre-rename - # producer still associates its traces to the evaluation). + # Emit the legacy evaluation and test-case keys on purpose: ingest normalizes them + # to the canonical name keys, so this asserts the dual-read path end to end (a + # pre-rename producer still associates its traces to the evaluation and test case). "nemo.experiment.id": "experiment-a", - "nemo.test_case.name": "case-a", + "nemo.test_case.id": "case-a", "deployment.environment.name": "prod", "tag.tags": ["trace-read"], "metadata": {"owner": "trace-test"}, diff --git a/services/intake/tests/test_spans_clickhouse_migrations.py b/services/intake/tests/test_spans_clickhouse_migrations.py index f63767ed0e..b9130af065 100644 --- a/services/intake/tests/test_spans_clickhouse_migrations.py +++ b/services/intake/tests/test_spans_clickhouse_migrations.py @@ -63,15 +63,16 @@ def test_trace_index_schema_is_root_span_projection(): assert "WHERE external_parent_span_id = ''" in ddl # Both identifiers resolve via canonical keys plus historical aliases, so the backfill keeps older # spans associated. - assert "{evaluation_id_expr} AS evaluation_id" in ddl - assert "{test_case_id_expr} AS test_case_id" in ddl + assert "{evaluation_name_expr} AS evaluation_name" in ddl + assert "{test_case_name_expr} AS test_case_name" in ddl assert "_coalesced_string_attribute(SpanAttributeField.EVALUATION_NAME)" in ddl assert "_coalesced_string_attribute(SpanAttributeField.TEST_CASE_NAME)" in ddl assert "root_status LowCardinality(String)" in ddl assert "root_input String" in ddl assert "PRIMARY KEY (workspace, root_started_at)" in ddl assert "ORDER BY (workspace, root_started_at, trace_id, root_span_id)" in ddl - assert "INDEX idx_evaluation_id evaluation_id" in ddl + assert "INDEX idx_evaluation_name evaluation_name" in ddl + assert "INDEX idx_test_case_name test_case_name" in ddl assert "index_granularity = 256" in ddl diff --git a/services/intake/tests/test_spans_schemas.py b/services/intake/tests/test_spans_schemas.py index 0cead077ef..3cc1d0b5c8 100644 --- a/services/intake/tests/test_spans_schemas.py +++ b/services/intake/tests/test_spans_schemas.py @@ -178,8 +178,8 @@ def test_trace_response_maps_core_trace_fields(): input="root input", output="root output", project="project-a", - evaluation_id="experiment-a", - test_case_id="case-a", + evaluation_name="experiment-a", + test_case_name="case-a", started_at=started_at, ended_at=ended_at, duration_ms=2500, diff --git a/services/intake/tests/test_traces_api.py b/services/intake/tests/test_traces_api.py index c74bd4abdb..1126b975aa 100644 --- a/services/intake/tests/test_traces_api.py +++ b/services/intake/tests/test_traces_api.py @@ -36,8 +36,8 @@ def test_trace_filter_maps_public_fields_to_repository_filter(): assert filters.session_id == "session-a" assert filters.status == SpanStatus.ERROR assert filters.started_at_gte == started_at - assert filters.evaluation_id == "experiment-a" - assert filters.test_case_id == "case-a" + assert filters.evaluation_name == "experiment-a" + assert filters.test_case_name == "case-a" def test_trace_filter_accepts_deprecated_identifier_aliases(): @@ -53,8 +53,8 @@ def test_trace_filter_accepts_deprecated_identifier_aliases(): ), ) - assert filters.evaluation_id == "experiment-a" - assert filters.test_case_id == "case-a" + assert filters.evaluation_name == "experiment-a" + assert filters.test_case_name == "case-a" def test_trace_filter_rejects_conflicting_identifier_aliases(): @@ -65,7 +65,7 @@ def test_trace_filter_rejects_conflicting_identifier_aliases(): ) assert exc_info.value.status_code == 400 - assert exc_info.value.detail == "Conflicting trace filters for evaluation_id" + assert exc_info.value.detail == "Conflicting trace filters for evaluation_name" def test_trace_filter_schema_exposes_canonical_names_and_deprecated_aliases(): diff --git a/services/intake/tests/test_traces_clickhouse_repository.py b/services/intake/tests/test_traces_clickhouse_repository.py index fe158b9abe..d7e60defdd 100644 --- a/services/intake/tests/test_traces_clickhouse_repository.py +++ b/services/intake/tests/test_traces_clickhouse_repository.py @@ -255,8 +255,8 @@ async def test_list_traces_maps_detailed_row(): assert trace.output == "root output" assert trace.duration_ms == 2500 assert trace.project == "project-a" - assert trace.evaluation_id == "experiment-a" - assert trace.test_case_id == "case-a" + assert trace.evaluation_name == "experiment-a" + assert trace.test_case_name == "case-a" assert trace.input_tokens == 420 assert trace.output_tokens == 310 assert trace.cached_tokens == 128 @@ -389,7 +389,7 @@ async def test_root_filters_use_trace_index_columns(): await repository.list_traces( filters=TraceListFilter( workspace="workspace-a", - evaluation_id="experiment-a", + evaluation_name="experiment-a", ), page=1, page_size=10, @@ -397,9 +397,9 @@ async def test_root_filters_use_trace_index_columns(): mode="detailed", ) - assert "trace_roots.evaluation_id = %(filter_evaluation_id)s" in client.queries[0] + assert "trace_roots.evaluation_name = %(filter_evaluation_name)s" in client.queries[0] assert "candidate_spans" not in client.queries[0] - assert client.parameters[0]["filter_evaluation_id"] == "experiment-a" + assert client.parameters[0]["filter_evaluation_name"] == "experiment-a" def _trace_row( @@ -420,8 +420,8 @@ def _trace_row( "input": "root input" if detailed else "", "output": "root output" if detailed else "", "project": "project-a", - "evaluation_id": "experiment-a", - "test_case_id": "case-a", + "evaluation_name": "experiment-a", + "test_case_name": "case-a", "started_at": started_at, "ended_at": ended_at, "status": "error", diff --git a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx index 7bae896827..07cc9dd0e1 100644 --- a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx +++ b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx @@ -53,7 +53,7 @@ describe('traceKeyValues', () => { const entries = buildEvaluationContextEntries(trace!.evaluation_context); - expect(entries.map((entry) => entry.label)).toEqual(['Evaluation ID', 'Test Case ID']); + expect(entries.map((entry) => entry.label)).toEqual(['Evaluation Name', 'Test Case Name']); }); it('returns no evaluation context entries when context is absent', () => { diff --git a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx index 8e7fb4daca..6d0902f492 100644 --- a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx +++ b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx @@ -128,8 +128,8 @@ const EVALUATION_CONTEXT_DESCRIPTORS: readonly { readonly key: keyof EvaluationContext | string; readonly label: string; }[] = [ - { key: 'evaluation_id', label: 'Evaluation ID' }, - { key: 'test_case_id', label: 'Test Case ID' }, + { key: 'evaluation_name', label: 'Evaluation Name' }, + { key: 'test_case_name', label: 'Test Case Name' }, ]; const collectDescriptorEntries = ( diff --git a/web/packages/studio/src/components/IntakeDetail/SessionDetailView.tsx b/web/packages/studio/src/components/IntakeDetail/SessionDetailView.tsx index a1ed620cff..30a83fa2ce 100644 --- a/web/packages/studio/src/components/IntakeDetail/SessionDetailView.tsx +++ b/web/packages/studio/src/components/IntakeDetail/SessionDetailView.tsx @@ -65,11 +65,11 @@ export const SessionDetailView: FC = ({ isTracesLoading, trajectories, explorer, - testCaseId, + testCaseName, } = useSessionTrajectories(workspace, sessionId); const title = - routeContext?.kind === 'evaluation' && testCaseId - ? `Test case: ${testCaseId}` + routeContext?.kind === 'evaluation' && testCaseName + ? `Test case: ${testCaseName}` : `Session ${sessionId}`; const sessionHref = getSessionHref(sessionId); const baseBreadcrumbs = useMemo( diff --git a/web/packages/studio/src/components/IntakeDetail/useSessionTrajectories.ts b/web/packages/studio/src/components/IntakeDetail/useSessionTrajectories.ts index c8a1db1896..fa4ea36185 100644 --- a/web/packages/studio/src/components/IntakeDetail/useSessionTrajectories.ts +++ b/web/packages/studio/src/components/IntakeDetail/useSessionTrajectories.ts @@ -15,7 +15,7 @@ const SESSION_TRACES_PAGE_SIZE = 1000; /** * Loads a session's summary, its traces, and their span trajectories — plus the - * explorer bundle the span views need and the session's producer test_case_id. + * explorer bundle the span views need and the session's producer test case name. * * Shared by the single session detail view and the test-case comparison columns; * React Query dedupes the fetches when the same session renders in more than one @@ -101,8 +101,8 @@ export function useSessionTrajectories(workspace: string, sessionId: string) { [isSessionSpansFetching, sessionSpansError, sessionSpansResponse, trajectories] ); - const testCaseId = traces.find((trace) => trace.evaluation_context?.test_case_id) - ?.evaluation_context?.test_case_id; + const testCaseName = traces.find((trace) => trace.evaluation_context?.test_case_name) + ?.evaluation_context?.test_case_name; return { session, @@ -113,6 +113,6 @@ export function useSessionTrajectories(workspace: string, sessionId: string) { isTracesLoading, trajectories, explorer, - testCaseId, + testCaseName, }; } diff --git a/web/packages/studio/src/mocks/intake/telemetry.ts b/web/packages/studio/src/mocks/intake/telemetry.ts index bc1650656f..300752918f 100644 --- a/web/packages/studio/src/mocks/intake/telemetry.ts +++ b/web/packages/studio/src/mocks/intake/telemetry.ts @@ -36,8 +36,8 @@ const trace1: Trace = { span_count: 4, error_count: 0, evaluation_context: { - evaluation_id: 'support-policy-smoke', - test_case_id: 'case-0042', + evaluation_name: 'support-policy-smoke', + test_case_name: 'case-0042', }, }; diff --git a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx index 3d119707ca..3737daf2b6 100644 --- a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx +++ b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx @@ -17,7 +17,7 @@ import { type FC, type ReactNode, useEffect } from 'react'; interface TestCaseCompareProps { workspace: string; experimentName: string; - testCaseId: string | null | undefined; + testCaseName: string | null | undefined; /** The run shown in the left column (the session the route is on). */ primarySessionId: string; primaryRun: EvaluationSessionResponse | undefined; @@ -41,7 +41,7 @@ const CompareEmpty: FC<{ heading: string; message: string }> = ({ heading, messa export const TestCaseCompare: FC = ({ workspace, experimentName, - testCaseId, + testCaseName, primarySessionId, primaryRun, compareSessionId, @@ -63,16 +63,16 @@ export const TestCaseCompare: FC = ({ setBreadcrumbs(breadcrumbs); }, [setBreadcrumbs, workspace, experimentName]); - const heading = testCaseId - ? `Test case comparison — Test case ${testCaseId}` + const heading = testCaseName + ? `Test case comparison — Test case ${testCaseName}` : 'Test case comparison'; const renderRightColumn = () => { - if (!testCaseId) { + if (!testCaseName) { return ( ); } diff --git a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx index 3f30d02d0b..2a640c6bdc 100644 --- a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx @@ -32,7 +32,7 @@ const EvaluationSessionCompare: FC<{ onSelectCompare: (sessionId: string) => void; onClearCompare: () => void; }> = ({ workspace, experimentName, sessionId, compareWith, onSelectCompare, onClearCompare }) => { - const { testCaseId, runs, isRunsLoading } = useSessionCompareRuns( + const { testCaseName, runs, isRunsLoading } = useSessionCompareRuns( workspace, experimentName, sessionId @@ -44,7 +44,7 @@ const EvaluationSessionCompare: FC<{ ['runs']; isRunsLoading: boolean; } { - // The session's traces supply the test_case_id every run is matched on. - const { testCaseId } = useSessionTrajectories(workspace, sessionId); + // The session's traces supply the test case name every run is matched on. + const { testCaseName } = useSessionTrajectories(workspace, sessionId); const { data: group } = useGetExperiment(workspace, experimentName); const { data: evaluationsPage } = useListEvaluations( @@ -40,8 +40,9 @@ export function useSessionCompareRuns( const { runs, isLoading: isRunsLoading } = useTestCaseRuns({ workspace, evaluationNames, - testCaseId, + // Evaluation sessions still expose this lookup through their test_case_id field. + testCaseId: testCaseName, }); - return { testCaseId, runs, isRunsLoading }; + return { testCaseName, runs, isRunsLoading }; } From d4c8d0dfcc62e68d7010b582becac073e1329ca2 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:07:36 -0600 Subject: [PATCH 10/10] more renames in evaluations sessions api Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Signed-off-by: Brian Newsom --- openapi/ga/individual/platform.openapi.yaml | 27 +++-- openapi/ga/openapi.yaml | 27 +++-- openapi/openapi.yaml | 27 +++-- .../nemo-platform/.nmpcontext/openapi.yaml | 27 +++-- .../resources/evaluations/sessions.py | 8 +- .../types/evaluations/evaluation_response.py | 6 +- .../evaluation_session_filter_param.py | 5 +- .../evaluation_session_response.py | 5 +- .../types/evaluations/session_list_params.py | 4 +- .../evaluations/test_sessions.py | 2 + .../intake/api/v2/experiments/endpoints.py | 24 ++++- .../nmp/intake/api/v2/experiments/schemas.py | 21 +++- .../nmp/intake/experiments/denormalizer.py | 2 +- .../nmp/intake/experiments/read_service.py | 6 +- .../clickhouse/evaluation_rollup.py | 101 ++++++++++-------- .../clickhouse/evaluation_session.py | 38 +++---- .../intake/repository/evaluation_rollup.py | 6 +- .../intake/repository/evaluation_session.py | 4 +- .../spans/test_experiment_sessions.py | 48 +++++++-- .../integration/test_experiments_crud.py | 2 +- .../tests/test_evaluation_denormalizer.py | 10 +- .../test_evaluation_denormalizer_self_heal.py | 2 +- .../tests/test_evaluation_read_service.py | 10 +- ...valuation_session_clickhouse_repository.py | 12 ++- .../test_experiment_rollup_repository.py | 30 ++++-- .../tests/test_experiment_session_schemas.py | 20 +++- services/intake/tests/test_experiment_sort.py | 12 ++- .../EvaluationSessionsDataView/index.test.tsx | 4 +- .../EvaluationSessionsDataView/index.tsx | 10 +- web/packages/studio/src/mocks/handlers.ts | 4 +- .../studio/src/mocks/intake/experiments.ts | 20 ++-- .../useSessionCompareRuns.ts | 3 +- .../useTestCaseRuns.ts | 10 +- 33 files changed, 341 insertions(+), 196 deletions(-) diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 0fffa7679a..1abfc4f38a 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -4083,16 +4083,16 @@ paths: schema: description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending\ - \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at,\ + \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted,\ \ sessions are ordered by started_at ascending." title: Sort type: string description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending \u2014\ - \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at, ended_at,\ - \ latency_ms, status, cost_total_usd, tokens. When omitted, sessions are\ - \ ordered by started_at ascending." + \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ + \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, sessions\ + \ are ordered by started_at ascending." - in: query name: filter style: deepObject @@ -4100,7 +4100,7 @@ paths: explode: true schema: $ref: '#/components/schemas/EvaluationSessionFilter' - description: Filter sessions by test_case_id and status. + description: Filter sessions by test_case_name and status. responses: '200': description: Successful Response @@ -11393,7 +11393,7 @@ components: type: integer title: Test Case Count description: Number of distinct test cases in the evaluation, i.e. distinct - test_case_id values (sessions with no test_case_id each count as their + test_case_name values (sessions with no test_case_name each count as their own). A test case run k times counts once; the rollup metrics are averaged per test case before pooling across test cases. default: 0 @@ -11452,8 +11452,13 @@ components: additionalProperties: false description: Filter for listing EvaluationSessions. properties: + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string test_case_id: - description: Filter by producer-supplied test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string status: @@ -11473,10 +11478,14 @@ components: session_id: type: string title: Session Id + test_case_name: + title: Test Case Name + description: Test case name; null when the producer did not set one. + type: string test_case_id: title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string trace_id: type: string diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 0fffa7679a..1abfc4f38a 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -4083,16 +4083,16 @@ paths: schema: description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending\ - \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at,\ + \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted,\ \ sessions are ordered by started_at ascending." title: Sort type: string description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending \u2014\ - \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at, ended_at,\ - \ latency_ms, status, cost_total_usd, tokens. When omitted, sessions are\ - \ ordered by started_at ascending." + \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ + \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, sessions\ + \ are ordered by started_at ascending." - in: query name: filter style: deepObject @@ -4100,7 +4100,7 @@ paths: explode: true schema: $ref: '#/components/schemas/EvaluationSessionFilter' - description: Filter sessions by test_case_id and status. + description: Filter sessions by test_case_name and status. responses: '200': description: Successful Response @@ -11393,7 +11393,7 @@ components: type: integer title: Test Case Count description: Number of distinct test cases in the evaluation, i.e. distinct - test_case_id values (sessions with no test_case_id each count as their + test_case_name values (sessions with no test_case_name each count as their own). A test case run k times counts once; the rollup metrics are averaged per test case before pooling across test cases. default: 0 @@ -11452,8 +11452,13 @@ components: additionalProperties: false description: Filter for listing EvaluationSessions. properties: + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string test_case_id: - description: Filter by producer-supplied test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string status: @@ -11473,10 +11478,14 @@ components: session_id: type: string title: Session Id + test_case_name: + title: Test Case Name + description: Test case name; null when the producer did not set one. + type: string test_case_id: title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string trace_id: type: string diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 0fffa7679a..1abfc4f38a 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4083,16 +4083,16 @@ paths: schema: description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending\ - \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at,\ + \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted,\ \ sessions are ordered by started_at ascending." title: Sort type: string description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending \u2014\ - \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at, ended_at,\ - \ latency_ms, status, cost_total_usd, tokens. When omitted, sessions are\ - \ ordered by started_at ascending." + \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ + \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, sessions\ + \ are ordered by started_at ascending." - in: query name: filter style: deepObject @@ -4100,7 +4100,7 @@ paths: explode: true schema: $ref: '#/components/schemas/EvaluationSessionFilter' - description: Filter sessions by test_case_id and status. + description: Filter sessions by test_case_name and status. responses: '200': description: Successful Response @@ -11393,7 +11393,7 @@ components: type: integer title: Test Case Count description: Number of distinct test cases in the evaluation, i.e. distinct - test_case_id values (sessions with no test_case_id each count as their + test_case_name values (sessions with no test_case_name each count as their own). A test case run k times counts once; the rollup metrics are averaged per test case before pooling across test cases. default: 0 @@ -11452,8 +11452,13 @@ components: additionalProperties: false description: Filter for listing EvaluationSessions. properties: + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string test_case_id: - description: Filter by producer-supplied test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string status: @@ -11473,10 +11478,14 @@ components: session_id: type: string title: Session Id + test_case_name: + title: Test Case Name + description: Test case name; null when the producer did not set one. + type: string test_case_id: title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string trace_id: type: string diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index d57b67f9ad..d85678a56d 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -4086,16 +4086,16 @@ paths: schema: description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending\ - \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at,\ + \ \u2014 e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted,\ \ sessions are ordered by started_at ascending." title: Sort type: string description: "Comma-separated list of fields to sort by, applied in order\ \ (the first field dominates); prefix a field with '-' for descending \u2014\ - \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at, ended_at,\ - \ latency_ms, status, cost_total_usd, tokens. When omitted, sessions are\ - \ ordered by started_at ascending." + \ e.g. '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at,\ + \ ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, sessions\ + \ are ordered by started_at ascending." - in: query name: filter style: deepObject @@ -4103,7 +4103,7 @@ paths: explode: true schema: $ref: '#/components/schemas/EvaluationSessionFilter' - description: Filter sessions by test_case_id and status. + description: Filter sessions by test_case_name and status. responses: '200': description: Successful Response @@ -11396,7 +11396,7 @@ components: type: integer title: Test Case Count description: Number of distinct test cases in the evaluation, i.e. distinct - test_case_id values (sessions with no test_case_id each count as their + test_case_name values (sessions with no test_case_name each count as their own). A test case run k times counts once; the rollup metrics are averaged per test case before pooling across test cases. default: 0 @@ -11455,8 +11455,13 @@ components: additionalProperties: false description: Filter for listing EvaluationSessions. properties: + test_case_name: + description: Filter by test case name. + title: Test Case Name + type: string test_case_id: - description: Filter by producer-supplied test case id. + deprecated: true + description: Deprecated alias for test_case_name. Use test_case_name instead. title: Test Case Id type: string status: @@ -11476,10 +11481,14 @@ components: session_id: type: string title: Session Id + test_case_name: + title: Test Case Name + description: Test case name; null when the producer did not set one. + type: string test_case_id: title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + description: Deprecated alias for test_case_name. Use test_case_name instead. + deprecated: true type: string trace_id: type: string diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/sessions.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/sessions.py index 25045abb08..024dc7952b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/sessions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/sessions.py @@ -81,7 +81,7 @@ def list( List Evaluation Sessions Args: - filter: Filter sessions by test_case_id and status. + filter: Filter sessions by test_case_name and status. mode: Response mode. summary omits root-span input and output; preview includes both truncated to 300 characters; detailed returns full root-span payloads. @@ -92,7 +92,7 @@ def list( sort: Comma-separated list of fields to sort by, applied in order (the first field dominates); prefix a field with '-' for descending — e.g. - '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at, ended_at, + '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at, ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, sessions are ordered by started_at ascending. @@ -176,7 +176,7 @@ def list( List Evaluation Sessions Args: - filter: Filter sessions by test_case_id and status. + filter: Filter sessions by test_case_name and status. mode: Response mode. summary omits root-span input and output; preview includes both truncated to 300 characters; detailed returns full root-span payloads. @@ -187,7 +187,7 @@ def list( sort: Comma-separated list of fields to sort by, applied in order (the first field dominates); prefix a field with '-' for descending — e.g. - '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at, ended_at, + '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at, ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, sessions are ordered by started_at ascending. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py index 3764cccfac..a84e9c3465 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py @@ -95,9 +95,9 @@ class EvaluationResponse(BaseModel): test_case_count: Optional[int] = None """Number of distinct test cases in the evaluation, i.e. - distinct test_case_id values (sessions with no test_case_id each count as their - own). A test case run k times counts once; the rollup metrics are averaged per - test case before pooling across test cases. + distinct test_case_name values (sessions with no test_case_name each count as + their own). A test case run k times counts once; the rollup metrics are averaged + per test case before pooling across test cases. """ tokens: Optional[EvaluatorAggregate] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_filter_param.py index 6bc7f9d8dd..cca51ca4b5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_filter_param.py @@ -29,4 +29,7 @@ class EvaluationSessionFilterParam(TypedDict, total=False): """Filter by root-span status (success, error, cancelled, unknown).""" test_case_id: str - """Filter by producer-supplied test case id.""" + """Deprecated alias for test_case_name. Use test_case_name instead.""" + + test_case_name: str + """Filter by test case name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py index fc453c4add..9ee8ab1739 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py @@ -82,4 +82,7 @@ class EvaluationSessionResponse(BaseModel): """Sum of output tokens across this session's spans.""" test_case_id: Optional[str] = None - """Producer-supplied test case identifier; null when the producer did not set one.""" + """Deprecated alias for test_case_name. Use test_case_name instead.""" + + test_case_name: Optional[str] = None + """Test case name; null when the producer did not set one.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/session_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/session_list_params.py index 75ebc5c5e5..de29744327 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/session_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/session_list_params.py @@ -28,7 +28,7 @@ class SessionListParams(TypedDict, total=False): workspace: str filter: EvaluationSessionFilterParam - """Filter sessions by test_case_id and status.""" + """Filter sessions by test_case_name and status.""" mode: Literal["summary", "preview", "detailed"] """Response mode. @@ -47,7 +47,7 @@ class SessionListParams(TypedDict, total=False): """ Comma-separated list of fields to sort by, applied in order (the first field dominates); prefix a field with '-' for descending — e.g. - '-cost_total_usd,latency_ms'. Fields: test_case_id, started_at, ended_at, + '-cost_total_usd,latency_ms'. Fields: test_case_name, started_at, ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, sessions are ordered by started_at ascending. """ diff --git a/sdk/python/nemo-platform/tests/api_resources/evaluations/test_sessions.py b/sdk/python/nemo-platform/tests/api_resources/evaluations/test_sessions.py index ca09c4ad42..96fabd08ba 100644 --- a/sdk/python/nemo-platform/tests/api_resources/evaluations/test_sessions.py +++ b/sdk/python/nemo-platform/tests/api_resources/evaluations/test_sessions.py @@ -51,6 +51,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: filter={ "status": "status", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, mode="summary", page=1, @@ -126,6 +127,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform filter={ "status": "status", "test_case_id": "test_case_id", + "test_case_name": "test_case_name", }, mode="summary", page=1, diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index c823f7f556..d26dccf418 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -82,6 +82,7 @@ # expression in the ClickHouse repository. _SESSION_SORT_FIELDS = frozenset( { + "test_case_name", "test_case_id", "started_at", "ended_at", @@ -746,7 +747,7 @@ async def unpin_evaluation( }, openapi_extra=generate_openapi_extra_params( filter_schema=EvaluationSessionFilter, - filter_description="Filter sessions by test_case_id and status.", + filter_description="Filter sessions by test_case_name and status.", ), ) async def list_evaluation_sessions( @@ -769,21 +770,21 @@ async def list_evaluation_sessions( description=( "Comma-separated list of fields to sort by, applied in order (the first field dominates); " "prefix a field with '-' for descending — e.g. '-cost_total_usd,latency_ms'. Fields: " - "test_case_id, started_at, ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, " + "test_case_name, started_at, ended_at, latency_ms, status, cost_total_usd, tokens. When omitted, " "sessions are ordered by started_at ascending." ), ), ) -> Page[EvaluationSessionResponse]: validate_list_query_params(request, additional_params={"mode"}) sort_keys = _parse_session_sort_keys(sort) if sort is not None else None - test_case_id: str | None = parsed.extract("test_case_id") + test_case_name = _session_test_case_name(parsed) status_raw: str | None = parsed.extract("status") try: result = await read_service.list_sessions( workspace=workspace, evaluation_name=name, status=status_raw, - test_case_id=test_case_id, + test_case_name=test_case_name, page=page, page_size=page_size, mode=mode, @@ -803,7 +804,7 @@ async def list_evaluation_sessions( f"This query selects {exc.total} sessions, exceeding the maximum of " f"{exc.limit} that can be sorted by cost or tokens in one request. " "Narrow the result with a filter (e.g. filter[status]=success) or sort by a " - "different field (started_at, latency_ms, status, test_case_id)." + "different field (started_at, latency_ms, status, test_case_name)." ), ) from exc except EvaluationTelemetryUnavailableError as exc: @@ -1300,6 +1301,8 @@ def _parse_session_sort_keys(sort: str) -> list[tuple[str, bool]]: sort_field = field_token[1:] if descending else field_token if sort_field not in _SESSION_SORT_FIELDS: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Unsupported sort field: {sort_field}") + if sort_field == "test_case_id": + sort_field = "test_case_name" sort_keys.append((sort_field, descending)) if not sort_keys: raise HTTPException( @@ -1309,6 +1312,17 @@ def _parse_session_sort_keys(sort: str) -> list[tuple[str, bool]]: return sort_keys +def _session_test_case_name(parsed: ParsedFilter) -> str | None: + test_case_name: str | None = parsed.extract("test_case_name") + deprecated_test_case_id: str | None = parsed.extract("test_case_id") + if test_case_name is not None and deprecated_test_case_id is not None and test_case_name != deprecated_test_case_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Conflicting evaluation session filters for test_case_name", + ) + return test_case_name if test_case_name is not None else deprecated_test_case_id + + def _is_metric_field(field: str) -> bool: """True if `field` is *intended* as a rollup metric (by head), valid path or not. diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 5e1a6afaaa..18b395d40c 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -285,8 +285,8 @@ class EvaluationResponse(BaseModel): test_case_count: int = Field( default=0, description=( - "Number of distinct test cases in the evaluation, i.e. distinct test_case_id values " - "(sessions with no test_case_id each count as their own). A test case run k times counts once; " + "Number of distinct test cases in the evaluation, i.e. distinct test_case_name values " + "(sessions with no test_case_name each count as their own). A test case run k times counts once; " "the rollup metrics are averaged per test case before pooling across test cases." ), ) @@ -458,7 +458,12 @@ class EvaluationFilter(Filter): class EvaluationSessionFilter(Filter): """Filter for listing EvaluationSessions.""" - test_case_id: str | None = Field(default=None, description="Filter by producer-supplied test case id.") + test_case_name: str | None = Field(default=None, description="Filter by test case name.") + test_case_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for test_case_name. Use test_case_name instead.", + ) status: str | None = Field( default=None, description="Filter by root-span status (success, error, cancelled, unknown)." ) @@ -474,9 +479,14 @@ class EvaluationSessionResponse(BaseModel): workspace: str evaluation_name: str session_id: str + test_case_name: str | None = Field( + default=None, + description="Test case name; null when the producer did not set one.", + ) test_case_id: str | None = Field( default=None, - description="Producer-supplied test case identifier; null when the producer did not set one.", + deprecated=True, + description="Deprecated alias for test_case_name. Use test_case_name instead.", ) trace_id: str root_span_id: str @@ -525,7 +535,8 @@ def from_row( workspace=row.workspace, evaluation_name=row.evaluation_name, session_id=row.session_id, - test_case_id=row.test_case_id, + test_case_name=row.test_case_name, + test_case_id=row.test_case_name, trace_id=row.trace_id, root_span_id=row.root_span_id, started_at=row.started_at, diff --git a/services/intake/src/nmp/intake/experiments/denormalizer.py b/services/intake/src/nmp/intake/experiments/denormalizer.py index 2751727f04..95687a3a53 100644 --- a/services/intake/src/nmp/intake/experiments/denormalizer.py +++ b/services/intake/src/nmp/intake/experiments/denormalizer.py @@ -111,7 +111,7 @@ async def flush(self) -> None: self.mark_dirty(workspace=workspace, evaluation_name=evaluation_name) async def _refresh_workspace(self, workspace: str, evaluation_names: list[str]) -> None: - rollups = await self._rollup_repository.get_rollups(workspace=workspace, evaluation_ids=evaluation_names) + rollups = await self._rollup_repository.get_rollups(workspace=workspace, evaluation_names=evaluation_names) for evaluation_name in evaluation_names: rollup = rollups.get(evaluation_name) if rollup is None: diff --git a/services/intake/src/nmp/intake/experiments/read_service.py b/services/intake/src/nmp/intake/experiments/read_service.py index 98a26d410e..53201de34b 100644 --- a/services/intake/src/nmp/intake/experiments/read_service.py +++ b/services/intake/src/nmp/intake/experiments/read_service.py @@ -133,7 +133,7 @@ async def list_sessions( workspace: str, evaluation_name: str, status: str | None, - test_case_id: str | None, + test_case_name: str | None, page: int, page_size: int, mode: IntakeResponseMode, @@ -153,7 +153,7 @@ async def list_sessions( workspace=workspace, evaluation_name=evaluation_name, status=status_filter, - test_case_id=test_case_id, + test_case_name=test_case_name, page=page, page_size=page_size, mode=mode, @@ -190,7 +190,7 @@ async def _get_rollups( return ( await self._rollups.get_rollups( workspace=workspace, - evaluation_ids=evaluation_names, + evaluation_names=evaluation_names, ), True, ) diff --git a/services/intake/src/nmp/intake/repository/clickhouse/evaluation_rollup.py b/services/intake/src/nmp/intake/repository/clickhouse/evaluation_rollup.py index 032cb0d509..263e70b006 100644 --- a/services/intake/src/nmp/intake/repository/clickhouse/evaluation_rollup.py +++ b/services/intake/src/nmp/intake/repository/clickhouse/evaluation_rollup.py @@ -18,13 +18,15 @@ class ClickHouseEvaluationRollupRepository(EvaluationRollupRepository): def __init__(self, executor: ClickHouseExecutor) -> None: self._executor = executor - async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dict[str, EvaluationRollup]: - evaluation_ids = list(dict.fromkeys(evaluation_ids)) - rollups = {evaluation_id: EvaluationRollup(evaluation_id=evaluation_id) for evaluation_id in evaluation_ids} - if not evaluation_ids: + async def get_rollups(self, *, workspace: str, evaluation_names: list[str]) -> dict[str, EvaluationRollup]: + evaluation_names = list(dict.fromkeys(evaluation_names)) + rollups = { + evaluation_name: EvaluationRollup(evaluation_name=evaluation_name) for evaluation_name in evaluation_names + } + if not evaluation_names: return rollups - evaluation_names_sql, evaluation_parameters = _evaluation_id_parameters(evaluation_ids) + evaluation_names_sql, evaluation_parameters = _evaluation_name_parameters(evaluation_names) parameters = {"workspace": workspace, **evaluation_parameters} trace_index_table = self._executor.table(ClickHouseTable.TRACE_INDEX) @@ -35,8 +37,8 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic parameters=parameters, ) ): - rollups[row["evaluation_id"]].run_count = int(row["run_count"]) - rollups[row["evaluation_id"]].test_case_count = int(row["test_case_count"]) + rollups[row["evaluation_name"]].run_count = int(row["run_count"]) + rollups[row["evaluation_name"]].test_case_count = int(row["test_case_count"]) for row in await self._executor.fetch_all( ClickHouseQuery( @@ -49,7 +51,7 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic parameters=parameters, ) ): - rollups[row["evaluation_id"]].evaluator_scores[row["evaluator_name"]] = ScoreRollup( + rollups[row["evaluation_name"]].evaluator_scores[row["evaluator_name"]] = ScoreRollup( sum=float_or_none(row["sum"]), mean=float_or_none(row["mean"]), median=float_or_none(row["median"]), @@ -78,7 +80,7 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic }, ) ): - rollup = rollups[row["evaluation_id"]] + rollup = rollups[row["evaluation_name"]] rollup.model_names = _string_list(row["model_names"]) rollup.agent_names = _string_list(row["agent_names"]) rollup.agent_versions = _string_list(row["agent_versions"]) @@ -89,38 +91,43 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic return rollups -def _evaluation_id_parameters(evaluation_ids: list[str]) -> tuple[str, dict[str, str]]: - parameters = {f"evaluation_id_{index}": evaluation_id for index, evaluation_id in enumerate(evaluation_ids)} +def _evaluation_name_parameters(evaluation_names: list[str]) -> tuple[str, dict[str, str]]: + parameters = {f"evaluation_name_{index}": evaluation_name for index, evaluation_name in enumerate(evaluation_names)} return ", ".join(f"%({name})s" for name in parameters), parameters def _scoped_sessions_sql(trace_index_table: str, evaluation_names_sql: str) -> str: return f""" - SELECT workspace, evaluation_id, session_id, test_case_id, latency_ms + SELECT + workspace, + evaluation_name, + session_id, + test_case_name, + latency_ms FROM {trace_index_table} FINAL WHERE workspace = %(workspace)s AND is_deleted = 0 - AND evaluation_id IN ({evaluation_names_sql}) + AND evaluation_name IN ({evaluation_names_sql}) ORDER BY root_started_at ASC, root_span_id ASC - LIMIT 1 BY workspace, session_id, evaluation_id + LIMIT 1 BY workspace, session_id, evaluation_name """ def _run_counts_sql(trace_index_table: str, evaluation_names_sql: str) -> str: # run_count is every ingested session; test_case_count is the distinct test cases those sessions - # belong to. Sessions with no test_case_id aren't attributable to a test case, so they don't count + # belong to. Sessions with no test_case_name aren't attributable to a test case, so they don't count # toward test_case_count (and are excluded from the test-case-weighted rollups below). return f""" WITH scoped_sessions AS ( {_scoped_sessions_sql(trace_index_table, evaluation_names_sql)} ) SELECT - evaluation_id, + evaluation_name, count() AS run_count, - uniqExactIf(test_case_id, test_case_id != '') AS test_case_count + uniqExactIf(test_case_name, test_case_name != '') AS test_case_count FROM scoped_sessions - GROUP BY evaluation_id - ORDER BY evaluation_id ASC + GROUP BY evaluation_name + ORDER BY evaluation_name ASC """ @@ -172,7 +179,7 @@ def _score_rollups_sql(*, trace_index_table: str, evaluator_results_table: str, evaluators — the evaluator axis of the per-test-case grid test_case_scores — stage 2: one value per (test case, evaluator), zero-filled The final SELECT takes the distribution (sum/mean/quantiles/count) across test cases. Sessions with - no test_case_id can't be attributed to a test case and are dropped. + no test_case_name can't be attributed to a test case and are dropped. """ return f""" WITH @@ -192,12 +199,12 @@ def _score_rollups_sql(*, trace_index_table: str, evaluator_results_table: str, {_test_case_scores_cte()} ) SELECT - evaluation_id, + evaluation_name, evaluator_name, {_stat_columns("value")} FROM test_case_scores - GROUP BY evaluation_id, evaluator_name - ORDER BY evaluation_id ASC, evaluator_name ASC + GROUP BY evaluation_name, evaluator_name + ORDER BY evaluation_name ASC, evaluator_name ASC """ @@ -206,7 +213,7 @@ def _sessions_join_scored_results(evaluator_results_table: str, *, columns: str) ``columns`` is the projection taken from evaluator_results ("name, value" or "name"). The inner subquery pre-filters to scoped sessions so ClickHouse prunes evaluator_results before the join, and - the trailing WHERE keeps only sessions that carry a test_case_id — the ones the rollup is over. + the trailing WHERE keeps only sessions that carry a test_case_name — the ones the rollup is over. """ return f"""FROM scoped_sessions AS sessions INNER JOIN ( @@ -222,7 +229,7 @@ def _sessions_join_scored_results(evaluator_results_table: str, *, columns: str) ) AS results ON sessions.workspace = results.workspace AND sessions.session_id = results.session_id - WHERE sessions.test_case_id != ''""" + WHERE sessions.test_case_name != ''""" def _session_scores_cte(evaluator_results_table: str) -> str: @@ -233,12 +240,12 @@ def _session_scores_cte(evaluator_results_table: str) -> str: """ return f""" SELECT - sessions.evaluation_id AS evaluation_id, - sessions.test_case_id AS test_case_key, + sessions.evaluation_name AS evaluation_name, + sessions.test_case_name AS test_case_key, results.name AS evaluator_name, avg(results.value) AS value {_sessions_join_scored_results(evaluator_results_table, columns="name, value")} - GROUP BY sessions.evaluation_id, sessions.session_id, sessions.test_case_id, results.name""" + GROUP BY sessions.evaluation_name, sessions.session_id, sessions.test_case_name, results.name""" def _test_case_sessions_cte() -> str: @@ -249,12 +256,12 @@ def _test_case_sessions_cte() -> str: """ return """ SELECT - evaluation_id, - test_case_id AS test_case_key, + evaluation_name, + test_case_name AS test_case_key, count(DISTINCT session_id) AS session_count FROM scoped_sessions - WHERE test_case_id != '' - GROUP BY evaluation_id, test_case_id""" + WHERE test_case_name != '' + GROUP BY evaluation_name, test_case_name""" def _evaluators_cte(evaluator_results_table: str) -> str: @@ -266,7 +273,7 @@ def _evaluators_cte(evaluator_results_table: str) -> str: """ return f""" SELECT DISTINCT - sessions.evaluation_id AS evaluation_id, + sessions.evaluation_name AS evaluation_name, results.name AS evaluator_name {_sessions_join_scored_results(evaluator_results_table, columns="name")}""" @@ -280,18 +287,18 @@ def _test_case_scores_cte() -> str: """ return """ SELECT - test_cases.evaluation_id AS evaluation_id, + test_cases.evaluation_name AS evaluation_name, test_cases.test_case_key AS test_case_key, evaluators.evaluator_name AS evaluator_name, coalesce(sum(scores.value), 0) / test_cases.session_count AS value FROM test_case_sessions AS test_cases - INNER JOIN evaluators ON evaluators.evaluation_id = test_cases.evaluation_id + INNER JOIN evaluators ON evaluators.evaluation_name = test_cases.evaluation_name LEFT JOIN session_scores AS scores - ON scores.evaluation_id = test_cases.evaluation_id + ON scores.evaluation_name = test_cases.evaluation_name AND scores.test_case_key = test_cases.test_case_key AND scores.evaluator_name = evaluators.evaluator_name GROUP BY - test_cases.evaluation_id, test_cases.test_case_key, evaluators.evaluator_name, test_cases.session_count""" + test_cases.evaluation_name, test_cases.test_case_key, evaluators.evaluator_name, test_cases.session_count""" def _current_session_span_metrics_sql(spans_table: str) -> str: @@ -332,7 +339,7 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_ # Two-level rollup: per-attempt cost/latency, then averaged per test case (avg per attempt — the # number must not scale with k), then the distribution across test cases (test-case-weighted). # Attempts with no cost/latency are excluded from a test case's average rather than counted as zero; - # sessions with no test_case_id aren't attributable to a test case, so they're dropped. + # sessions with no test_case_name aren't attributable to a test case, so they're dropped. return f""" WITH scoped_sessions AS ( @@ -341,8 +348,8 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_ current_session_spans AS {_current_session_span_metrics_sql(spans_table)}, session_costs AS ( SELECT - sessions.evaluation_id AS evaluation_id, - sessions.test_case_id AS test_case_key, + sessions.evaluation_name AS evaluation_name, + sessions.test_case_name AS test_case_key, sessions.latency_ms AS latency_ms, if( countIf(spans.cost_present) = 0, @@ -363,12 +370,12 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_ ON sessions.workspace = spans.workspace AND sessions.session_id = spans.dedup_session_id AND spans.del_flag = 0 - WHERE sessions.test_case_id != '' - GROUP BY sessions.evaluation_id, sessions.session_id, sessions.test_case_id, sessions.latency_ms + WHERE sessions.test_case_name != '' + GROUP BY sessions.evaluation_name, sessions.session_id, sessions.test_case_name, sessions.latency_ms ), test_case_metrics AS ( SELECT - evaluation_id, + evaluation_name, test_case_key, if(countIf(isNotNull(cost_usd)) = 0, NULL, avgIf(cost_usd, isNotNull(cost_usd))) AS cost_usd, if(countIf(isNotNull(latency_ms)) = 0, NULL, avgIf(latency_ms, isNotNull(latency_ms))) AS latency_ms, @@ -377,10 +384,10 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_ arrayDistinct(arrayFlatten(groupArray(agent_names))) AS agent_names, arrayDistinct(arrayFlatten(groupArray(agent_versions))) AS agent_versions FROM session_costs - GROUP BY evaluation_id, test_case_key + GROUP BY evaluation_name, test_case_key ) SELECT - evaluation_id, + evaluation_name, arraySort(arrayDistinct(arrayFlatten(groupArray(model_names)))) AS model_names, arraySort(arrayDistinct(arrayFlatten(groupArray(agent_names)))) AS agent_names, arraySort(arrayDistinct(arrayFlatten(groupArray(agent_versions)))) AS agent_versions, @@ -388,8 +395,8 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_ {_stat_columns("latency_ms", prefix="latency", guarded=True)}, {_stat_columns("tokens", prefix="tokens", guarded=True)} FROM test_case_metrics - GROUP BY evaluation_id - ORDER BY evaluation_id ASC + GROUP BY evaluation_name + ORDER BY evaluation_name ASC """ diff --git a/services/intake/src/nmp/intake/repository/clickhouse/evaluation_session.py b/services/intake/src/nmp/intake/repository/clickhouse/evaluation_session.py index 0d3d0419fd..a3938b0616 100644 --- a/services/intake/src/nmp/intake/repository/clickhouse/evaluation_session.py +++ b/services/intake/src/nmp/intake/repository/clickhouse/evaluation_session.py @@ -46,7 +46,7 @@ "ended_at": "end_time", "latency_ms": "latency_ms", "status": "root_span_status", - "test_case_id": "test_case_id", + "test_case_name": "test_case_name", "cost_total_usd": "pm.cost_total_usd", "tokens": "pm.total_tokens", } @@ -92,7 +92,7 @@ async def list_sessions( workspace: str, evaluation_name: str, status: SpanStatus | None = None, - test_case_id: str | None = None, + test_case_name: str | None = None, page: int, page_size: int, mode: IntakeResponseMode, @@ -102,7 +102,7 @@ async def list_sessions( spans_table = self._executor.table(ClickHouseTable.SPANS) evaluator_results_table = self._executor.table(ClickHouseTable.EVALUATOR_RESULTS) - scoped_filter_sql, scoped_filter_parameters = _scoped_filter(test_case_id=test_case_id, status=status) + scoped_filter_sql, scoped_filter_parameters = _scoped_filter(test_case_name=test_case_name, status=status) base_parameters: dict[str, Any] = { "workspace": workspace, @@ -190,12 +190,12 @@ async def list_sessions( return EvaluationSessionPage(rows=rows, total=total) -def _scoped_filter(*, test_case_id: str | None, status: SpanStatus | None) -> tuple[str, dict[str, Any]]: +def _scoped_filter(*, test_case_name: str | None, status: SpanStatus | None) -> tuple[str, dict[str, Any]]: clauses: list[str] = [] parameters: dict[str, Any] = {} - if test_case_id is not None: - parameters["test_case_id"] = test_case_id - clauses.append("test_case_id = %(test_case_id)s") + if test_case_name is not None: + parameters["test_case_name"] = test_case_name + clauses.append("test_case_name = %(test_case_name)s") if status is not None: parameters["status"] = status.value clauses.append("root_status = %(status)s") @@ -210,9 +210,9 @@ def _scoped_sessions_sql( ) -> str: select_columns = [ "workspace", - "evaluation_id", + "evaluation_name", "session_id", - "test_case_id", + "test_case_name", "trace_id", "root_span_id", "root_started_at AS start_time", @@ -233,10 +233,10 @@ def _scoped_sessions_sql( FROM {trace_index_table} FINAL WHERE workspace = %(workspace)s AND is_deleted = 0 - AND evaluation_id = %(evaluation_name)s + AND evaluation_name = %(evaluation_name)s {scoped_filter_sql} ORDER BY root_started_at ASC, root_span_id ASC - LIMIT 1 BY workspace, session_id, evaluation_id + LIMIT 1 BY workspace, session_id, evaluation_name """ @@ -381,9 +381,9 @@ def _hydrate_by_refs_sql( """Hydrate payloads, span metrics, and evaluator scores for one selected page.""" select_columns = [ "workspace", - "evaluation_id", + "evaluation_name", "session_id", - "test_case_id", + "test_case_name", "trace_id", "root_span_id", "root_started_at AS start_time", @@ -412,7 +412,7 @@ def _hydrate_by_refs_sql( FROM {trace_index_table} FINAL WHERE workspace = %(workspace)s AND is_deleted = 0 - AND evaluation_id = %(evaluation_name)s + AND evaluation_name = %(evaluation_name)s -- The flat predicates engage the bloom indexes; the tuple preserves -- exact trace_index identity. The time range engages the primary key. AND session_id IN %(page_session_ids)s @@ -424,7 +424,7 @@ def _hydrate_by_refs_sql( trace_id, root_span_id ) IN %(page_storage_keys)s - LIMIT 1 BY workspace, session_id, evaluation_id + LIMIT 1 BY workspace, session_id, evaluation_name ), current_page_spans AS ( {page_spans} @@ -466,9 +466,9 @@ def _hydrate_by_refs_sql( ) SELECT sessions.workspace AS workspace, - sessions.evaluation_id AS evaluation_id, + sessions.evaluation_name, sessions.session_id AS session_id, - sessions.test_case_id AS test_case_id, + sessions.test_case_name, sessions.trace_id AS trace_id, sessions.root_span_id AS root_span_id, sessions.start_time AS start_time, @@ -520,9 +520,9 @@ def _guarded_sum_sql(parameter_name: str, *, scale: int = 1) -> str: def _row(record: dict[str, Any]) -> EvaluationSessionRow: return EvaluationSessionRow( workspace=record["workspace"], - evaluation_name=record["evaluation_id"], + evaluation_name=record["evaluation_name"], session_id=record["session_id"], - test_case_id=str_or_none(record["test_case_id"]), + test_case_name=str_or_none(record["test_case_name"]), trace_id=record["trace_id"], root_span_id=record["root_span_id"], started_at=record["start_time"], diff --git a/services/intake/src/nmp/intake/repository/evaluation_rollup.py b/services/intake/src/nmp/intake/repository/evaluation_rollup.py index 9ade2fc20e..2606b89f2b 100644 --- a/services/intake/src/nmp/intake/repository/evaluation_rollup.py +++ b/services/intake/src/nmp/intake/repository/evaluation_rollup.py @@ -20,7 +20,7 @@ class ScoreRollup: @dataclass class EvaluationRollup: - evaluation_id: str + evaluation_name: str run_count: int = 0 test_case_count: int = 0 model_names: list[str] = field(default_factory=list) @@ -44,7 +44,7 @@ async def get_rollups( self, *, workspace: str, - evaluation_ids: list[str], + evaluation_names: list[str], ) -> dict[str, EvaluationRollup]: - """Return rollups keyed by Evaluation ID.""" + """Return rollups keyed by Evaluation name.""" pass diff --git a/services/intake/src/nmp/intake/repository/evaluation_session.py b/services/intake/src/nmp/intake/repository/evaluation_session.py index 019dccae14..f5c95a18d6 100644 --- a/services/intake/src/nmp/intake/repository/evaluation_session.py +++ b/services/intake/src/nmp/intake/repository/evaluation_session.py @@ -26,7 +26,7 @@ class EvaluationSessionRow: workspace: str evaluation_name: str session_id: str - test_case_id: str | None + test_case_name: str | None trace_id: str root_span_id: str started_at: datetime @@ -58,7 +58,7 @@ async def list_sessions( workspace: str, evaluation_name: str, status: SpanStatus | None = None, - test_case_id: str | None = None, + test_case_name: str | None = None, page: int, page_size: int, mode: IntakeResponseMode, diff --git a/services/intake/tests/integration/spans/test_experiment_sessions.py b/services/intake/tests/integration/spans/test_experiment_sessions.py index b85b663194..cd0d48f8ee 100644 --- a/services/intake/tests/integration/spans/test_experiment_sessions.py +++ b/services/intake/tests/integration/spans/test_experiment_sessions.py @@ -77,8 +77,9 @@ def test_list_evaluation_sessions_returns_joined_session_rows(client: TestClient assert body["pagination"]["total_results"] == 3 assert len(body["data"]) == 3 - rows_by_case = {row["test_case_id"]: row for row in body["data"]} + rows_by_case = {row["test_case_name"]: row for row in body["data"]} assert set(rows_by_case) == {"case-a", "case-b", "case-c"} + assert all(row["test_case_id"] == row["test_case_name"] for row in body["data"]) case_a = rows_by_case["case-a"] assert case_a["evaluation_name"] == evaluation_name @@ -105,7 +106,7 @@ def test_list_evaluation_sessions_returns_joined_session_rows(client: TestClient params={"mode": "preview", "page_size": 3}, ) assert preview.status_code == 200, preview.text - preview_by_case = {row["test_case_id"]: row for row in preview.json()["data"]} + preview_by_case = {row["test_case_name"]: row for row in preview.json()["data"]} assert preview_by_case["case-a"]["output"] == "solved case-a" paged = client.get(f"{EVALUATIONS}/{evaluation_name}/sessions", params={"page": 2, "page_size": 1}) @@ -113,7 +114,7 @@ def test_list_evaluation_sessions_returns_joined_session_rows(client: TestClient paged_body = paged.json() assert paged_body["pagination"]["total_results"] == 3 assert len(paged_body["data"]) == 1 - assert paged_body["data"][0]["test_case_id"] == "case-b" + assert paged_body["data"][0]["test_case_name"] == "case-b" assert paged_body["data"][0]["evaluator_scores"] == {"reward": pytest.approx(0.5)} latency_sorted = client.get( @@ -121,21 +122,35 @@ def test_list_evaluation_sessions_returns_joined_session_rows(client: TestClient params={"sort": "-latency_ms", "page_size": 3}, ) assert latency_sorted.status_code == 200, latency_sorted.text - assert [row["test_case_id"] for row in latency_sorted.json()["data"]] == ["case-c", "case-b", "case-a"] + assert [row["test_case_name"] for row in latency_sorted.json()["data"]] == ["case-c", "case-b", "case-a"] cost_sorted = client.get( f"{EVALUATIONS}/{evaluation_name}/sessions", params={"sort": "-cost_total_usd", "page_size": 3}, ) assert cost_sorted.status_code == 200, cost_sorted.text - assert [row["test_case_id"] for row in cost_sorted.json()["data"]] == ["case-c", "case-b", "case-a"] + assert [row["test_case_name"] for row in cost_sorted.json()["data"]] == ["case-c", "case-b", "case-a"] tokens_sorted = client.get( f"{EVALUATIONS}/{evaluation_name}/sessions", params={"sort": "tokens", "page_size": 3}, ) assert tokens_sorted.status_code == 200, tokens_sorted.text - assert [row["test_case_id"] for row in tokens_sorted.json()["data"]] == ["case-a", "case-b", "case-c"] + assert [row["test_case_name"] for row in tokens_sorted.json()["data"]] == ["case-a", "case-b", "case-c"] + + test_case_sorted = client.get( + f"{EVALUATIONS}/{evaluation_name}/sessions", + params={"sort": "-test_case_name", "page_size": 3}, + ) + assert test_case_sorted.status_code == 200, test_case_sorted.text + assert [row["test_case_name"] for row in test_case_sorted.json()["data"]] == ["case-c", "case-b", "case-a"] + + deprecated_sort = client.get( + f"{EVALUATIONS}/{evaluation_name}/sessions", + params={"sort": "-test_case_id", "page_size": 3}, + ) + assert deprecated_sort.status_code == 200, deprecated_sort.text + assert [row["test_case_name"] for row in deprecated_sort.json()["data"]] == ["case-c", "case-b", "case-a"] def test_list_evaluation_sessions_filter_by_test_case(client: TestClient) -> None: @@ -174,13 +189,30 @@ def test_list_evaluation_sessions_filter_by_test_case(client: TestClient) -> Non filtered = client.get( f"{EVALUATIONS}/{evaluation_name}/sessions", - params={"filter[test_case_id]": adversarial_test_case_id}, + params={"filter[test_case_name]": adversarial_test_case_id}, ) assert filtered.status_code == 200, filtered.text body = filtered.json() assert body["pagination"]["total_results"] == 1 assert len(body["data"]) == 1 - assert body["data"][0]["test_case_id"] == adversarial_test_case_id + assert body["data"][0]["test_case_name"] == adversarial_test_case_id + + deprecated_filter = client.get( + f"{EVALUATIONS}/{evaluation_name}/sessions", + params={"filter[test_case_id]": adversarial_test_case_id}, + ) + assert deprecated_filter.status_code == 200, deprecated_filter.text + assert deprecated_filter.json()["data"][0]["test_case_name"] == adversarial_test_case_id + + conflicting_filters = client.get( + f"{EVALUATIONS}/{evaluation_name}/sessions", + params={ + "filter[test_case_name]": adversarial_test_case_id, + "filter[test_case_id]": "beta", + }, + ) + assert conflicting_filters.status_code == 400, conflicting_filters.text + assert conflicting_filters.json()["detail"] == "Conflicting evaluation session filters for test_case_name" def test_list_evaluation_sessions_filter_by_status(client: TestClient) -> None: diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 69bf1cd064..2c68d5727c 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -377,7 +377,7 @@ def test_filter_experiments_by_baseline_evaluation_name(client: TestClient) -> N def test_evaluation_read_degrades_when_rollup_hydration_fails(client: TestClient) -> None: class FailingRollupRepository: - async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dict: + async def get_rollups(self, *, workspace: str, evaluation_names: list[str]) -> dict: raise RuntimeError("clickhouse unavailable") group = _create_group(client) diff --git a/services/intake/tests/test_evaluation_denormalizer.py b/services/intake/tests/test_evaluation_denormalizer.py index 54aa9dd028..9b51d08c45 100644 --- a/services/intake/tests/test_evaluation_denormalizer.py +++ b/services/intake/tests/test_evaluation_denormalizer.py @@ -15,9 +15,9 @@ from nmp.intake.repository.evaluation_rollup import EvaluationRollup, EvaluationRollupRepository -def _sample_rollup(evaluation_id: str) -> EvaluationRollup: +def _sample_rollup(evaluation_name: str) -> EvaluationRollup: return EvaluationRollup( - evaluation_id=evaluation_id, + evaluation_name=evaluation_name, model_names=["provider/model-a", "provider/model-b"], agent_names=["agent-x"], agent_versions=["1.0", "1.1"], @@ -29,11 +29,11 @@ def __init__(self, *, error: Exception | None = None) -> None: self.calls: list[tuple[str, list[str]]] = [] self._error = error - async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dict[str, EvaluationRollup]: - self.calls.append((workspace, list(evaluation_ids))) + async def get_rollups(self, *, workspace: str, evaluation_names: list[str]) -> dict[str, EvaluationRollup]: + self.calls.append((workspace, list(evaluation_names))) if self._error is not None: raise self._error - return {evaluation_id: _sample_rollup(evaluation_id) for evaluation_id in evaluation_ids} + return {evaluation_name: _sample_rollup(evaluation_name) for evaluation_name in evaluation_names} class _FakeEntityClient: diff --git a/services/intake/tests/test_evaluation_denormalizer_self_heal.py b/services/intake/tests/test_evaluation_denormalizer_self_heal.py index fe5a2d64db..8e554a8990 100644 --- a/services/intake/tests/test_evaluation_denormalizer_self_heal.py +++ b/services/intake/tests/test_evaluation_denormalizer_self_heal.py @@ -49,7 +49,7 @@ def _rollup( name: str, *, agent_names: list[str], agent_versions: list[str], model_names: list[str] ) -> EvaluationRollup: return EvaluationRollup( - evaluation_id=name, + evaluation_name=name, agent_names=agent_names, agent_versions=agent_versions, model_names=model_names, diff --git a/services/intake/tests/test_evaluation_read_service.py b/services/intake/tests/test_evaluation_read_service.py index f68917dda9..055d136bc8 100644 --- a/services/intake/tests/test_evaluation_read_service.py +++ b/services/intake/tests/test_evaluation_read_service.py @@ -30,9 +30,9 @@ async def get_rollups( self, *, workspace: str, - evaluation_ids: list[str], + evaluation_names: list[str], ) -> dict[str, EvaluationRollup]: - self.calls.append((workspace, evaluation_ids)) + self.calls.append((workspace, evaluation_names)) if isinstance(self._rollups, Exception): raise self._rollups return self._rollups @@ -48,7 +48,7 @@ async def list_sessions( workspace: str, evaluation_name: str, status: SpanStatus | None = None, - test_case_id: str | None = None, + test_case_name: str | None = None, page: int, page_size: int, mode: IntakeResponseMode, @@ -86,7 +86,7 @@ async def test_list_evaluations_batches_rollup_enrichment() -> None: data=[first, second], pagination=SimpleNamespace(total_results=2), ) - first_rollup = EvaluationRollup(evaluation_id=first.name, run_count=4) + first_rollup = EvaluationRollup(evaluation_name=first.name, run_count=4) rollups = _RollupRepository({first.name: first_rollup}) service = EvaluationReadService( entity_client=client, @@ -195,7 +195,7 @@ async def test_session_failures_are_translated_after_entity_validation() -> None workspace="default", evaluation_name="eval-1", status=None, - test_case_id=None, + test_case_name=None, page=1, page_size=100, mode="detailed", diff --git a/services/intake/tests/test_evaluation_session_clickhouse_repository.py b/services/intake/tests/test_evaluation_session_clickhouse_repository.py index 8103937e81..5da836b12a 100644 --- a/services/intake/tests/test_evaluation_session_clickhouse_repository.py +++ b/services/intake/tests/test_evaluation_session_clickhouse_repository.py @@ -42,9 +42,9 @@ def _session_record(session_id: str) -> dict[str, Any]: now = datetime(2026, 1, 1, tzinfo=timezone.utc) return { "workspace": "default", - "evaluation_id": "evaluation-a", + "evaluation_name": "evaluation-a", "session_id": session_id, - "test_case_id": "case-a", + "test_case_name": "case-a", "trace_id": f"trace-{session_id}", "root_span_id": f"root-{session_id}", "start_time": now, @@ -81,7 +81,7 @@ async def test_list_sessions_maps_rows_and_binds_all_request_values() -> None: workspace=workspace, evaluation_name=evaluation_name, status=SpanStatus.ERROR, - test_case_id=test_case_id, + test_case_name=test_case_id, page=2, page_size=5, mode="preview", @@ -107,9 +107,12 @@ async def test_list_sessions_maps_rows_and_binds_all_request_values() -> None: assert test_case_id not in query.statement assert query.parameters["workspace"] == workspace assert query.parameters["evaluation_name"] == evaluation_name + assert "evaluation_name = %(evaluation_name)s" in query.statement + assert "evaluation_id = %(evaluation_name)s" not in query.statement for query in executor.queries[:2]: - assert query.parameters["test_case_id"] == test_case_id + assert query.parameters["test_case_name"] == test_case_id assert query.parameters["status"] == "error" + assert "test_case_name = %(test_case_name)s" in query.statement assert executor.queries[1].parameters["limit"] == 5 assert executor.queries[1].parameters["offset"] == 5 assert "'' AS input" in executor.queries[1].statement @@ -188,6 +191,7 @@ async def test_metric_sort_rejects_unbounded_session_set_after_count() -> None: def test_build_order_by_uses_registered_expressions_in_key_order() -> None: + assert _SORT_EXPR_PAGE["test_case_name"] == "test_case_name" assert ( _build_order_by( [("cost_total_usd", True), ("latency_ms", False)], diff --git a/services/intake/tests/test_experiment_rollup_repository.py b/services/intake/tests/test_experiment_rollup_repository.py index 2e9496c525..0dd5a2c30b 100644 --- a/services/intake/tests/test_experiment_rollup_repository.py +++ b/services/intake/tests/test_experiment_rollup_repository.py @@ -30,10 +30,10 @@ def _repository(executor: _Executor) -> ClickHouseEvaluationRollupRepository: async def test_evaluation_rollups_anchor_on_root_session_membership(): executor = _Executor( [ - [{"evaluation_id": "exp-a", "run_count": 3, "test_case_count": 2}], + [{"evaluation_name": "exp-a", "run_count": 3, "test_case_count": 2}], [ { - "evaluation_id": "exp-a", + "evaluation_name": "exp-a", "evaluator_name": "reward", "sum": 3.0, "mean": 0.75, @@ -46,7 +46,7 @@ async def test_evaluation_rollups_anchor_on_root_session_membership(): ], [ { - "evaluation_id": "exp-a", + "evaluation_name": "exp-a", "model_names": ["model-b", "model-a"], "agent_names": ["agent-a"], "agent_versions": ["1.0.0", "1.0.1"], @@ -77,7 +77,7 @@ async def test_evaluation_rollups_anchor_on_root_session_membership(): ) repository = _repository(executor) - rollups = await repository.get_rollups(workspace="default", evaluation_ids=["exp-a"]) + rollups = await repository.get_rollups(workspace="default", evaluation_names=["exp-a"]) rollup = rollups["exp-a"] assert rollup.run_count == 3 @@ -125,10 +125,13 @@ async def test_evaluation_rollups_anchor_on_root_session_membership(): ] statements = [query.statement for query in executor.queries] assert "FROM trace_index FINAL" in statements[0] + assert "evaluation_name" in statements[0] + assert "test_case_name" in statements[0] assert "count() AS run_count" in statements[0] - assert "uniqExactIf(test_case_id, test_case_id != '')" in statements[0] + assert "uniqExactIf(test_case_name, test_case_name != '')" in statements[0] assert "AS test_case_count" in statements[0] - assert "evaluation_id IN (%(evaluation_id_0)s)" in statements[0] + assert "evaluation_name IN (%(evaluation_name_0)s)" in statements[0] + assert "LIMIT 1 BY workspace, session_id, evaluation_name" in statements[0] assert "ORDER BY root_started_at ASC, root_span_id ASC" in statements[0] assert "FROM evaluator_results FINAL" in statements[1] assert "quantileExact(0.5)(value) AS median" in statements[1] @@ -137,9 +140,11 @@ async def test_evaluation_rollups_anchor_on_root_session_membership(): assert "sessions.session_id = results.session_id" in statements[1] # Scores are reduced to one value per (session, evaluator), then averaged per test case before the # distribution rollup, so the mean is test-case-weighted and count tracks test cases. - assert "GROUP BY sessions.evaluation_id, sessions.session_id, sessions.test_case_id, results.name" in statements[1] + assert ( + "GROUP BY sessions.evaluation_name, sessions.session_id, sessions.test_case_name, results.name" in statements[1] + ) assert "test_case_scores AS" in statements[1] - assert "WHERE sessions.test_case_id != ''" in statements[1] + assert "WHERE sessions.test_case_name != ''" in statements[1] assert "test_case_metrics AS" in statements[2] assert "current_session_spans AS" in statements[2] assert "(workspace, session_id) IN (SELECT DISTINCT workspace, session_id FROM scoped_sessions)" in statements[2] @@ -152,7 +157,7 @@ async def test_evaluation_rollups_anchor_on_root_session_membership(): assert "latency_p99" in statements[2] assert "tokens_p99" in statements[2] assert "sessions.trace_id = spans.trace_id" not in statements[2] - assert executor.queries[0].parameters["evaluation_id_0"] == "exp-a" + assert executor.queries[0].parameters["evaluation_name_0"] == "exp-a" assert executor.queries[2].parameters["model_key"] == "gen_ai.request.model" assert "input_tokens_key" in executor.queries[2].parameters assert "output_tokens_key" in executor.queries[2].parameters @@ -170,7 +175,10 @@ def test_score_rollup_cte_builders_compose_the_pipeline(): session_scores = _session_scores_cte("evaluator_results") assert "avg(results.value) AS value" in session_scores assert "FROM evaluator_results FINAL" in session_scores - assert "GROUP BY sessions.evaluation_id, sessions.session_id, sessions.test_case_id, results.name" in session_scores + assert ( + "GROUP BY sessions.evaluation_name, sessions.session_id, sessions.test_case_name, results.name" + in session_scores + ) # Fixed denominator: distinct sessions per test case. assert "count(DISTINCT session_id) AS session_count" in _test_case_sessions_cte() @@ -185,4 +193,4 @@ def test_score_rollup_cte_builders_compose_the_pipeline(): test_case_scores = _test_case_scores_cte() assert "coalesce(sum(scores.value), 0) / test_cases.session_count AS value" in test_case_scores assert "LEFT JOIN session_scores AS scores" in test_case_scores - assert "INNER JOIN evaluators ON evaluators.evaluation_id = test_cases.evaluation_id" in test_case_scores + assert "INNER JOIN evaluators ON evaluators.evaluation_name = test_cases.evaluation_name" in test_case_scores diff --git a/services/intake/tests/test_experiment_session_schemas.py b/services/intake/tests/test_experiment_session_schemas.py index dc3eaa4178..98222abcf1 100644 --- a/services/intake/tests/test_experiment_session_schemas.py +++ b/services/intake/tests/test_experiment_session_schemas.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone -from nmp.intake.api.v2.experiments.schemas import EvaluationSessionResponse +from nmp.intake.api.v2.experiments.schemas import EvaluationSessionFilter, EvaluationSessionResponse from nmp.intake.repository.evaluation_session import EvaluationSessionRow from nmp.intake.spans.domain import SpanStatus @@ -17,6 +17,22 @@ def test_evaluation_session_from_row_preserves_detailed_payloads() -> None: assert response.input == input_text assert response.output == output_text + assert response.test_case_name == "case" + assert response.model_dump()["test_case_id"] == "case" + + +def test_evaluation_session_schema_deprecates_only_test_case_id() -> None: + properties = EvaluationSessionResponse.model_json_schema()["properties"] + + assert properties["test_case_name"].get("deprecated") is not True + assert properties["test_case_id"]["deprecated"] is True + + +def test_evaluation_session_filter_deprecates_only_test_case_id() -> None: + properties = EvaluationSessionFilter.model_json_schema()["properties"] + + assert properties["test_case_name"].get("deprecated") is not True + assert properties["test_case_id"]["deprecated"] is True def test_evaluation_session_from_row_applies_payload_modes() -> None: @@ -37,7 +53,7 @@ def _session_row(input_text: str, output_text: str) -> EvaluationSessionRow: workspace="default", evaluation_name="evaluation", session_id="session", - test_case_id="case", + test_case_name="case", trace_id="trace", root_span_id="root", started_at=now, diff --git a/services/intake/tests/test_experiment_sort.py b/services/intake/tests/test_experiment_sort.py index 62c9552a35..6da54225b4 100644 --- a/services/intake/tests/test_experiment_sort.py +++ b/services/intake/tests/test_experiment_sort.py @@ -7,7 +7,12 @@ import pytest from fastapi import HTTPException -from nmp.intake.api.v2.experiments.endpoints import _parse_sort_keys, _sort_evaluations, _validate_sort_field +from nmp.intake.api.v2.experiments.endpoints import ( + _parse_session_sort_keys, + _parse_sort_keys, + _sort_evaluations, + _validate_sort_field, +) from nmp.intake.api.v2.experiments.schemas import EvaluationResponse, EvaluatorAggregate @@ -140,6 +145,11 @@ def test_parse_sort_keys_rejects_empty() -> None: assert exc.value.status_code == 400 +def test_parse_session_sort_uses_test_case_name_and_accepts_deprecated_alias() -> None: + assert _parse_session_sort_keys("-test_case_name") == [("test_case_name", True)] + assert _parse_session_sort_keys("-test_case_id") == [("test_case_name", True)] + + def test_multi_field_sort_ranks_by_first_key_then_tiebreak() -> None: # Switchyard's default ranking: reward desc, then cost asc as the tiebreak. rows = [ diff --git a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx index b64d6e6739..86f7d3b28a 100644 --- a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx +++ b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx @@ -30,7 +30,7 @@ const mockSession = { root_span_id: 'span-root-1', started_at: '2025-01-01T00:00:00Z', status: 'success', - test_case_id: 'case-1', + test_case_name: 'case-1', input: 'Session input', output: 'Session output', }; @@ -142,7 +142,7 @@ describe('EvaluationSessionsDataView', () => { http.get(SESSIONS_URL, () => HttpResponse.json({ ...mockSessionsPage, - data: [{ ...mockSession, trace_id: '', test_case_id: 'no-trace-case' }], + data: [{ ...mockSession, trace_id: '', test_case_name: 'no-trace-case' }], }) ) ); diff --git a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx index 60c382adc4..e2d7fe93e1 100644 --- a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx @@ -74,7 +74,7 @@ const listEvaluationSessionsWithModeFallback = async ( // Column id → API sort field. All session sort fields are direct 1:1 matches so no // translation is needed beyond listing the sortable ids. const SESSION_SORT_FIELD_MAP: Readonly> = { - test_case_id: 'test_case_id', + test_case_name: 'test_case_name', started_at: 'started_at', ended_at: 'ended_at', latency_ms: 'latency_ms', @@ -120,7 +120,7 @@ export const EvaluationSessionsDataView: FC = ( filter: { ...dataViewState.apiFilter.filter, ...(dataViewState.debouncedSearchBar && { - test_case_id: dataViewState.debouncedSearchBar, + test_case_name: dataViewState.debouncedSearchBar, }), }, }; @@ -185,12 +185,12 @@ export const EvaluationSessionsDataView: FC = ( const makeColumns: ComponentProps>['makeColumns'] = ({ accessor, }) => [ - accessor('test_case_id', { + accessor('test_case_name', { header: 'Test case', enableSorting: true, size: 200, cell: ({ row }) => { - const value = row.original.test_case_id; + const value = row.original.test_case_name; if (!value) return -; return ( @@ -298,7 +298,7 @@ export const EvaluationSessionsDataView: FC = ( { if (row.trace_id) { navigate( diff --git a/web/packages/studio/src/mocks/handlers.ts b/web/packages/studio/src/mocks/handlers.ts index 27890818ff..8922fb2440 100644 --- a/web/packages/studio/src/mocks/handlers.ts +++ b/web/packages/studio/src/mocks/handlers.ts @@ -496,8 +496,8 @@ export const handlers = [ http.get( '*/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions', ({ request, params }) => { - const testCaseId = new URL(request.url).searchParams.get('filter[test_case_id]'); - return HttpResponse.json(mockEvaluationSessionsPage(String(params['name']), testCaseId)); + const testCaseName = new URL(request.url).searchParams.get('filter[test_case_name]'); + return HttpResponse.json(mockEvaluationSessionsPage(String(params['name']), testCaseName)); } ), http.get('*/apis/intake/v2/workspaces/:workspace/traces', ({ request }) => { diff --git a/web/packages/studio/src/mocks/intake/experiments.ts b/web/packages/studio/src/mocks/intake/experiments.ts index 2407766138..28f230ba57 100644 --- a/web/packages/studio/src/mocks/intake/experiments.ts +++ b/web/packages/studio/src/mocks/intake/experiments.ts @@ -46,12 +46,12 @@ export const mockExperimentsPage = (): ExperimentResponsesPage => ({ const mockRun = ( evaluationName: string, sessionId: string, - testCaseId: string + testCaseName: string ): EvaluationSessionResponse => ({ workspace: WORKSPACE, evaluation_name: evaluationName, session_id: sessionId, - test_case_id: testCaseId, + test_case_name: testCaseName, trace_id: `trace-${sessionId}`, root_span_id: `${sessionId}-root`, started_at: '2026-01-01T00:00:00Z', @@ -61,19 +61,19 @@ const mockRun = ( // Runs of a test case, keyed by evaluation name. The primary session // `session-agent-run-001` lives in `my-experiment` alongside a sibling run, and // `my-experiment-v2` contributes a third run — so the compare selector has options. -const RUNS_BY_EVALUATION: Record EvaluationSessionResponse[]> = { - 'my-experiment': (testCaseId) => [ - mockRun('my-experiment', 'session-agent-run-001', testCaseId), - mockRun('my-experiment', 'session-agent-run-002', testCaseId), +const RUNS_BY_EVALUATION: Record EvaluationSessionResponse[]> = { + 'my-experiment': (testCaseName) => [ + mockRun('my-experiment', 'session-agent-run-001', testCaseName), + mockRun('my-experiment', 'session-agent-run-002', testCaseName), ], - 'my-experiment-v2': (testCaseId) => [ - mockRun('my-experiment-v2', 'session-agent-run-101', testCaseId), + 'my-experiment-v2': (testCaseName) => [ + mockRun('my-experiment-v2', 'session-agent-run-101', testCaseName), ], }; export const mockEvaluationSessionsPage = ( evaluationName: string, - testCaseId: string | null + testCaseName: string | null ): EvaluationSessionResponsesPage => ({ - data: testCaseId ? (RUNS_BY_EVALUATION[evaluationName]?.(testCaseId) ?? []) : [], + data: testCaseName ? (RUNS_BY_EVALUATION[evaluationName]?.(testCaseName) ?? []) : [], }); diff --git a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useSessionCompareRuns.ts b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useSessionCompareRuns.ts index 663df9d2e5..6b9953bde9 100644 --- a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useSessionCompareRuns.ts +++ b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useSessionCompareRuns.ts @@ -40,8 +40,7 @@ export function useSessionCompareRuns( const { runs, isLoading: isRunsLoading } = useTestCaseRuns({ workspace, evaluationNames, - // Evaluation sessions still expose this lookup through their test_case_id field. - testCaseId: testCaseName, + testCaseName, }); return { testCaseName, runs, isRunsLoading }; diff --git a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useTestCaseRuns.ts b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useTestCaseRuns.ts index e292330761..9fdb01f14d 100644 --- a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useTestCaseRuns.ts +++ b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/useTestCaseRuns.ts @@ -9,7 +9,7 @@ import type { EvaluationSessionResponse } from '@nemo/sdk/generated/platform/sch import { useQueries } from '@tanstack/react-query'; /** - * Every run (session) of `testCaseId` across the evaluations in an experiment group. + * Every run (session) of `testCaseName` across the evaluations in an experiment group. * * The sessions endpoint is scoped to a single evaluation, so this fans out one * query per evaluation (Option A) and flattens the results. A group-scoped @@ -18,17 +18,17 @@ import { useQueries } from '@tanstack/react-query'; export function useTestCaseRuns({ workspace, evaluationNames, - testCaseId, + testCaseName, }: { workspace: string; evaluationNames: string[]; - testCaseId: string | null | undefined; + testCaseName: string | null | undefined; }): { runs: EvaluationSessionResponse[]; isLoading: boolean } { - const enabled = Boolean(testCaseId) && evaluationNames.length > 0; + const enabled = Boolean(testCaseName) && evaluationNames.length > 0; const results = useQueries({ queries: evaluationNames.map((name) => { - const params = { filter: { test_case_id: testCaseId ?? '' }, page_size: 1000 }; + const params = { filter: { test_case_name: testCaseName ?? '' }, page_size: 1000 }; return { queryKey: getListEvaluationSessionsQueryKey(workspace, name, params), queryFn: ({ signal }: { signal: AbortSignal }) =>