diff --git a/.github/actions/changes/action.yaml b/.github/actions/changes/action.yaml index 89ebd57552..3296311c14 100644 --- a/.github/actions/changes/action.yaml +++ b/.github/actions/changes/action.yaml @@ -16,6 +16,9 @@ outputs: fabric: description: "'true' if the Fabric agent-eval runtime, its tests, or its dependency extra changed" value: ${{ steps.filter.outputs.fabric }} + evaluator-sdk-closure: + description: "'true' if anything that can change the agent_eval import closure changed" + value: ${{ steps.filter.outputs.evaluator-sdk-closure }} e2e: description: "'true' if any e2e test files changed" value: ${{ steps.filter.outputs.e2e }} @@ -95,6 +98,13 @@ runs: - 'packages/nemo_evaluator_sdk/pyproject.toml' - '.github/workflows/ci.yaml' - '.github/actions/changes/action.yaml' + evaluator-sdk-closure: + # Paths that can change what `import ...harbor_runtime` loads or how the smoke job runs. + - 'packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/**' # package-wide import closure + - 'packages/nemo_evaluator_sdk/pyproject.toml' # declared [project] dependencies + - 'uv.lock' # locked versions installed for the package + - '.github/workflows/ci.yaml' # smoke job definition + - '.github/actions/changes/action.yaml' # this filter (self-coverage) e2e: - 'e2e/**' docs: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e45c75c05..7af62daf74 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,6 +44,7 @@ jobs: test: ${{ steps.changes.outputs.test }} deps: ${{ steps.changes.outputs.deps }} fabric: ${{ steps.changes.outputs.fabric }} + evaluator-sdk-closure: ${{ steps.changes.outputs.evaluator-sdk-closure }} e2e: ${{ steps.changes.outputs.e2e }} docs: ${{ steps.changes.outputs.docs }} web-studio: ${{ steps.changes.outputs.web-studio }} @@ -1000,6 +1001,50 @@ jobs: uv run --frozen --no-sync pytest \ packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py -v + # Installs only nemo-evaluator-sdk (no workspace/extras/dev) and imports + # agent_eval.runtimes.harbor_runtime. Catches undeclared deps (workspace sync hides them) + # and broken lazy barrels that pull in the metric/execution stack. + evaluator-sdk-closure-smoke: + name: Evaluator SDK dependency-closure smoke (Linux, py${{ matrix.python-version }}) + needs: [changes] + if: > + !cancelled() && + (needs.changes.outputs.deps == 'true' || needs.changes.outputs.evaluator-sdk-closure == 'true') + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + # Must satisfy uv.lock's requires-python (>=3.12,<3.14), which this job syncs against. + python-version: ["3.12"] + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install nemo-evaluator-sdk with no extras and no dev group + run: uv sync --frozen --package nemo-evaluator-sdk --no-dev + # pytest is not in the package-only sync; install the runner only (do not add SDK deps). + # WHY BOTH THIS JOB AND THE UNIT TEST: + # - Unit-test CI syncs the whole workspace: a sibling (e.g. nemo-platform-sdk) may already + # provide httpx, so an undeclared SDK dep still imports — green here, broken for consumers + # who only installed nemo-evaluator-sdk. + # - This job: `uv sync --package nemo-evaluator-sdk --no-dev` (no extras/siblings) so that + # undeclared-dep case fails, then runs test_lazy_public_api (laziness assertions; one source + # of truth — no duplicated heredoc probe). + - name: Run lazy-import closure unit test under the stripped install + run: | + uv pip install --python .venv/bin/python pytest + uv run --frozen --no-sync pytest \ + packages/nemo_evaluator_sdk/tests/test_lazy_public_api.py::test_agent_eval_import_does_not_pull_the_execution_stack -v + python-integration-test: name: Python integration tests needs: [policy-wasm] @@ -1993,6 +2038,7 @@ jobs: - python-unit-test-tools - python-unit-test - fabric-wheel-smoke + - evaluator-sdk-closure-smoke # Enable if you want this required # - python-integration-test - require-nvskills diff --git a/packages/nemo_evaluator_sdk/pyproject.toml b/packages/nemo_evaluator_sdk/pyproject.toml index 7ca2798b77..6de714d919 100644 --- a/packages/nemo_evaluator_sdk/pyproject.toml +++ b/packages/nemo_evaluator_sdk/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "pyarrow>=19.0.1", "pandas>=1.5.3", "openai>=1.61.0", + "httpx>=0.27.0,<1", "sacrebleu>=2.5.1", "rouge_score==0.1.2", "ragas==0.4.3", diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py index fd959d5b40..0f251c80d3 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py @@ -6,9 +6,9 @@ The public surface resolves lazily (PEP 562). Importing this package must not drag in the execution/backend or metric stack: importing any submodule runs this module first, so eager re-exports made ``import nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime`` — all the -optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, pyarrow, ...) instead -of ~480, and turned every one of those transitive packages into an evaluation-time failure mode -for the SDK-backed evaluator. +optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, ...) instead of ~485, and +turned every one of those transitive packages into an evaluation-time failure mode for the +SDK-backed evaluator. Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in. @@ -16,14 +16,19 @@ # ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. -from importlib import import_module -from importlib.metadata import PackageNotFoundError +from importlib import import_module as _import_module +from importlib.metadata import PackageNotFoundError as _PackageNotFoundError from importlib.metadata import version as _package_version from typing import TYPE_CHECKING if TYPE_CHECKING: # Annotations and static analysis only; these must never execute at run time. Listing the # names in ``__all__`` is what marks them as re-exports for ruff and the type checkers. + # + # AGENTS.md ("Python Style notes") says not to import types under TYPE_CHECKING and to use a + # regular import "when possible". A regular import is exactly what this module exists to + # remove, so the exception is deliberate: these names are re-exports, not annotations, and + # every one of them resolves for real through ``__getattr__`` below. from nemo_evaluator_sdk.agent_stream_translation import ( AgentStreamTranslation, AgentStreamTranslationContext, @@ -97,10 +102,24 @@ SecretRef, ) -try: - version = _package_version("nemo-evaluator-sdk") -except PackageNotFoundError: - version = "0.0.0" + +def _resolve_version() -> str: + """Report the version of whichever distribution actually shipped this code. + + ``nemo-evaluator-sdk`` is not published on its own — this package is also vendored into the + ``nemo-platform`` wheel as ``nemo_platform.beta.evaluator``. There the SDK distribution does + not exist, so resolving only that name reported ``"0.0.0"`` unconditionally and any telemetry + or support log that read it got a useless constant. + """ + for distribution in ("nemo-evaluator-sdk", "nemo-platform"): + try: + return _package_version(distribution) + except _PackageNotFoundError: + continue + return "0.0.0" + + +version = _resolve_version() # Re-exported name -> the submodule that defines it, relative to this package. Relative on # purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting @@ -251,21 +270,29 @@ def __getattr__(name: str) -> object: """Import the submodule that defines ``name`` on first access (PEP 562). - ``AttributeError`` is the required failure mode, not ``ImportError``: ``from pkg import sub`` - only falls back to importing a submodule when attribute lookup raises ``AttributeError``, and - ``hasattr`` checks against this package depend on it too. + An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only + falls back to importing a submodule when attribute lookup raises ``AttributeError``. + + A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, and + that is deliberate — ``ModuleNotFoundError: No module named 'sacrebleu'`` is far more useful + than an ``AttributeError`` claiming ``BLEUMetric`` does not exist. The consequence is that + ``hasattr(nemo_evaluator_sdk, name)`` raises rather than returning ``False`` when a name's + dependencies are not installed, since ``hasattr`` only swallows ``AttributeError``. To probe + for an optional part of the surface, catch ``ImportError`` around the access instead of using + ``hasattr``; to probe only for name membership, test against ``__all__``. """ submodule = _LAZY_ATTRS.get(name) if submodule is None: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - value = getattr(import_module(submodule, __name__), name) + value = getattr(_import_module(submodule, __name__), name) globals()[name] = value # cache, so later lookups skip __getattr__ entirely return value def __dir__() -> list[str]: - # The declared surface plus any submodule the caller has already imported. ``import_module`` - # and ``TYPE_CHECKING`` are machinery for __getattr__, not API, so keep them out of - # autocomplete and inspect.getmembers. - public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING", "import_module"} + # The declared surface plus any submodule the caller has already imported. Everything this + # module needs for its own machinery is imported under a leading underscore so the filter + # below keeps it out of autocomplete and inspect.getmembers without a name-by-name denylist; + # ``TYPE_CHECKING`` is the one exception, kept unaliased so type checkers still recognise it. + public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"} return sorted(set(__all__) | public) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py index 298ef1335f..2dac3c9480 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py @@ -53,9 +53,8 @@ class SandboxSDK: def _load_agents_sdk() -> SandboxSDK: try: - # The OpenAI Agents SDK ships under the `nemo-evaluator-sdk[agent-runtimes]` extra and is - # imported only when this Docker runtime is actually used, so it is absent from the default - # type-checking environment. + # The OpenAI Agents SDK is imported only when this Docker runtime is actually used, so it + # is absent from the default type-checking environment. from agents.run import RunConfig # ty: ignore[unresolved-import] from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig # ty: ignore[unresolved-import] from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE # ty: ignore[unresolved-import] @@ -68,7 +67,14 @@ def _load_agents_sdk() -> SandboxSDK: from agents import Runner # ty: ignore[unresolved-import] from docker import from_env as docker_from_env except ImportError as exc: - raise RuntimeError("DockerSandboxAgentRuntime requires `nemo-evaluator-sdk[agent-runtimes]`") from exc + # Audience split is in the error text: SDK extras are not propagated into the + # vendored nemo_platform.beta.evaluator mirror. + raise RuntimeError( + "DockerSandboxAgentRuntime requires the openai-agents[docker] Python packages. " + "Standalone SDK: pip install 'nemo-evaluator-sdk[agent-runtimes]'. " + "Vendored nemo-platform.beta.evaluator (no SDK extras): " + "pip install 'openai-agents[docker]'" + ) from exc return SandboxSDK( Runner=Runner, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_inference.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_inference.py index fcde7d8d62..a9b95197f2 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_inference.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_inference.py @@ -27,7 +27,6 @@ import httpx from httpx import Timeout -from jsonpath_ng import parse as jsonpath_parse from pydantic import BaseModel, ConfigDict, Field from nemo_evaluator_sdk.agent_stream_translation import ( @@ -788,6 +787,10 @@ def _extract_jsonpath( required: bool = True, ) -> Any: """Extract a value from data using a JSONPath expression.""" + # Imported here rather than at module scope: this is jsonpath_ng's only use in the module, and + # the module sits on the agent_eval run-time path via agent_eval/evaluator.py. + from jsonpath_ng import parse as jsonpath_parse + expr = jsonpath_parse(path) matches = expr.find(data) if not matches: diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py index 4a626e9c1a..def34d4e6b 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py @@ -3,10 +3,12 @@ """Aggregation data structures and computations for metric results.""" +from __future__ import annotations + import math from collections import OrderedDict, defaultdict from collections.abc import Mapping, Sequence -from typing import Protocol, cast, runtime_checkable +from typing import TYPE_CHECKING, Protocol, cast, runtime_checkable from nemo_evaluator_sdk.metrics.protocol import ( BooleanValue, @@ -28,7 +30,11 @@ RubricScoreStat, ScoreStats, ) -from nemo_evaluator_sdk.values.scores import RubricScore, Score + +if TYPE_CHECKING: + # Importing the score configuration module loads jsonschema. Aggregation's lightweight helpers + # (notably compute_percentiles) do not need it, so keep this typing-only edge deferred. + from nemo_evaluator_sdk.values.scores import Score def is_aggregateable_output_spec(output_spec: MetricOutputSpec) -> bool: @@ -445,6 +451,8 @@ def aggregate_metrics( def rubric_definitions_from_scores(scores: Sequence[Score]) -> dict[str, list[RubricScoreStat]]: """Return declared rubric buckets keyed by score name.""" + from nemo_evaluator_sdk.values.scores import RubricScore + definitions: dict[str, list[RubricScoreStat]] = {} for score in scores: if not isinstance(score, RubricScore): diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py index a92357e26f..37f80764a5 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py @@ -1,121 +1,241 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Public value types for evaluator SDK runtime.""" +"""Public value types for evaluator SDK runtime. + +The public interface resolves lazily (PEP 562), for the same reason the package root does: +every ``from nemo_evaluator_sdk.values.X import ...`` runs this barrel first, so +eagerly re-exporting all 97 names dragged ``.datasets``/``.results`` (pyarrow, numpy) and +``.metrics``/``.scores`` (jsonschema, jinja2) into ``agent_eval``, which uses none of them. +Measured: 485 modules and +57 MB RSS for ``import agent_eval.runtimes.harbor_runtime`` before, +300 modules and pydantic alone after. + +Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a +module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in. +""" # ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. -from nemo_evaluator_sdk.values.agents import ( - Agent, - AgentBase, - GenericAgent, - NatAgentConfig, - NemoAgentToolkitAgent, -) -from nemo_evaluator_sdk.values.atif import ( - FinalMetrics, - Metrics, - Observation, - ObservationResult, - Step, - ToolCall, - Trajectory, -) -from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes -from nemo_evaluator_sdk.values.dataset_schemas import ( - FieldMapping, - InputSchema, -) -from nemo_evaluator_sdk.values.datasets import DatasetInput, DatasetRows -from nemo_evaluator_sdk.values.evidence import ( - CandidateEvidence, - CommandResult, - EvidenceDescriptor, - FilesystemDiff, - FilesystemEntry, - LocalFilesystemEvidence, - LogHandle, - TraceHandle, - WellKnownEvidenceKey, - parse_atif, -) -from nemo_evaluator_sdk.values.metrics import ( - BLEU, - F1, - ROUGE, - AgentGoalAccuracy, - AnswerAccuracy, - ContextEntityRecall, - ContextPrecision, - ContextRecall, - ContextRelevance, - ExactMatch, - Faithfulness, - LLMJudge, - MetricBase, - NemoAgentToolkitRemote, - NoiseSensitivity, - NumberCheck, - Remote, - ResponseGroundedness, - ResponseRelevancy, - StringCheck, - ToolCallAccuracy, - ToolCalling, - TopicAdherence, - TunableRagEvaluator, -) -from nemo_evaluator_sdk.values.models import Model, ModelRef, ReasoningParams -from nemo_evaluator_sdk.values.params import ( - InferenceParams, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) -from nemo_evaluator_sdk.values.protocol import ( - BooleanValue, - CandidateOutput, - ContinuousScore, - DatasetRow, - DiscreteScore, - Label, - MetricDescriptor, - MetricDiagnostic, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, - MetricTypeName, -) -from nemo_evaluator_sdk.values.results import ( - AggregatedMetricResult, - AggregateFieldName, - AggregateRangeScore, - AggregateRubricScore, - AggregateScore, - AggregateScoreBase, - DefaultAggregateFieldName, - EvaluationResult, - Histogram, - HistogramBin, - MetricScore, - Percentiles, - RowScore, - RubricScoreStat, - RubricScoreValue, - SampleResult, - ScoreStats, -) -from nemo_evaluator_sdk.values.scores import ( - JSONScoreParser, - RangeScore, - RegexScoreParser, - RemoteScore, - Rubric, - RubricScore, - Score, - score_discriminator, -) +from importlib import import_module as _import_module +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from nemo_evaluator_sdk.values.agents import ( + Agent, + AgentBase, + GenericAgent, + NatAgentConfig, + NemoAgentToolkitAgent, + ) + from nemo_evaluator_sdk.values.atif import ( + FinalMetrics, + Metrics, + Observation, + ObservationResult, + Step, + ToolCall, + Trajectory, + ) + from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes + from nemo_evaluator_sdk.values.dataset_schemas import ( + FieldMapping, + InputSchema, + ) + from nemo_evaluator_sdk.values.datasets import DatasetInput, DatasetRows + from nemo_evaluator_sdk.values.evidence import ( + CandidateEvidence, + CommandResult, + EvidenceDescriptor, + FilesystemDiff, + FilesystemEntry, + LocalFilesystemEvidence, + LogHandle, + TraceHandle, + WellKnownEvidenceKey, + parse_atif, + ) + from nemo_evaluator_sdk.values.metrics import ( + BLEU, + F1, + ROUGE, + AgentGoalAccuracy, + AnswerAccuracy, + ContextEntityRecall, + ContextPrecision, + ContextRecall, + ContextRelevance, + ExactMatch, + Faithfulness, + LLMJudge, + MetricBase, + NemoAgentToolkitRemote, + NoiseSensitivity, + NumberCheck, + Remote, + ResponseGroundedness, + ResponseRelevancy, + StringCheck, + ToolCallAccuracy, + ToolCalling, + TopicAdherence, + TunableRagEvaluator, + ) + from nemo_evaluator_sdk.values.models import Model, ModelRef, ReasoningParams + from nemo_evaluator_sdk.values.params import ( + InferenceParams, + RunConfig, + RunConfigOnline, + RunConfigOnlineModel, + ) + from nemo_evaluator_sdk.values.protocol import ( + BooleanValue, + CandidateOutput, + ContinuousScore, + DatasetRow, + DiscreteScore, + Label, + MetricDescriptor, + MetricDiagnostic, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, + MetricTypeName, + ) + from nemo_evaluator_sdk.values.results import ( + AggregatedMetricResult, + AggregateFieldName, + AggregateRangeScore, + AggregateRubricScore, + AggregateScore, + AggregateScoreBase, + DefaultAggregateFieldName, + EvaluationResult, + Histogram, + HistogramBin, + MetricScore, + Percentiles, + RowScore, + RubricScoreStat, + RubricScoreValue, + SampleResult, + ScoreStats, + ) + from nemo_evaluator_sdk.values.scores import ( + JSONScoreParser, + RangeScore, + RegexScoreParser, + RemoteScore, + Rubric, + RubricScore, + Score, + score_discriminator, + ) + + +# Re-exported name -> the submodule that defines it, relative to this package. Relative on +# purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting +# module paths, and a relative name has nothing to rewrite, so the mirror is correct by +# construction. Mirrors the TYPE_CHECKING block above, in the same order. +_LAZY_ATTRS: dict[str, str] = { + "Agent": ".agents", + "AgentBase": ".agents", + "GenericAgent": ".agents", + "NatAgentConfig": ".agents", + "NemoAgentToolkitAgent": ".agents", + "FinalMetrics": ".atif", + "Metrics": ".atif", + "Observation": ".atif", + "ObservationResult": ".atif", + "Step": ".atif", + "ToolCall": ".atif", + "Trajectory": ".atif", + "SecretRef": ".common", + "SupportedJobTypes": ".common", + "FieldMapping": ".dataset_schemas", + "InputSchema": ".dataset_schemas", + "DatasetInput": ".datasets", + "DatasetRows": ".datasets", + "CandidateEvidence": ".evidence", + "CommandResult": ".evidence", + "EvidenceDescriptor": ".evidence", + "FilesystemDiff": ".evidence", + "FilesystemEntry": ".evidence", + "LocalFilesystemEvidence": ".evidence", + "LogHandle": ".evidence", + "TraceHandle": ".evidence", + "WellKnownEvidenceKey": ".evidence", + "parse_atif": ".evidence", + "BLEU": ".metrics", + "F1": ".metrics", + "ROUGE": ".metrics", + "AgentGoalAccuracy": ".metrics", + "AnswerAccuracy": ".metrics", + "ContextEntityRecall": ".metrics", + "ContextPrecision": ".metrics", + "ContextRecall": ".metrics", + "ContextRelevance": ".metrics", + "ExactMatch": ".metrics", + "Faithfulness": ".metrics", + "LLMJudge": ".metrics", + "MetricBase": ".metrics", + "NemoAgentToolkitRemote": ".metrics", + "NoiseSensitivity": ".metrics", + "NumberCheck": ".metrics", + "Remote": ".metrics", + "ResponseGroundedness": ".metrics", + "ResponseRelevancy": ".metrics", + "StringCheck": ".metrics", + "ToolCallAccuracy": ".metrics", + "ToolCalling": ".metrics", + "TopicAdherence": ".metrics", + "TunableRagEvaluator": ".metrics", + "Model": ".models", + "ModelRef": ".models", + "ReasoningParams": ".models", + "InferenceParams": ".params", + "RunConfig": ".params", + "RunConfigOnline": ".params", + "RunConfigOnlineModel": ".params", + "BooleanValue": ".protocol", + "CandidateOutput": ".protocol", + "ContinuousScore": ".protocol", + "DatasetRow": ".protocol", + "DiscreteScore": ".protocol", + "Label": ".protocol", + "MetricDescriptor": ".protocol", + "MetricDiagnostic": ".protocol", + "MetricInput": ".protocol", + "MetricOutput": ".protocol", + "MetricOutputSpec": ".protocol", + "MetricResult": ".protocol", + "MetricTypeName": ".protocol", + "AggregatedMetricResult": ".results", + "AggregateFieldName": ".results", + "AggregateRangeScore": ".results", + "AggregateRubricScore": ".results", + "AggregateScore": ".results", + "AggregateScoreBase": ".results", + "DefaultAggregateFieldName": ".results", + "EvaluationResult": ".results", + "Histogram": ".results", + "HistogramBin": ".results", + "MetricScore": ".results", + "Percentiles": ".results", + "RowScore": ".results", + "RubricScoreStat": ".results", + "RubricScoreValue": ".results", + "SampleResult": ".results", + "ScoreStats": ".results", + "JSONScoreParser": ".scores", + "RangeScore": ".scores", + "RegexScoreParser": ".scores", + "RemoteScore": ".scores", + "Rubric": ".scores", + "RubricScore": ".scores", + "Score": ".scores", + "score_discriminator": ".scores", +} __all__ = [ "Agent", @@ -217,3 +337,30 @@ "TopicAdherence", "TunableRagEvaluator", ] + + +def __getattr__(name: str) -> object: + """Import the submodule that defines ``name`` on first access (PEP 562). + + An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only + falls back to importing a submodule when attribute lookup raises ``AttributeError``. + + A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, so + the real cause is not hidden behind a bogus "no attribute". The consequence is that + ``hasattr`` raises rather than returning ``False`` when a name's dependencies are missing; + catch ``ImportError`` around the access, or test membership against ``__all__``. + """ + submodule = _LAZY_ATTRS.get(name) + if submodule is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(_import_module(submodule, __name__), name) + globals()[name] = value # cache, so later lookups skip __getattr__ entirely + return value + + +def __dir__() -> list[str]: + # The declared surface plus any submodule the caller has already imported. Machinery is + # imported under a leading underscore so the filter keeps it out of autocomplete without a + # denylist; ``TYPE_CHECKING`` is the one exception, unaliased so type checkers recognise it. + public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"} + return sorted(set(__all__) | public) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py index 02eef50977..9c06fefd76 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py @@ -9,11 +9,13 @@ import math from collections.abc import Mapping from difflib import get_close_matches -from typing import Any, Literal, Self +from typing import TYPE_CHECKING, Any, Literal, Self -import pyarrow as pa from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_serializer +if TYPE_CHECKING: + import pyarrow as pa + from nemo_evaluator_sdk.values.protocol import MetricDiagnostic, MetricOutput, MetricResult ResultView = Literal["rows", "aggregate"] @@ -792,6 +794,12 @@ def to_table(self, view: ResultView = "rows") -> pa.Table: Returns: Table built from ``to_records(view=view)``. """ + # Imported here rather than at module scope: pyarrow (plus its numpy tail) costs ~31 MB + # RSS and 223 modules, this is its only runtime use in the module, and the module is on + # the agent_eval import path, which never calls this method. The return annotation is a + # string already (`from __future__ import annotations`), so it needs no import. + import pyarrow as pa + return pa.Table.from_pylist(self.to_records(view=view)) def to_pandas(self, view: ResultView = "rows"): diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py index c33faae163..9d300b967e 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py @@ -196,9 +196,13 @@ def fake_import( monkeypatch.setattr(builtins, "__import__", fake_import) - with pytest.raises(RuntimeError, match=r"nemo-evaluator-sdk\[agent-runtimes\]"): + with pytest.raises(RuntimeError, match=r"nemo-evaluator-sdk\[agent-runtimes\]") as exc_info: docker_sandbox._load_agents_sdk() + message = str(exc_info.value) + assert "pip install 'openai-agents[docker]'" in message + assert "openai-agents[docker]>=" not in message + def test_manifest_uses_instruction_and_never_leaks_intent() -> None: # The instruction is surfaced verbatim; `task.intent` is eval-side metadata and must never leak to diff --git a/packages/nemo_evaluator_sdk/tests/test_lazy_public_api.py b/packages/nemo_evaluator_sdk/tests/test_lazy_public_api.py index ae24309a5d..771251e1c9 100644 --- a/packages/nemo_evaluator_sdk/tests/test_lazy_public_api.py +++ b/packages/nemo_evaluator_sdk/tests/test_lazy_public_api.py @@ -1,29 +1,77 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Guards the lazy (PEP 562) public surface of ``nemo_evaluator_sdk/__init__.py``. +"""Guards the lazy (PEP 562) public surfaces of the evaluator SDK. -Importing any submodule runs the package ``__init__`` first, so a single convenience -``from nemo_evaluator_sdk.… import …`` added at module scope silently re-drags the whole -backend/benchmark/metric stack into every consumer that only wanted ``agent_eval`` — the -regression these tests exist to catch (AALGO-429). +Two barrels re-export lazily, and importing any submodule runs both in turn, so a single +convenience ``from … import …`` added at either module scope silently re-drags the whole +backend/benchmark/metric stack into every consumer that only wanted ``agent_eval``: + +* ``nemo_evaluator_sdk/__init__.py`` (AALGO-429) — the execution/backend and metric stack. +* ``nemo_evaluator_sdk/values/__init__.py`` (AALGO-311) — pyarrow, numpy, jinja2 and jsonschema, + together with the deferred pyarrow import in ``values/results.py``. + +Both are covered here in their source form and in the ``nemo_platform.beta.evaluator`` mirror the +vendoring tool generates. + +Every *assertion* runs out-of-process. Resolving a whole public surface imports openai, sacrebleu, +ragas and the execution stack; doing that in-process would leave them in ``sys.modules`` for every +test that runs after it in the session, so a future in-process "module X must not be imported" +check — the natural way someone would extend this file — would depend on collection order. The one +in-process call is the ``find_spec`` availability probe below, which is deliberately restricted to +a top-level name so that it imports nothing. """ +import importlib.util import json import subprocess import sys import pytest +_VENDORED_MIRROR = "nemo_platform.beta.evaluator" + # Imported out-of-process on purpose: by the time this module runs under pytest, sibling suites # have already pulled the execution stack into sys.modules, so an in-process check proves nothing. -_PROBE = """ +_IMPORT_SURFACE_PROBE = """ import json, sys from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner assert HarborAgentTaskRunner is not None print(json.dumps(sorted(sys.modules))) """ +# Resolves every declared name for real. This is also the only thing left that validates each +# re-exported submodule still imports at all: before the package went lazy, `import +# nemo_evaluator_sdk` executed all of them, so a syntax error or a broken third-party import +# failed immediately everywhere. Narrowing this loop, skipping this test, or dropping a name from +# `__all__` silently gives that guarantee up. +_PUBLIC_SURFACE_PROBE = """ +import json, sys + +name, submodule_name = sys.argv[1], sys.argv[2] +module = __import__(name, fromlist=["__all__"]) + +for attribute in module.__all__: # AttributeError/ImportError names the offender in stderr + getattr(module, attribute) + +try: + module.NoSuchName +except AttributeError: + pass +else: + raise AssertionError("unknown attribute did not raise AttributeError") + +# The `from pkg import submodule` fallback, which only fires when __getattr__ raises +# AttributeError rather than KeyError or ImportError. +submodule = getattr(__import__(name, fromlist=[submodule_name]), submodule_name) +assert submodule.__name__ == f"{name}.{submodule_name}", submodule.__name__ + +print(json.dumps({ + "resolved": len(module.__all__), + "missing_from_dir": sorted(set(module.__all__) - set(dir(module))), +})) +""" + def test_agent_eval_import_does_not_pull_the_execution_stack() -> None: """The optimizer imports only ``agent_eval``; it must not pay for backends and benchmarks. @@ -32,7 +80,7 @@ def test_agent_eval_import_does_not_pull_the_execution_stack() -> None: SDK-path failure at evaluation time. Keeping the boundary tight is what lets the consumer's deferred SDK import actually contain a broken install. """ - proc = subprocess.run([sys.executable, "-c", _PROBE], capture_output=True, text=True, timeout=120) + proc = subprocess.run([sys.executable, "-c", _IMPORT_SURFACE_PROBE], capture_output=True, text=True, timeout=120) assert proc.returncode == 0, proc.stderr # Last line only: the child inherits the environment, so a sitecustomize banner or an @@ -42,40 +90,62 @@ def test_agent_eval_import_does_not_pull_the_execution_stack() -> None: leaked = sorted(name for name in modules if name.startswith("nemo_evaluator_sdk.execution")) assert leaked == [], f"the package __init__ re-drags the execution stack: {leaked}" - # Exactly the packages the eager __init__ used to drag in; each one goes red if it returns. + # Every one of these was on this path before the two barrels went lazy, so each goes red if it + # returns: openai/sacrebleu/zstandard came from the root __init__, and pyarrow/numpy/jinja2/ + # jsonschema from the values/ barrel plus values/results.py's module-scope pyarrow import. # rouge_score is deliberately absent: metrics/rouge.py already defers it into a # cached_property, so it was never on this path and asserting it would prove nothing. - heavy = {"openai", "sacrebleu", "zstandard"} & modules - assert heavy == set(), f"heavy metric-stack dependencies pulled in: {sorted(heavy)}" - - # A canary, not a spec: measured at 483 modules when this landed, down from 1416. Raise the - # bound if a genuine agent_eval dependency lands; a jump of >100 means something re-drags a - # barrel module and should be investigated rather than accommodated. - assert len(modules) < 700, f"agent_eval import surface grew to {len(modules)} modules" - - -def test_every_public_name_resolves() -> None: - """``__all__`` and ``_LAZY_ATTRS`` must not drift apart. + heavy = {"openai", "sacrebleu", "zstandard", "pyarrow", "numpy", "jinja2", "jsonschema"} & modules + assert heavy == set(), f"heavy dependencies pulled into the agent_eval path: {sorted(heavy)}" + + # A canary, not a spec: measured at 300 modules once both barrels went lazy, down from 1416. + # The bound is deliberately close — the assertions above enumerate known offenders, so only + # this catches a re-drag through some other route. Raise it only for a dependency agent_eval + # genuinely needs, and say which in the commit message. + assert len(modules) < 380, f"agent_eval import surface grew to {len(modules)} modules" + + +@pytest.mark.parametrize( + ("module_name", "submodule_name"), + [ + ("nemo_evaluator_sdk", "values"), + ("nemo_evaluator_sdk.values", "models"), + (_VENDORED_MIRROR, "values"), + (f"{_VENDORED_MIRROR}.values", "models"), + ], +) +def test_every_public_name_resolves(module_name: str, submodule_name: str) -> None: + """``__all__`` and ``_LAZY_ATTRS`` must not drift apart, in the source or the vendored mirror. A typo in the lazy table is invisible until a consumer hits that one attribute, so resolve the whole surface in one pass. - """ - import nemo_evaluator_sdk - - for name in nemo_evaluator_sdk.__all__: - assert getattr(nemo_evaluator_sdk, name) is not None, name - assert set(nemo_evaluator_sdk.__all__) <= set(dir(nemo_evaluator_sdk)) - - -def test_unknown_attribute_raises_attribute_error() -> None: - """``from pkg import submodule`` and ``hasattr`` both rely on ``AttributeError`` here.""" - import nemo_evaluator_sdk - - with pytest.raises(AttributeError): - nemo_evaluator_sdk.NoSuchName # noqa: B018 + The mirror leg is what keeps the relative module paths in ``_LAZY_ATTRS`` honest. They are + relative precisely so the vendoring tool has nothing to rewrite; if someone switches them to + an f-string (which the rewriter does not touch) or to absolute literals under a changed + rewriter, the mirror resolves to the wrong package. Existing tests import *through* the mirror + but never resolve its surface, so nothing else would notice. + """ + # Probe the top-level package only. `find_spec` on a dotted name RAISES ModuleNotFoundError + # when a parent is missing rather than returning None, so probing the full path would error + # in exactly the case this guard exists for (no nemo-platform installed). It also imports + # every parent in-process, which would defeat this module's isolation. + root_package = module_name.partition(".")[0] + if importlib.util.find_spec(root_package) is None: + pytest.skip(f"{root_package} is not installed in this environment") + + proc = subprocess.run( + [sys.executable, "-c", _PUBLIC_SURFACE_PROBE, module_name, submodule_name], + capture_output=True, + text=True, + timeout=120, + ) + assert proc.returncode == 0, proc.stderr - # The submodule fallback that AttributeError enables. - from nemo_evaluator_sdk import values + result = json.loads(proc.stdout.strip().splitlines()[-1]) + assert result["missing_from_dir"] == [], f"__all__ names absent from dir(): {result['missing_from_dir']}" + assert result["resolved"] > 0 - assert values.__name__ == "nemo_evaluator_sdk.values" + # `version` is deliberately not asserted here: this workspace installs nemo-evaluator-sdk + # itself, and its declared version is literally "0.0.0", so the distribution-fallback in + # _resolve_version() only changes behaviour in a built wheel, where the SDK is absent. diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index 6a7aa8af0b..e2acf75f19 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -292,6 +292,7 @@ nemo-evaluator-sdk = [ "pyarrow>=19.0.1", "pandas>=1.5.3", "openai>=1.61.0", + "httpx>=0.27.0,<1", "sacrebleu>=2.5.1", "rouge_score==0.1.2", "ragas==0.4.3", diff --git a/sdk/python/nemo-platform/pyproject.toml b/sdk/python/nemo-platform/pyproject.toml index 3cdbb33916..d45b878903 100644 --- a/sdk/python/nemo-platform/pyproject.toml +++ b/sdk/python/nemo-platform/pyproject.toml @@ -62,6 +62,7 @@ nemo-evaluator-sdk = [ "langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0", "nemo-relay>=0.6.0,<0.7", "nemo-fabric>=0.1.1,<0.3.0", + "httpx>=0.27.0,<1", ] [project.entry-points."nemo.skills"] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py index 54b5891b76..48cda897cc 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py @@ -6,9 +6,9 @@ The public surface resolves lazily (PEP 562). Importing this package must not drag in the execution/backend or metric stack: importing any submodule runs this module first, so eager re-exports made ``import nemo_platform.beta.evaluator.agent_eval.runtimes.harbor_runtime`` — all the -optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, pyarrow, ...) instead -of ~480, and turned every one of those transitive packages into an evaluation-time failure mode -for the SDK-backed evaluator. +optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, ...) instead of ~485, and +turned every one of those transitive packages into an evaluation-time failure mode for the +SDK-backed evaluator. Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in. @@ -16,14 +16,19 @@ # ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. -from importlib import import_module -from importlib.metadata import PackageNotFoundError +from importlib import import_module as _import_module +from importlib.metadata import PackageNotFoundError as _PackageNotFoundError from importlib.metadata import version as _package_version from typing import TYPE_CHECKING if TYPE_CHECKING: # Annotations and static analysis only; these must never execute at run time. Listing the # names in ``__all__`` is what marks them as re-exports for ruff and the type checkers. + # + # AGENTS.md ("Python Style notes") says not to import types under TYPE_CHECKING and to use a + # regular import "when possible". A regular import is exactly what this module exists to + # remove, so the exception is deliberate: these names are re-exports, not annotations, and + # every one of them resolves for real through ``__getattr__`` below. from nemo_platform.beta.evaluator.agent_stream_translation import ( AgentStreamTranslation, AgentStreamTranslationContext, @@ -97,10 +102,24 @@ SecretRef, ) -try: - version = _package_version("nemo-evaluator-sdk") -except PackageNotFoundError: - version = "0.0.0" + +def _resolve_version() -> str: + """Report the version of whichever distribution actually shipped this code. + + ``nemo-evaluator-sdk`` is not published on its own — this package is also vendored into the + ``nemo-platform`` wheel as ``nemo_platform.beta.evaluator``. There the SDK distribution does + not exist, so resolving only that name reported ``"0.0.0"`` unconditionally and any telemetry + or support log that read it got a useless constant. + """ + for distribution in ("nemo-evaluator-sdk", "nemo-platform"): + try: + return _package_version(distribution) + except _PackageNotFoundError: + continue + return "0.0.0" + + +version = _resolve_version() # Re-exported name -> the submodule that defines it, relative to this package. Relative on # purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting @@ -251,21 +270,29 @@ def __getattr__(name: str) -> object: """Import the submodule that defines ``name`` on first access (PEP 562). - ``AttributeError`` is the required failure mode, not ``ImportError``: ``from pkg import sub`` - only falls back to importing a submodule when attribute lookup raises ``AttributeError``, and - ``hasattr`` checks against this package depend on it too. + An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only + falls back to importing a submodule when attribute lookup raises ``AttributeError``. + + A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, and + that is deliberate — ``ModuleNotFoundError: No module named 'sacrebleu'`` is far more useful + than an ``AttributeError`` claiming ``BLEUMetric`` does not exist. The consequence is that + ``hasattr(nemo_evaluator_sdk, name)`` raises rather than returning ``False`` when a name's + dependencies are not installed, since ``hasattr`` only swallows ``AttributeError``. To probe + for an optional part of the surface, catch ``ImportError`` around the access instead of using + ``hasattr``; to probe only for name membership, test against ``__all__``. """ submodule = _LAZY_ATTRS.get(name) if submodule is None: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - value = getattr(import_module(submodule, __name__), name) + value = getattr(_import_module(submodule, __name__), name) globals()[name] = value # cache, so later lookups skip __getattr__ entirely return value def __dir__() -> list[str]: - # The declared surface plus any submodule the caller has already imported. ``import_module`` - # and ``TYPE_CHECKING`` are machinery for __getattr__, not API, so keep them out of - # autocomplete and inspect.getmembers. - public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING", "import_module"} + # The declared surface plus any submodule the caller has already imported. Everything this + # module needs for its own machinery is imported under a leading underscore so the filter + # below keeps it out of autocomplete and inspect.getmembers without a name-by-name denylist; + # ``TYPE_CHECKING`` is the one exception, kept unaliased so type checkers still recognise it. + public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"} return sorted(set(__all__) | public) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py index ac7f2544d5..bdf1cc979e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py @@ -53,9 +53,8 @@ class SandboxSDK: def _load_agents_sdk() -> SandboxSDK: try: - # The OpenAI Agents SDK ships under the `nemo-evaluator-sdk[agent-runtimes]` extra and is - # imported only when this Docker runtime is actually used, so it is absent from the default - # type-checking environment. + # The OpenAI Agents SDK is imported only when this Docker runtime is actually used, so it + # is absent from the default type-checking environment. from agents.run import RunConfig # ty: ignore[unresolved-import] from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig # ty: ignore[unresolved-import] from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE # ty: ignore[unresolved-import] @@ -68,7 +67,14 @@ def _load_agents_sdk() -> SandboxSDK: from agents import Runner # ty: ignore[unresolved-import] from docker import from_env as docker_from_env except ImportError as exc: - raise RuntimeError("DockerSandboxAgentRuntime requires `nemo-evaluator-sdk[agent-runtimes]`") from exc + # Audience split is in the error text: SDK extras are not propagated into the + # vendored nemo_platform.beta.evaluator mirror. + raise RuntimeError( + "DockerSandboxAgentRuntime requires the openai-agents[docker] Python packages. " + "Standalone SDK: pip install 'nemo-evaluator-sdk[agent-runtimes]'. " + "Vendored nemo-platform.beta.evaluator (no SDK extras): " + "pip install 'openai-agents[docker]'" + ) from exc return SandboxSDK( Runner=Runner, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py index 2e759fe3e2..181b29bbdf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py @@ -27,7 +27,6 @@ import httpx from httpx import Timeout -from jsonpath_ng import parse as jsonpath_parse from pydantic import BaseModel, ConfigDict, Field from nemo_platform.beta.evaluator.agent_stream_translation import ( @@ -788,6 +787,10 @@ def _extract_jsonpath( required: bool = True, ) -> Any: """Extract a value from data using a JSONPath expression.""" + # Imported here rather than at module scope: this is jsonpath_ng's only use in the module, and + # the module sits on the agent_eval run-time path via agent_eval/evaluator.py. + from jsonpath_ng import parse as jsonpath_parse + expr = jsonpath_parse(path) matches = expr.find(data) if not matches: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py index 3df0f7a395..22698fab8f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py @@ -3,10 +3,12 @@ """Aggregation data structures and computations for metric results.""" +from __future__ import annotations + import math from collections import OrderedDict, defaultdict from collections.abc import Mapping, Sequence -from typing import Protocol, cast, runtime_checkable +from typing import TYPE_CHECKING, Protocol, cast, runtime_checkable from nemo_platform.beta.evaluator.metrics.protocol import ( BooleanValue, @@ -28,7 +30,11 @@ RubricScoreStat, ScoreStats, ) -from nemo_platform.beta.evaluator.values.scores import RubricScore, Score + +if TYPE_CHECKING: + # Importing the score configuration module loads jsonschema. Aggregation's lightweight helpers + # (notably compute_percentiles) do not need it, so keep this typing-only edge deferred. + from nemo_platform.beta.evaluator.values.scores import Score def is_aggregateable_output_spec(output_spec: MetricOutputSpec) -> bool: @@ -445,6 +451,8 @@ def aggregate_metrics( def rubric_definitions_from_scores(scores: Sequence[Score]) -> dict[str, list[RubricScoreStat]]: """Return declared rubric buckets keyed by score name.""" + from nemo_platform.beta.evaluator.values.scores import RubricScore + definitions: dict[str, list[RubricScoreStat]] = {} for score in scores: if not isinstance(score, RubricScore): diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py index db93c9807b..18efea4ac9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py @@ -1,121 +1,241 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Public value types for evaluator SDK runtime.""" +"""Public value types for evaluator SDK runtime. + +The public interface resolves lazily (PEP 562), for the same reason the package root does: +every ``from nemo_platform.beta.evaluator.values.X import ...`` runs this barrel first, so +eagerly re-exporting all 97 names dragged ``.datasets``/``.results`` (pyarrow, numpy) and +``.metrics``/``.scores`` (jsonschema, jinja2) into ``agent_eval``, which uses none of them. +Measured: 485 modules and +57 MB RSS for ``import agent_eval.runtimes.harbor_runtime`` before, +300 modules and pydantic alone after. + +Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a +module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in. +""" # ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. -from nemo_platform.beta.evaluator.values.agents import ( - Agent, - AgentBase, - GenericAgent, - NatAgentConfig, - NemoAgentToolkitAgent, -) -from nemo_platform.beta.evaluator.values.atif import ( - FinalMetrics, - Metrics, - Observation, - ObservationResult, - Step, - ToolCall, - Trajectory, -) -from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes -from nemo_platform.beta.evaluator.values.dataset_schemas import ( - FieldMapping, - InputSchema, -) -from nemo_platform.beta.evaluator.values.datasets import DatasetInput, DatasetRows -from nemo_platform.beta.evaluator.values.evidence import ( - CandidateEvidence, - CommandResult, - EvidenceDescriptor, - FilesystemDiff, - FilesystemEntry, - LocalFilesystemEvidence, - LogHandle, - TraceHandle, - WellKnownEvidenceKey, - parse_atif, -) -from nemo_platform.beta.evaluator.values.metrics import ( - BLEU, - F1, - ROUGE, - AgentGoalAccuracy, - AnswerAccuracy, - ContextEntityRecall, - ContextPrecision, - ContextRecall, - ContextRelevance, - ExactMatch, - Faithfulness, - LLMJudge, - MetricBase, - NemoAgentToolkitRemote, - NoiseSensitivity, - NumberCheck, - Remote, - ResponseGroundedness, - ResponseRelevancy, - StringCheck, - ToolCallAccuracy, - ToolCalling, - TopicAdherence, - TunableRagEvaluator, -) -from nemo_platform.beta.evaluator.values.models import Model, ModelRef, ReasoningParams -from nemo_platform.beta.evaluator.values.params import ( - InferenceParams, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) -from nemo_platform.beta.evaluator.values.protocol import ( - BooleanValue, - CandidateOutput, - ContinuousScore, - DatasetRow, - DiscreteScore, - Label, - MetricDescriptor, - MetricDiagnostic, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, - MetricTypeName, -) -from nemo_platform.beta.evaluator.values.results import ( - AggregatedMetricResult, - AggregateFieldName, - AggregateRangeScore, - AggregateRubricScore, - AggregateScore, - AggregateScoreBase, - DefaultAggregateFieldName, - EvaluationResult, - Histogram, - HistogramBin, - MetricScore, - Percentiles, - RowScore, - RubricScoreStat, - RubricScoreValue, - SampleResult, - ScoreStats, -) -from nemo_platform.beta.evaluator.values.scores import ( - JSONScoreParser, - RangeScore, - RegexScoreParser, - RemoteScore, - Rubric, - RubricScore, - Score, - score_discriminator, -) +from importlib import import_module as _import_module +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from nemo_platform.beta.evaluator.values.agents import ( + Agent, + AgentBase, + GenericAgent, + NatAgentConfig, + NemoAgentToolkitAgent, + ) + from nemo_platform.beta.evaluator.values.atif import ( + FinalMetrics, + Metrics, + Observation, + ObservationResult, + Step, + ToolCall, + Trajectory, + ) + from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes + from nemo_platform.beta.evaluator.values.dataset_schemas import ( + FieldMapping, + InputSchema, + ) + from nemo_platform.beta.evaluator.values.datasets import DatasetInput, DatasetRows + from nemo_platform.beta.evaluator.values.evidence import ( + CandidateEvidence, + CommandResult, + EvidenceDescriptor, + FilesystemDiff, + FilesystemEntry, + LocalFilesystemEvidence, + LogHandle, + TraceHandle, + WellKnownEvidenceKey, + parse_atif, + ) + from nemo_platform.beta.evaluator.values.metrics import ( + BLEU, + F1, + ROUGE, + AgentGoalAccuracy, + AnswerAccuracy, + ContextEntityRecall, + ContextPrecision, + ContextRecall, + ContextRelevance, + ExactMatch, + Faithfulness, + LLMJudge, + MetricBase, + NemoAgentToolkitRemote, + NoiseSensitivity, + NumberCheck, + Remote, + ResponseGroundedness, + ResponseRelevancy, + StringCheck, + ToolCallAccuracy, + ToolCalling, + TopicAdherence, + TunableRagEvaluator, + ) + from nemo_platform.beta.evaluator.values.models import Model, ModelRef, ReasoningParams + from nemo_platform.beta.evaluator.values.params import ( + InferenceParams, + RunConfig, + RunConfigOnline, + RunConfigOnlineModel, + ) + from nemo_platform.beta.evaluator.values.protocol import ( + BooleanValue, + CandidateOutput, + ContinuousScore, + DatasetRow, + DiscreteScore, + Label, + MetricDescriptor, + MetricDiagnostic, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, + MetricTypeName, + ) + from nemo_platform.beta.evaluator.values.results import ( + AggregatedMetricResult, + AggregateFieldName, + AggregateRangeScore, + AggregateRubricScore, + AggregateScore, + AggregateScoreBase, + DefaultAggregateFieldName, + EvaluationResult, + Histogram, + HistogramBin, + MetricScore, + Percentiles, + RowScore, + RubricScoreStat, + RubricScoreValue, + SampleResult, + ScoreStats, + ) + from nemo_platform.beta.evaluator.values.scores import ( + JSONScoreParser, + RangeScore, + RegexScoreParser, + RemoteScore, + Rubric, + RubricScore, + Score, + score_discriminator, + ) + + +# Re-exported name -> the submodule that defines it, relative to this package. Relative on +# purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting +# module paths, and a relative name has nothing to rewrite, so the mirror is correct by +# construction. Mirrors the TYPE_CHECKING block above, in the same order. +_LAZY_ATTRS: dict[str, str] = { + "Agent": ".agents", + "AgentBase": ".agents", + "GenericAgent": ".agents", + "NatAgentConfig": ".agents", + "NemoAgentToolkitAgent": ".agents", + "FinalMetrics": ".atif", + "Metrics": ".atif", + "Observation": ".atif", + "ObservationResult": ".atif", + "Step": ".atif", + "ToolCall": ".atif", + "Trajectory": ".atif", + "SecretRef": ".common", + "SupportedJobTypes": ".common", + "FieldMapping": ".dataset_schemas", + "InputSchema": ".dataset_schemas", + "DatasetInput": ".datasets", + "DatasetRows": ".datasets", + "CandidateEvidence": ".evidence", + "CommandResult": ".evidence", + "EvidenceDescriptor": ".evidence", + "FilesystemDiff": ".evidence", + "FilesystemEntry": ".evidence", + "LocalFilesystemEvidence": ".evidence", + "LogHandle": ".evidence", + "TraceHandle": ".evidence", + "WellKnownEvidenceKey": ".evidence", + "parse_atif": ".evidence", + "BLEU": ".metrics", + "F1": ".metrics", + "ROUGE": ".metrics", + "AgentGoalAccuracy": ".metrics", + "AnswerAccuracy": ".metrics", + "ContextEntityRecall": ".metrics", + "ContextPrecision": ".metrics", + "ContextRecall": ".metrics", + "ContextRelevance": ".metrics", + "ExactMatch": ".metrics", + "Faithfulness": ".metrics", + "LLMJudge": ".metrics", + "MetricBase": ".metrics", + "NemoAgentToolkitRemote": ".metrics", + "NoiseSensitivity": ".metrics", + "NumberCheck": ".metrics", + "Remote": ".metrics", + "ResponseGroundedness": ".metrics", + "ResponseRelevancy": ".metrics", + "StringCheck": ".metrics", + "ToolCallAccuracy": ".metrics", + "ToolCalling": ".metrics", + "TopicAdherence": ".metrics", + "TunableRagEvaluator": ".metrics", + "Model": ".models", + "ModelRef": ".models", + "ReasoningParams": ".models", + "InferenceParams": ".params", + "RunConfig": ".params", + "RunConfigOnline": ".params", + "RunConfigOnlineModel": ".params", + "BooleanValue": ".protocol", + "CandidateOutput": ".protocol", + "ContinuousScore": ".protocol", + "DatasetRow": ".protocol", + "DiscreteScore": ".protocol", + "Label": ".protocol", + "MetricDescriptor": ".protocol", + "MetricDiagnostic": ".protocol", + "MetricInput": ".protocol", + "MetricOutput": ".protocol", + "MetricOutputSpec": ".protocol", + "MetricResult": ".protocol", + "MetricTypeName": ".protocol", + "AggregatedMetricResult": ".results", + "AggregateFieldName": ".results", + "AggregateRangeScore": ".results", + "AggregateRubricScore": ".results", + "AggregateScore": ".results", + "AggregateScoreBase": ".results", + "DefaultAggregateFieldName": ".results", + "EvaluationResult": ".results", + "Histogram": ".results", + "HistogramBin": ".results", + "MetricScore": ".results", + "Percentiles": ".results", + "RowScore": ".results", + "RubricScoreStat": ".results", + "RubricScoreValue": ".results", + "SampleResult": ".results", + "ScoreStats": ".results", + "JSONScoreParser": ".scores", + "RangeScore": ".scores", + "RegexScoreParser": ".scores", + "RemoteScore": ".scores", + "Rubric": ".scores", + "RubricScore": ".scores", + "Score": ".scores", + "score_discriminator": ".scores", +} __all__ = [ "Agent", @@ -217,3 +337,30 @@ "TopicAdherence", "TunableRagEvaluator", ] + + +def __getattr__(name: str) -> object: + """Import the submodule that defines ``name`` on first access (PEP 562). + + An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only + falls back to importing a submodule when attribute lookup raises ``AttributeError``. + + A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, so + the real cause is not hidden behind a bogus "no attribute". The consequence is that + ``hasattr`` raises rather than returning ``False`` when a name's dependencies are missing; + catch ``ImportError`` around the access, or test membership against ``__all__``. + """ + submodule = _LAZY_ATTRS.get(name) + if submodule is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(_import_module(submodule, __name__), name) + globals()[name] = value # cache, so later lookups skip __getattr__ entirely + return value + + +def __dir__() -> list[str]: + # The declared surface plus any submodule the caller has already imported. Machinery is + # imported under a leading underscore so the filter keeps it out of autocomplete without a + # denylist; ``TYPE_CHECKING`` is the one exception, unaliased so type checkers recognise it. + public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"} + return sorted(set(__all__) | public) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py index 52551f88c8..05220d061d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py @@ -9,11 +9,13 @@ import math from collections.abc import Mapping from difflib import get_close_matches -from typing import Any, Literal, Self +from typing import TYPE_CHECKING, Any, Literal, Self -import pyarrow as pa from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_serializer +if TYPE_CHECKING: + import pyarrow as pa + from nemo_platform.beta.evaluator.values.protocol import MetricDiagnostic, MetricOutput, MetricResult ResultView = Literal["rows", "aggregate"] @@ -792,6 +794,12 @@ def to_table(self, view: ResultView = "rows") -> pa.Table: Returns: Table built from ``to_records(view=view)``. """ + # Imported here rather than at module scope: pyarrow (plus its numpy tail) costs ~31 MB + # RSS and 223 modules, this is its only runtime use in the module, and the module is on + # the agent_eval import path, which never calls this method. The return annotation is a + # string already (`from __future__ import annotations`), so it needs no import. + import pyarrow as pa + return pa.Table.from_pylist(self.to_records(view=view)) def to_pandas(self, view: ResultView = "rows"): diff --git a/uv.lock b/uv.lock index 055cd5b1ad..83f66a183a 100644 --- a/uv.lock +++ b/uv.lock @@ -4534,6 +4534,7 @@ name = "nemo-evaluator-sdk" version = "0.0.0" source = { editable = "packages/nemo_evaluator_sdk" } dependencies = [ + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jsonpath-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -4574,6 +4575,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "harbor", marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = ">=0.16.1" }, + { name = "httpx", specifier = ">=0.27.0,<1" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "jsonpath-ng", specifier = ">=1.7.0" }, { name = "jsonschema", specifier = ">=4.23.0" }, @@ -5192,6 +5194,7 @@ nemo-evaluator-plugin = [ { name = "typer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nemo-evaluator-sdk = [ + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jsonpath-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -5621,6 +5624,7 @@ requires-dist = [ { name = "httpx", marker = "extra == 'nemo-anonymizer-plugin'", specifier = ">=0.27" }, { name = "httpx", marker = "extra == 'nemo-auditor-plugin'", specifier = ">=0.27" }, { name = "httpx", marker = "extra == 'nemo-data-designer-plugin'", specifier = ">=0.27" }, + { name = "httpx", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.27.0,<1" }, { name = "httpx", marker = "extra == 'nemo-platform-sdk'", specifier = ">=0.23.0,<1" }, { name = "httpx", marker = "extra == 'nemo-safe-synthesizer-plugin'", specifier = ">=0.27.2" }, { name = "httpx", marker = "extra == 'plugins'", specifier = ">=0.27" }, @@ -6206,6 +6210,7 @@ aiohttp = [ { name = "httpx-aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nemo-evaluator-sdk = [ + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jsonpath-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -6248,6 +6253,7 @@ requires-dist = [ { name = "docker", specifier = ">=7.0.0" }, { name = "fsspec", specifier = ">=2023.1.0" }, { name = "httpx", specifier = ">=0.23.0,<1" }, + { name = "httpx", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.27.0,<1" }, { name = "httpx-aiohttp", marker = "extra == 'aiohttp'", specifier = ">=0.1.9" }, { name = "jinja2", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=3.1.6" }, { name = "jsonpath-ng", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=1.7.0" },