perf(evaluator): lazy load values module

.github/actions/changes/action.yaml

@@ -16,6 +16,9 @@ outputs:
   fabric:
     description: "'true' if the Fabric agent-eval runtime, its tests, or its dependency extra changed"
     value: ${{ steps.filter.outputs.fabric }}
+  evaluator-sdk-closure:
+    description: "'true' if anything that can change the agent_eval import closure changed"
+    value: ${{ steps.filter.outputs.evaluator-sdk-closure }}
   e2e:
     description: "'true' if any e2e test files changed"
     value: ${{ steps.filter.outputs.e2e }}
@@ -95,6 +98,13 @@ runs:
             - 'packages/nemo_evaluator_sdk/pyproject.toml'
             - '.github/workflows/ci.yaml'
             - '.github/actions/changes/action.yaml'
+          evaluator-sdk-closure:
+            # Paths that can change what `import ...harbor_runtime` loads or how the smoke job runs.
+            - 'packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/**'  # package-wide import closure
+            - 'packages/nemo_evaluator_sdk/pyproject.toml'  # declared [project] dependencies
+            - 'uv.lock'  # locked versions installed for the package
+            - '.github/workflows/ci.yaml'  # smoke job definition
+            - '.github/actions/changes/action.yaml'  # this filter (self-coverage)
           e2e:
             - 'e2e/**'
           docs:

.github/workflows/ci.yaml

@@ -44,6 +44,7 @@ jobs:
       test: ${{ steps.changes.outputs.test }}
       deps: ${{ steps.changes.outputs.deps }}
       fabric: ${{ steps.changes.outputs.fabric }}
+      evaluator-sdk-closure: ${{ steps.changes.outputs.evaluator-sdk-closure }}
       e2e: ${{ steps.changes.outputs.e2e }}
       docs: ${{ steps.changes.outputs.docs }}
       web-studio: ${{ steps.changes.outputs.web-studio }}
@@ -1000,6 +1001,50 @@ jobs:
           uv run --frozen --no-sync pytest \
             packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py -v
 
+  # Installs only nemo-evaluator-sdk (no workspace/extras/dev) and imports
+  # agent_eval.runtimes.harbor_runtime. Catches undeclared deps (workspace sync hides them)
+  # and broken lazy barrels that pull in the metric/execution stack.
+  evaluator-sdk-closure-smoke:
+    name: Evaluator SDK dependency-closure smoke (Linux, py${{ matrix.python-version }})
+    needs: [changes]
+    if: >
+      !cancelled() &&
+      (needs.changes.outputs.deps == 'true' || needs.changes.outputs.evaluator-sdk-closure == 'true')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        # Must satisfy uv.lock's requires-python (>=3.12,<3.14), which this job syncs against.
+        python-version: ["3.12"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+      - name: Install nemo-evaluator-sdk with no extras and no dev group
+        run: uv sync --frozen --package nemo-evaluator-sdk --no-dev
+      # pytest is not in the package-only sync; install the runner only (do not add SDK deps).
+      # WHY BOTH THIS JOB AND THE UNIT TEST:
+      # - Unit-test CI syncs the whole workspace: a sibling (e.g. nemo-platform-sdk) may already
+      #   provide httpx, so an undeclared SDK dep still imports — green here, broken for consumers
+      #   who only installed nemo-evaluator-sdk.
+      # - This job: `uv sync --package nemo-evaluator-sdk --no-dev` (no extras/siblings) so that
+      #   undeclared-dep case fails, then runs test_lazy_public_api (laziness assertions; one source
+      #   of truth — no duplicated heredoc probe).
+      - name: Run lazy-import closure unit test under the stripped install
+        run: |
+          uv pip install --python .venv/bin/python pytest
+          uv run --frozen --no-sync pytest \
+            packages/nemo_evaluator_sdk/tests/test_lazy_public_api.py::test_agent_eval_import_does_not_pull_the_execution_stack -v
+
   python-integration-test:
     name: Python integration tests
     needs: [policy-wasm]
@@ -1993,6 +2038,7 @@ jobs:
       - python-unit-test-tools
       - python-unit-test
       - fabric-wheel-smoke
+      - evaluator-sdk-closure-smoke
       # Enable if you want this required
       # - python-integration-test
       - require-nvskills

packages/nemo_evaluator_sdk/pyproject.toml

@@ -25,6 +25,7 @@ dependencies = [
   "pyarrow>=19.0.1",
   "pandas>=1.5.3",
   "openai>=1.61.0",
+  "httpx>=0.27.0,<1",
   "sacrebleu>=2.5.1",
   "rouge_score==0.1.2",
   "ragas==0.4.3",

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py

@@ -6,24 +6,29 @@
 The public surface resolves lazily (PEP 562). Importing this package must not drag in the
 execution/backend or metric stack: importing any submodule runs this module first, so eager
 re-exports made ``import nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime`` — all the
-optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, pyarrow, ...) instead
-of ~480, and turned every one of those transitive packages into an evaluation-time failure mode
-for the SDK-backed evaluator.
+optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, ...) instead of ~485, and
+turned every one of those transitive packages into an evaluation-time failure mode for the
+SDK-backed evaluator.
 
 Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a
 module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in.
 """
 
 # ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings.
 
-from importlib import import_module
-from importlib.metadata import PackageNotFoundError
+from importlib import import_module as _import_module
+from importlib.metadata import PackageNotFoundError as _PackageNotFoundError
 from importlib.metadata import version as _package_version
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     # Annotations and static analysis only; these must never execute at run time. Listing the
     # names in ``__all__`` is what marks them as re-exports for ruff and the type checkers.
+    #
+    # AGENTS.md ("Python Style notes") says not to import types under TYPE_CHECKING and to use a
+    # regular import "when possible". A regular import is exactly what this module exists to
+    # remove, so the exception is deliberate: these names are re-exports, not annotations, and
+    # every one of them resolves for real through ``__getattr__`` below.
     from nemo_evaluator_sdk.agent_stream_translation import (
         AgentStreamTranslation,
         AgentStreamTranslationContext,
@@ -97,10 +102,24 @@
         SecretRef,
     )
 
-try:
-    version = _package_version("nemo-evaluator-sdk")
-except PackageNotFoundError:
-    version = "0.0.0"
+
+def _resolve_version() -> str:
+    """Report the version of whichever distribution actually shipped this code.
+
+    ``nemo-evaluator-sdk`` is not published on its own — this package is also vendored into the
+    ``nemo-platform`` wheel as ``nemo_platform.beta.evaluator``. There the SDK distribution does
+    not exist, so resolving only that name reported ``"0.0.0"`` unconditionally and any telemetry
+    or support log that read it got a useless constant.
+    """
+    for distribution in ("nemo-evaluator-sdk", "nemo-platform"):
+        try:
+            return _package_version(distribution)
+        except _PackageNotFoundError:
+            continue
+    return "0.0.0"
+
+
+version = _resolve_version()
 
 # Re-exported name -> the submodule that defines it, relative to this package. Relative on
 # purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting
@@ -251,21 +270,29 @@
 def __getattr__(name: str) -> object:
     """Import the submodule that defines ``name`` on first access (PEP 562).
 
-    ``AttributeError`` is the required failure mode, not ``ImportError``: ``from pkg import sub``
-    only falls back to importing a submodule when attribute lookup raises ``AttributeError``, and
-    ``hasattr`` checks against this package depend on it too.
+    An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only
+    falls back to importing a submodule when attribute lookup raises ``AttributeError``.
+
+    A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, and
+    that is deliberate — ``ModuleNotFoundError: No module named 'sacrebleu'`` is far more useful
+    than an ``AttributeError`` claiming ``BLEUMetric`` does not exist. The consequence is that
+    ``hasattr(nemo_evaluator_sdk, name)`` raises rather than returning ``False`` when a name's
+    dependencies are not installed, since ``hasattr`` only swallows ``AttributeError``. To probe
+    for an optional part of the surface, catch ``ImportError`` around the access instead of using
+    ``hasattr``; to probe only for name membership, test against ``__all__``.
     """
     submodule = _LAZY_ATTRS.get(name)
     if submodule is None:
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-    value = getattr(import_module(submodule, __name__), name)
+    value = getattr(_import_module(submodule, __name__), name)
     globals()[name] = value  # cache, so later lookups skip __getattr__ entirely
     return value
 
 
 def __dir__() -> list[str]:
-    # The declared surface plus any submodule the caller has already imported. ``import_module``
-    # and ``TYPE_CHECKING`` are machinery for __getattr__, not API, so keep them out of
-    # autocomplete and inspect.getmembers.
-    public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING", "import_module"}
+    # The declared surface plus any submodule the caller has already imported. Everything this
+    # module needs for its own machinery is imported under a leading underscore so the filter
+    # below keeps it out of autocomplete and inspect.getmembers without a name-by-name denylist;
+    # ``TYPE_CHECKING`` is the one exception, kept unaliased so type checkers still recognise it.
+    public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"}
     return sorted(set(__all__) | public)

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py

@@ -53,9 +53,8 @@ class SandboxSDK:
 
 def _load_agents_sdk() -> SandboxSDK:
     try:
-        # The OpenAI Agents SDK ships under the `nemo-evaluator-sdk[agent-runtimes]` extra and is
-        # imported only when this Docker runtime is actually used, so it is absent from the default
-        # type-checking environment.
+        # The OpenAI Agents SDK is imported only when this Docker runtime is actually used, so it
+        # is absent from the default type-checking environment.
         from agents.run import RunConfig  # ty: ignore[unresolved-import]
         from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig  # ty: ignore[unresolved-import]
         from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE  # ty: ignore[unresolved-import]
@@ -68,7 +67,14 @@ def _load_agents_sdk() -> SandboxSDK:
         from agents import Runner  # ty: ignore[unresolved-import]
         from docker import from_env as docker_from_env
     except ImportError as exc:
-        raise RuntimeError("DockerSandboxAgentRuntime requires `nemo-evaluator-sdk[agent-runtimes]`") from exc
+        # Audience split is in the error text: SDK extras are not propagated into the
+        # vendored nemo_platform.beta.evaluator mirror.
+        raise RuntimeError(
+            "DockerSandboxAgentRuntime requires the openai-agents[docker] Python packages. "
+            "Standalone SDK: pip install 'nemo-evaluator-sdk[agent-runtimes]'. "
+            "Vendored nemo-platform.beta.evaluator (no SDK extras): "
+            "pip install 'openai-agents[docker]'"
+        ) from exc
 
     return SandboxSDK(
         Runner=Runner,

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_inference.py

@@ -27,7 +27,6 @@
 
 import httpx
 from httpx import Timeout
-from jsonpath_ng import parse as jsonpath_parse
 from pydantic import BaseModel, ConfigDict, Field
 
 from nemo_evaluator_sdk.agent_stream_translation import (
@@ -788,6 +787,10 @@ def _extract_jsonpath(
     required: bool = True,
 ) -> Any:
     """Extract a value from data using a JSONPath expression."""
+    # Imported here rather than at module scope: this is jsonpath_ng's only use in the module, and
+    # the module sits on the agent_eval run-time path via agent_eval/evaluator.py.
+    from jsonpath_ng import parse as jsonpath_parse
+
     expr = jsonpath_parse(path)
     matches = expr.find(data)
     if not matches:

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py

@@ -3,10 +3,12 @@
 
 """Aggregation data structures and computations for metric results."""
 
+from __future__ import annotations
+
 import math
 from collections import OrderedDict, defaultdict
 from collections.abc import Mapping, Sequence
-from typing import Protocol, cast, runtime_checkable
+from typing import TYPE_CHECKING, Protocol, cast, runtime_checkable
 
 from nemo_evaluator_sdk.metrics.protocol import (
     BooleanValue,
@@ -28,7 +30,11 @@
     RubricScoreStat,
     ScoreStats,
 )
-from nemo_evaluator_sdk.values.scores import RubricScore, Score
+
+if TYPE_CHECKING:
+    # Importing the score configuration module loads jsonschema. Aggregation's lightweight helpers
+    # (notably compute_percentiles) do not need it, so keep this typing-only edge deferred.
+    from nemo_evaluator_sdk.values.scores import Score
 
 
 def is_aggregateable_output_spec(output_spec: MetricOutputSpec) -> bool:
@@ -445,6 +451,8 @@ def aggregate_metrics(
 
 def rubric_definitions_from_scores(scores: Sequence[Score]) -> dict[str, list[RubricScoreStat]]:
     """Return declared rubric buckets keyed by score name."""
+    from nemo_evaluator_sdk.values.scores import RubricScore
+
     definitions: dict[str, list[RubricScoreStat]] = {}
     for score in scores:
         if not isinstance(score, RubricScore):