From 4664ecc119daf08aa12cb6ecf8e369d1a9a47474 Mon Sep 17 00:00:00 2001 From: Alec Khoury Date: Thu, 20 Aug 2026 10:58:05 -0500 Subject: [PATCH 1/2] feat(insights): migrate Analyst context to Ethos Use ETHOS.md as the preferred Analyst context and label README fallbacks as unvalidated. Signed-off-by: Alec Khoury --- docs/agents/insight-driven-optimization.mdx | 22 ++-- plugins/nemo-insights/README.md | 16 +-- plugins/nemo-insights/evaluation/adapters.py | 5 +- plugins/nemo-insights/evaluation/tau2run.py | 4 +- .../src/nemo_insights_plugin/analyst/agent.py | 30 ++--- .../src/nemo_insights_plugin/analyst/run.py | 9 +- .../src/nemo_insights_plugin/cli.py | 57 ++++----- .../nemo_insights_plugin/contracts/profile.py | 10 +- .../src/nemo_insights_plugin/jobs/analyze.py | 6 +- .../src/nemo_insights_plugin/preflight.py | 109 ++++++++++-------- .../src/nemo_insights_plugin/profile.py | 10 +- .../skills/nemo-analyst/SKILL.md | 11 +- .../tests/contracts/test_profile_contract.py | 20 ++-- .../tests/evaluation/test_adapters.py | 21 +++- .../nemo-insights/tests/test_analyst_agent.py | 17 ++- .../nemo-insights/tests/test_analyst_run.py | 10 +- .../nemo-insights/tests/test_cli_profile.py | 62 ++++++++-- .../tests/test_periodic_analysis.py | 6 +- plugins/nemo-insights/tests/test_preflight.py | 53 ++++++--- plugins/nemo-insights/tests/test_profile.py | 14 +-- 20 files changed, 309 insertions(+), 183 deletions(-) diff --git a/docs/agents/insight-driven-optimization.mdx b/docs/agents/insight-driven-optimization.mdx index db17b7c794..5541cecf78 100644 --- a/docs/agents/insight-driven-optimization.mdx +++ b/docs/agents/insight-driven-optimization.mdx @@ -94,14 +94,14 @@ dependencies: Both halves of the loop read a shared per-agent profile, `optimizer.yaml`, discovered by walking up from the current directory. The Analyst consumes only -its analysis subset (`agent`, `agent_spec`, `workspace`); the Experimenter +its analysis subset (`agent`, `ethos`, `workspace`); the Experimenter validates the full schema: ```yaml agent: research-agent -agent_spec: AGENT-SPEC.md # optional; falls back to AGENT-SPEC.md, then README.md -workspace: default # optional; defaults to "default" -agent_source: . # local dir or git URL with optional @ref +ethos: ETHOS.md # optional; source order is ETHOS.md, then README.md +workspace: default # optional; defaults to "default" +agent_source: . # local dir or git URL with optional @ref task_template: ./task-template datasets: train: ./harbor_eval/dataset/train @@ -113,6 +113,10 @@ a profile is found, without replacing variables already set in the shell. `NMP_BASE_URL` is the base-URL environment variable for this workflow, and `--base-url` takes precedence over it. +The Analyst labels `README.md` content exactly +`README analysis context (not ETHOS)`. It treats that content as unvalidated +repository context and doesn't validate either file with `parse_ethos()`. + ## How It Works ### The Loop @@ -214,8 +218,8 @@ Its method is to survey sessions broadly, gather evidence (starting from negative feedback and error spans), cluster similar failures across many sessions, check for existing Insights, then emit a single result containing new Insights and evidence appended to existing ones. Give the Analyst an optional -agent spec (`--agent-spec AGENT-SPEC.md`) so it can flag divergence from -intended behavior. +Ethos file (`--ethos ETHOS.md`) so it can flag divergence from intended +behavior. ### Experimenter @@ -354,7 +358,7 @@ nemo agents analyst run \ Useful flags: -- `--agent-spec AGENT-SPEC.md`: Append a spec so the Analyst can flag +- `--ethos ETHOS.md`: Append Ethos content so the Analyst can flag divergence from intended behavior. - `--insights-file-output tmp/insights.yaml`: Read and write Insights from a specific local YAML file. Trace reads still hit `--base-url`. @@ -541,7 +545,7 @@ Run the Analyst for one analysis of an agent's traces. | Flag | Required | Default | Description | |------|----------|---------|-------------| | `--agent` | yes, unless a profile supplies it | profile `agent` | Agent under test the Analyst should focus on. | -| `--agent-spec` | no | profile `agent_spec`, else `AGENT-SPEC.md` or `README.md` beside the profile | Path to a Markdown spec for the AUT. | +| `--ethos` | no | profile `ethos`, else `ETHOS.md` or `README.md` beside the profile | Path to Ethos or unvalidated README analysis context for the AUT. | | `--workspace` | no | profile `workspace`, else `default` | Workspace to operate in. | | `--base-url` | no | `NMP_BASE_URL`, else `http://localhost:8080` | Running platform instance the Analyst's tools call. | | `--profile` | no | discovered by walking up from the current working directory | Path to `optimizer.yaml`. | @@ -570,7 +574,7 @@ Run the local Experimenter loop. | `--insight-id` | no | Not applicable | Select an exact ID, exact title, or zero-based index from a local multi-Insight file. | | `--no-insight` | no | off | Run against a dataset directly rather than guided by an Insight. | | `--agent` | no | profile `agent_source` | Baseline agent override: a local directory or a git URL with optional ref (`...repo.git@main`). A git source records provenance and enables opening a draft PR for the winner. | -| `--agent-spec` | no | profile `agent_spec` | Location of a Markdown file describing the AUT. | +| `--ethos` | no | profile `ethos` | Location of Ethos or unvalidated analysis context for the AUT. | | `--train-dataset` | yes, unless the profile supplies it | profile `datasets.train` | Train dataset. Local path for the Harbor evaluator. | | `--validation-dataset` | yes, unless the profile supplies it | profile `datasets.validation` | Validation dataset. Local path for the Harbor evaluator. | | `--task-template` | required with an Insight | profile `task_template` | Evaluator-specific task-template URI, used to build the Insight-specific evaluation suite. | diff --git a/plugins/nemo-insights/README.md b/plugins/nemo-insights/README.md index d1c30c7c67..94203e5d2b 100644 --- a/plugins/nemo-insights/README.md +++ b/plugins/nemo-insights/README.md @@ -34,20 +34,22 @@ The profile contract consumed by Insights is deliberately small: ```yaml agent: research-agent -agent_spec: AGENT-SPEC.md # optional -workspace: default # optional; defaults to "default" +ethos: ETHOS.md # optional +workspace: default # optional; defaults to "default" ``` -Only `agent`, `agent_spec`, and `workspace` are consumed by Insights. +Only `agent`, `ethos`, and `workspace` are consumed by Insights. Unknown experiment-owned fields are ignored, while the reserved `profile_dir` -field is rejected. `agent` is required. Relative `agent_spec` paths are -resolved relative to the profile. When it is omitted, Insights looks for -`AGENT-SPEC.md`, then `README.md`, beside the profile. +field is rejected. `agent` is required. Relative `ethos` paths resolve from the +profile directory. When `ethos` is omitted, Insights looks for `ETHOS.md`, then +`README.md`, beside the profile. The Analyst labels README content exactly +`README analysis context (not ETHOS)`. It doesn't validate either file with +`parse_ethos()`. An adjacent `.env` is loaded when a profile is found, without replacing variables already set in the shell. For this shared profile workflow, `NMP_BASE_URL` is the only base-URL environment variable. Resolution order is -explicit command-line flags, then profile values (for `agent`, `agent_spec`, +explicit command-line flags, then profile values (for `agent`, `ethos`, and `workspace`) or `NMP_BASE_URL` (for the base URL), then the built-in defaults. `--base-url` takes precedence over `NMP_BASE_URL`. diff --git a/plugins/nemo-insights/evaluation/adapters.py b/plugins/nemo-insights/evaluation/adapters.py index 3e82e452f5..43cd3d1df8 100644 --- a/plugins/nemo-insights/evaluation/adapters.py +++ b/plugins/nemo-insights/evaluation/adapters.py @@ -90,7 +90,7 @@ async def analyze( client = self._basic_auth_client() if cfg.get("auth") == "basic" else make_client(str(cfg["base_url"])) return await run_analyst( agent=cfg["agent"], - agent_spec=None, + ethos=None, workspace=cfg["workspace"], base_url=cfg["base_url"], client=client, @@ -297,7 +297,8 @@ async def analyze( ) return await run_analyst( agent=str(record["agent"]), - agent_spec=policy, + ethos=policy, + ethos_label="Benchmark policy analysis context (not ETHOS)", workspace=workspace, base_url=str(record["base_url"]), client=make_client(str(record["base_url"])), diff --git a/plugins/nemo-insights/evaluation/tau2run.py b/plugins/nemo-insights/evaluation/tau2run.py index cfb4fe2448..5b1d727dcf 100644 --- a/plugins/nemo-insights/evaluation/tau2run.py +++ b/plugins/nemo-insights/evaluation/tau2run.py @@ -133,12 +133,12 @@ def load_tasks(data_dir: Path, domain: str) -> dict[str, dict]: def read_policy(data_dir: Path, domain: str) -> str | None: - """Return the domain policy markdown (the analyst's agent_spec), or None. + """Return the domain policy Markdown as unvalidated Analyst context. A tau2 checkout nests domains under ``tau2/domains//``; some data dirs are flat (``domains//``). For each layout, tries ``policy.md`` then ``main_policy.md`` (Telecom uses the latter). Returns the first file - found, else ``None`` (the analyst then runs without the spec). + found, else ``None`` (the Analyst then runs without this context). """ for base in (data_dir / "tau2" / "domains", data_dir / "domains"): domain_dir = base / domain diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py index e9f27e8af3..e2f4806885 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py @@ -18,8 +18,8 @@ The result is delivered through Nooa's ``return_result`` helper and validated against the ``AnalystResult`` schema. -The analyst's persona, task, the agent-under-test name, and the optional AUT -spec are all formatted into the instructions by ``build_analyst_agent``; the +The analyst's persona, task, agent-under-test name, and optional Ethos content +are all formatted into the instructions by ``build_analyst_agent``; the run is seeded with only the minimal ``KICKOFF`` request. The per-run config the methods need is carried in :class:`~nemo_insights_plugin.analyst.deps.AnalystDeps`. Workspace and base URL aren't in the instructions because the methods are already @@ -50,7 +50,7 @@ # # This is the analyst context prompt and the only long-form prompt # the analyst gets — there is no separate user-message brief. ``{agent}`` is -# formatted in by ``build_analyst_agent`` and the optional AUT spec is appended +# formatted in by ``build_analyst_agent`` and optional Ethos content is appended # as the final paragraph. Nooa owns the CodeAct protocol and method catalog, so # this text covers only the analyst's persona, # principles, and method — it deliberately does not document the tools or @@ -146,13 +146,12 @@ """ -AGENT_SPEC_HEADER = """ -## Agent Spec +ETHOS_HEADER = """ +## {ethos_label} -Use this as the contract for what the agent is supposed to do, what -success looks like, and what behavior should be flagged as divergence. -Flag agent divergence from the spec. The spec was authored by the -developer of the application and should be considered the purpose and goals. +Use this content to understand what the agent is supposed to do and what +success looks like. Content labeled ETHOS is the agent's contract. Content +labeled README analysis context (not ETHOS) is unvalidated repository context. """ KICKOFF = ( @@ -170,7 +169,8 @@ def __init__( *, deps: AnalystDeps, agent: str, - agent_spec: str | None = None, + ethos: str | None = None, + ethos_label: str = "ETHOS", **kwargs: Any, ) -> None: super().__init__(llm=kwargs.pop("llm", None) or get_default_model(), **kwargs) @@ -183,8 +183,8 @@ def __init__( ) instructions = INSTRUCTIONS.format(agent=agent) - if agent_spec and agent_spec.strip(): - instructions = f"{instructions}\n{AGENT_SPEC_HEADER}\n\n{agent_spec.strip()}\n" + if ethos and ethos.strip(): + instructions = f"{instructions}\n{ETHOS_HEADER.format(ethos_label=ethos_label)}\n\n{ethos.strip()}\n" self.context["analyst_instructions"] = instructions async def fetch_spans( @@ -360,7 +360,8 @@ def build_analyst_agent( *, deps: AnalystDeps, agent: str, - agent_spec: str | None = None, + ethos: str | None = None, + ethos_label: str = "ETHOS", llm: UnifiedLLM | None = None, **kwargs: Any, ) -> Analyst: @@ -368,7 +369,8 @@ def build_analyst_agent( return Analyst( deps=deps, agent=agent, - agent_spec=agent_spec, + ethos=ethos, + ethos_label=ethos_label, llm=llm, **kwargs, ) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py index 36a1fb445c..64fd47111f 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py @@ -42,7 +42,7 @@ class ClientConstructionError(Exception): async def run_analyst( *, agent: str, - agent_spec: str | None, + ethos: str | None, workspace: str, base_url: str | None, client: AsyncNeMoPlatform, @@ -52,6 +52,7 @@ async def run_analyst( since: datetime | None = None, evaluation_id: str | None = None, model_refs: ConfiguredModelRefs | None = None, + ethos_label: str = "ETHOS", ) -> str: """Build and run the analyst agent against an agent's telemetry. @@ -60,7 +61,7 @@ async def run_analyst( Args: agent: Agent under test. - agent_spec: Optional markdown spec content for the agent under test. + ethos: Optional Ethos or analysis-context Markdown for the agent under test. workspace: Platform workspace. base_url: Platform base URL. ``None`` uses the active platform context. client: Platform client to use. This function closes it before returning. @@ -74,6 +75,7 @@ async def run_analyst( evaluation_id: Optional run scope; AND-pinned onto every span read. model_refs: Optional explicit default/fast Model Entity IDs. Unset uses the active Platform CLI context. + ethos_label: Source label for the supplied content. """ observability = None model_clients: ConfiguredModelClients | None = None @@ -104,7 +106,8 @@ async def run_analyst( analyst = build_analyst_agent( deps=deps, agent=agent, - agent_spec=agent_spec, + ethos=ethos, + ethos_label=ethos_label, ) result = await _run_agent(analyst, verbose=verbose) return await backend.persist_result(workspace=workspace, agent=agent, result=result) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/cli.py b/plugins/nemo-insights/src/nemo_insights_plugin/cli.py index 16af1bcdbf..471ccf29b7 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/cli.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/cli.py @@ -36,9 +36,9 @@ check_environment, check_models, check_profile, - read_agent_spec, + read_ethos, ) -from nemo_insights_plugin.profile import AnalysisProfile, load_profile, pick_agent_spec +from nemo_insights_plugin.profile import AnalysisProfile, load_profile, pick_ethos from nemo_platform import NeMoPlatformError from nemo_platform_plugin.cli import NemoCLI from nemo_platform_plugin.nooa_model_client import configured_model_refs @@ -51,12 +51,13 @@ @dataclass(frozen=True) class _ResolvedAnalysis: agent: str - agent_spec: str | None + ethos: str | None + ethos_label: str | None workspace: str base_url: str insights_output: Path | None profile_dir: Path | None - spec_checks: tuple[CheckResult, ...] + ethos_checks: tuple[CheckResult, ...] def _load_profile_or_error(profile_path: Path | None) -> tuple[AnalysisProfile | None, str | None]: @@ -99,7 +100,7 @@ def _one_line_error(exc: BaseException) -> str: def _resolve_analysis( *, agent: str | None, - agent_spec: Path | None, + ethos: Path | None, workspace: str | None, base_url: str | None, profile_path: Path | None, @@ -121,26 +122,27 @@ def _resolve_analysis( else: resolved_workspace = profile.workspace if profile is not None else DEFAULT_WORKSPACE - spec_path = agent_spec - spec_error: str | None = None - if spec_path is None and profile is not None: + ethos_path = ethos + ethos_error: str | None = None + if ethos_path is None and profile is not None: try: - spec_path = pick_agent_spec(profile) + ethos_path = pick_ethos(profile) except ProfileError as exc: - spec_error = str(exc) - spec_content, spec_checks = read_agent_spec(spec_path, spec_error) + ethos_error = str(exc) + ethos_content, ethos_label, ethos_checks = read_ethos(ethos_path, ethos_error) resolved_base_url = resolve_base_url(base_url) validate_insights_file(insights_output) return _ResolvedAnalysis( agent=resolved_agent, - agent_spec=spec_content, + ethos=ethos_content, + ethos_label=ethos_label, workspace=resolved_workspace, base_url=resolved_base_url, insights_output=insights_output, profile_dir=profile.profile_dir if profile is not None else None, - spec_checks=tuple(spec_checks), + ethos_checks=tuple(ethos_checks), ) @@ -168,7 +170,7 @@ def _prepare_mirror(insights_output: Path | None) -> Path | None: async def _run_analysis(analysis: _ResolvedAnalysis, *, verbose: bool) -> str: - checks = list(analysis.spec_checks) + checks = list(analysis.ethos_checks) checks.extend(check_models()) _preflight_or_exit(checks) @@ -180,7 +182,8 @@ async def _run_analysis(analysis: _ResolvedAnalysis, *, verbose: bool) -> str: raise ClientConstructionError(str(exc)) from None return await run_analyst( agent=analysis.agent, - agent_spec=analysis.agent_spec, + ethos=analysis.ethos, + ethos_label=analysis.ethos_label or "ETHOS", workspace=analysis.workspace, base_url=analysis.base_url, client=client, @@ -212,10 +215,10 @@ def analyze( "--agent", help="Name of the agent (agent under test) the analyst should focus on.", ), - agent_spec: Path | None = typer.Option( + ethos: Path | None = typer.Option( None, - "--agent-spec", - help="Path to a markdown file describing the agent under test (its spec).", + "--ethos", + help="Path to Markdown content that describes the agent under test.", exists=True, readable=True, ), @@ -259,8 +262,8 @@ def analyze( ) -> None: """Run the analyst agent against a running NMP instance. - Builds the analyst agent with ``--agent`` (and optional - ``--agent-spec``) formatted into its instructions and tools scoped + Builds the analyst agent with ``--agent`` and optional ``--ethos`` content + formatted into its instructions and tools scoped to ``--agent`` / ``--workspace`` / ``--base-url``, runs it, and prints whatever the agent returns. Insights are written to the platform, and mirrored to ``--insights-file-output`` when given. @@ -268,7 +271,7 @@ def analyze( try: analysis = _resolve_analysis( agent=agent, - agent_spec=agent_spec, + ethos=ethos, workspace=workspace, base_url=base_url, profile_path=profile_path, @@ -302,18 +305,18 @@ def doctor( profile, profile_error = _load_profile_or_error(profile_path) except ProfileError as exc: profile, profile_error = None, str(exc) - spec_path: Path | None = None - spec_error: str | None = None + ethos_path: Path | None = None + ethos_error: str | None = None if profile is not None: try: - spec_path = pick_agent_spec(profile) + ethos_path = pick_ethos(profile) except ProfileError as exc: - spec_error = str(exc) - _, spec_results = read_agent_spec(spec_path, spec_error) + ethos_error = str(exc) + _, _, ethos_results = read_ethos(ethos_path, ethos_error) async def _flow() -> list[CheckResult]: results = check_profile(profile, profile_error) - results.extend(spec_results) + results.extend(ethos_results) results.extend( await check_environment( agent=profile.agent if profile is not None else None, diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py b/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py index ad1682f17f..8ba5e0c012 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py @@ -13,7 +13,7 @@ PROFILE_FILENAME = "optimizer.yaml" DEFAULT_BASE_URL = "http://localhost:8080" -_AGENT_SPEC_FILENAMES = ("AGENT-SPEC.md", "README.md") +_ETHOS_FILENAMES = ("ETHOS.md", "README.md") ProfileModel = TypeVar("ProfileModel", bound=BaseModel) @@ -92,14 +92,14 @@ def load_env_file(path: Path, env: MutableMapping[str, str] = os.environ) -> lis return loaded -def resolve_agent_spec_path(profile_dir: Path, configured: str | None) -> Path | None: - """Resolve a configured agent spec or the conventional profile-local file.""" +def resolve_ethos_path(profile_dir: Path, configured: str | None) -> Path | None: + """Resolve configured Ethos content or a conventional profile-local file.""" if configured is not None: path = resolve_profile_path(configured, profile_dir) if not path.is_file(): - raise ProfileError(f"Profile agent_spec {configured!r} does not exist (resolved to {path})") + raise ProfileError(f"Profile ethos {configured!r} does not exist (resolved to {path})") return path - for name in _AGENT_SPEC_FILENAMES: + for name in _ETHOS_FILENAMES: candidate = profile_dir / name if candidate.is_file(): return candidate diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/jobs/analyze.py b/plugins/nemo-insights/src/nemo_insights_plugin/jobs/analyze.py index 294cb65a59..ed670e2b2b 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/jobs/analyze.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/jobs/analyze.py @@ -43,9 +43,9 @@ class AnalyzeSpec(BaseModel): model_config = ConfigDict(extra="forbid") agent: str = Field(description="Agent under test.") - agent_spec: str | None = Field( + ethos: str | None = Field( default=None, - description="Optional markdown spec content for the agent under test.", + description="Optional Ethos Markdown for the agent under test.", ) base_url: str | None = Field( default=None, @@ -142,7 +142,7 @@ def run( report = asyncio.run( run_analyst( agent=spec.agent, - agent_spec=spec.agent_spec, + ethos=spec.ethos, workspace=ctx.workspace, base_url=spec.base_url, client=async_client, diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py b/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py index e73d9ab673..4a1bb47ce5 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py @@ -95,61 +95,78 @@ def check_profile( ] -def check_agent_spec( - spec_path: Path | None, - spec_error: str | None, +def check_ethos( + ethos_path: Path | None, + ethos_error: str | None, ) -> list[CheckResult]: - """Check the optional agent-spec artifact, including explicit UTF-8 readability.""" - return read_agent_spec(spec_path, spec_error)[1] + """Check the optional Ethos content, including explicit UTF-8 readability.""" + return read_ethos(ethos_path, ethos_error)[2] -def read_agent_spec( - spec_path: Path | None, - spec_error: str | None, -) -> tuple[str | None, list[CheckResult]]: - """Read the optional agent spec as UTF-8 and return its readiness check.""" - if spec_error is not None: - return None, [ - CheckResult( - name="agent-spec", - group="artifacts", - status="fail", - severity="required", - message=spec_error, - ) - ] - if spec_path is None: - return None, [ - CheckResult( - name="agent-spec", - group="artifacts", - status="pass", - severity="advisory", - message="agent spec omitted (optional)", - ) - ] +def read_ethos( + ethos_path: Path | None, + ethos_error: str | None, +) -> tuple[str | None, str | None, list[CheckResult]]: + """Read optional Ethos or README context and return its source label and check.""" + if ethos_error is not None: + return ( + None, + None, + [ + CheckResult( + name="ethos", + group="artifacts", + status="fail", + severity="required", + message=ethos_error, + ) + ], + ) + if ethos_path is None: + return ( + None, + None, + [ + CheckResult( + name="ethos", + group="artifacts", + status="pass", + severity="advisory", + message="Ethos omitted (optional)", + ) + ], + ) try: - content = spec_path.read_text(encoding="utf-8") + content = ethos_path.read_text(encoding="utf-8") except (OSError, UnicodeError) as exc: - return None, [ + return ( + None, + None, + [ + CheckResult( + name="ethos", + group="artifacts", + status="fail", + severity="required", + message=f"Could not read Ethos content {ethos_path} as UTF-8: {exc}", + hint="ensure the file is readable and encoded as UTF-8", + ) + ], + ) + label = "README analysis context (not ETHOS)" if ethos_path.name == "README.md" else "ETHOS" + return ( + content, + label, + [ CheckResult( - name="agent-spec", + name="ethos", group="artifacts", - status="fail", + status="pass", severity="required", - message=f"Could not read agent spec {spec_path} as UTF-8: {exc}", - hint="ensure the file is readable and encoded as UTF-8", + message=f"{label} readable at {ethos_path}", ) - ] - return content, [ - CheckResult( - name="agent-spec", - group="artifacts", - status="pass", - severity="required", - message=f"agent spec readable at {spec_path}", - ) - ] + ], + ) def check_models() -> list[CheckResult]: diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/profile.py b/plugins/nemo-insights/src/nemo_insights_plugin/profile.py index f5dd08b282..7f7c6a7ff7 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/profile.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/profile.py @@ -5,7 +5,7 @@ from pathlib import Path -from nemo_insights_plugin.contracts.profile import load_profile_model, resolve_agent_spec_path +from nemo_insights_plugin.contracts.profile import load_profile_model, resolve_ethos_path from pydantic import BaseModel, ConfigDict, Field @@ -15,7 +15,7 @@ class AnalysisProfile(BaseModel): model_config = ConfigDict(extra="ignore") agent: str = Field(min_length=1) - agent_spec: str | None = None + ethos: str | None = None workspace: str = "default" profile_dir: Path @@ -25,6 +25,6 @@ def load_profile(path: Path) -> AnalysisProfile: return load_profile_model(path, AnalysisProfile) -def pick_agent_spec(profile: AnalysisProfile) -> Path | None: - """Resolve the profile's configured or conventional agent spec.""" - return resolve_agent_spec_path(profile.profile_dir, profile.agent_spec) +def pick_ethos(profile: AnalysisProfile) -> Path | None: + """Resolve the profile's configured or conventional Ethos content.""" + return resolve_ethos_path(profile.profile_dir, profile.ethos) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md b/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md index 0ebaf9e407..1c5eade70a 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md +++ b/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md @@ -69,10 +69,13 @@ The Analyst reads telemetry; it cannot create it. Confirm all three: it names what is missing and how to set it. Don't reach for the Experimentalist's configuration; that is a different contract. -An `AGENT-SPEC.md` is optional but makes the Analyst materially better. It +An `ETHOS.md` file is optional but makes the Analyst materially better. It carries the intent behind the agent — what it is for, its constraints, what counts as success — none of which is recoverable from code or traces, so -without it the Analyst can only judge an agent against itself. +without it the Analyst can only judge an agent against itself. If the profile +has no `ETHOS.md` file, the Analyst uses `README.md` as unvalidated +`README analysis context (not ETHOS)`. It doesn't validate either file with +`parse_ethos()`. ## Pre-flight @@ -93,12 +96,12 @@ only ever warn. nemo agents analyst run --agent --workspace ``` -Add `--agent-spec AGENT-SPEC.md` to tell it what the agent is supposed to do, +Add `--ethos ETHOS.md` to tell it what the agent is supposed to do, and `--verbose` to stream its tool calls and reasoning to stderr. Expect several minutes; it surveys many sessions before drilling into any of them. From an agent directory, an `optimizer.yaml` profile supplies `agent`, -`workspace`, and `agent_spec`, so the flags above become optional: +`workspace`, and `ethos`, so these flags become optional: ```bash nemo agents analyst run diff --git a/plugins/nemo-insights/tests/contracts/test_profile_contract.py b/plugins/nemo-insights/tests/contracts/test_profile_contract.py index b1d19ca5ff..90e14b3d48 100644 --- a/plugins/nemo-insights/tests/contracts/test_profile_contract.py +++ b/plugins/nemo-insights/tests/contracts/test_profile_contract.py @@ -11,8 +11,8 @@ discover_profile, load_env_file, load_profile_model, - resolve_agent_spec_path, resolve_base_url, + resolve_ethos_path, resolve_profile_path, ) from pydantic import BaseModel, ConfigDict @@ -134,10 +134,10 @@ def test_load_env_file_wraps_permission_errors( path.write_text("KEY=value\n", encoding="utf-8") original_read_text = Path.read_text - def deny(candidate: Path, *args: object, **kwargs: object) -> str: + def deny(candidate: Path, encoding: str | None = None, errors: str | None = None) -> str: if candidate == path: raise PermissionError("permission denied") - return original_read_text(candidate, *args, **kwargs) + return original_read_text(candidate, encoding=encoding, errors=errors) monkeypatch.setattr(Path, "read_text", deny) @@ -147,18 +147,18 @@ def deny(candidate: Path, *args: object, **kwargs: object) -> str: assert exc_info.value.__cause__ is None -def test_resolve_agent_spec_uses_configured_then_conventional_precedence(tmp_path: Path) -> None: +def test_resolve_ethos_uses_configured_then_conventional_precedence(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text("# Readme", encoding="utf-8") - assert resolve_agent_spec_path(tmp_path, None) == readme + assert resolve_ethos_path(tmp_path, None) == readme - spec = tmp_path / "AGENT-SPEC.md" - spec.write_text("# Spec", encoding="utf-8") - assert resolve_agent_spec_path(tmp_path, None) == spec - assert resolve_agent_spec_path(tmp_path, "./README.md") == readme.resolve() + ethos = tmp_path / "ETHOS.md" + ethos.write_text("# Ethos", encoding="utf-8") + assert resolve_ethos_path(tmp_path, None) == ethos + assert resolve_ethos_path(tmp_path, "./README.md") == readme.resolve() with pytest.raises(ProfileError, match="does not exist"): - resolve_agent_spec_path(tmp_path, "./missing.md") + resolve_ethos_path(tmp_path, "./missing.md") def test_resolve_base_url_uses_only_explicit_nmp_and_default() -> None: diff --git a/plugins/nemo-insights/tests/evaluation/test_adapters.py b/plugins/nemo-insights/tests/evaluation/test_adapters.py index ab0d095c79..02ae893a12 100644 --- a/plugins/nemo-insights/tests/evaluation/test_adapters.py +++ b/plugins/nemo-insights/tests/evaluation/test_adapters.py @@ -144,7 +144,7 @@ async def fake_run_analyst(**kwargs): assert calls["agent"] == "a" assert calls["workspace"] == "w" assert calls["base_url"] == "u" - assert calls["agent_spec"] is None + assert calls["ethos"] is None assert calls["client"] is built_client @@ -354,11 +354,23 @@ async def test_benchmark_analyze_uses_record(monkeypatch, tmp_path): seen: dict[str, object] = {} async def fake_run_analyst( - *, agent, agent_spec, workspace, base_url, client, insights_output, local_only, verbose, since, evaluation_id + *, + agent, + ethos, + ethos_label, + workspace, + base_url, + client, + insights_output, + local_only, + verbose, + since, + evaluation_id, ): seen.update( agent=agent, - agent_spec=agent_spec, + ethos=ethos, + ethos_label=ethos_label, workspace=workspace, base_url=base_url, evaluation_id=evaluation_id, @@ -384,7 +396,8 @@ async def fake_run_analyst( assert out == "REPORT-OK" assert ran_tau2["v"] is False # analyze never runs tau2 assert seen["agent"] == "tau2-airline" - assert seen["agent_spec"] == "POLICY" + assert seen["ethos"] == "POLICY" + assert seen["ethos_label"] == "Benchmark policy analysis context (not ETHOS)" assert seen["workspace"] == "tau2-airline" # the stable REALISTIC workspace, never the oracle one assert seen["evaluation_id"] == "tau2-airline-20260626-000000-abcd" # run-scoped assert seen["base_url"] == "http://localhost:8080" diff --git a/plugins/nemo-insights/tests/test_analyst_agent.py b/plugins/nemo-insights/tests/test_analyst_agent.py index 563956a033..5cec3ca465 100644 --- a/plugins/nemo-insights/tests/test_analyst_agent.py +++ b/plugins/nemo-insights/tests/test_analyst_agent.py @@ -51,7 +51,7 @@ async def test_nooa_codeact_returns_typed_analyst_result_and_receives_prompt() - analyst = build_analyst_agent( deps=AnalystDeps(agent="target-agent", workspace="private-workspace"), agent="target-agent", - agent_spec="# Expected behavior\nBe accurate.", + ethos="# Expected behavior\nBe accurate.", llm=fake, ) @@ -63,6 +63,7 @@ async def test_nooa_codeact_returns_typed_analyst_result_and_receives_prompt() - assert result.updated_insights == [] rendered_messages = json.dumps(fake.last_messages) assert "target-agent" in rendered_messages + assert "## ETHOS" in rendered_messages assert "Expected behavior" in rendered_messages assert "One method, two modes" in rendered_messages assert "private-workspace" not in rendered_messages @@ -70,6 +71,20 @@ async def test_nooa_codeact_returns_typed_analyst_result_and_receives_prompt() - assert {tool.name for tool in fake.last_tools} == {"execute_python", "return_result"} +def test_readme_context_header_does_not_claim_ethos() -> None: + analyst = build_analyst_agent( + deps=AnalystDeps(agent="target-agent", workspace="workspace"), + agent="target-agent", + ethos="# Repository context", + ethos_label="README analysis context (not ETHOS)", + llm=FakeLLMClient(), + ) + + instructions = cast(Any, analyst.context)["analyst_instructions"] + assert "## README analysis context (not ETHOS)" in instructions + assert "## ETHOS\n" not in instructions + + class _SpanBackend: def __init__(self) -> None: self.kwargs: dict[str, object] | None = None diff --git a/plugins/nemo-insights/tests/test_analyst_run.py b/plugins/nemo-insights/tests/test_analyst_run.py index 8a347f66d3..0adf6d7cd4 100644 --- a/plugins/nemo-insights/tests/test_analyst_run.py +++ b/plugins/nemo-insights/tests/test_analyst_run.py @@ -75,7 +75,7 @@ async def test_injected_client_is_used_and_closed(monkeypatch: pytest.MonkeyPatc report = await run_module.run_analyst( agent="agent", - agent_spec=None, + ethos=None, workspace="workspace", base_url="https://platform", client=cast(AsyncNeMoPlatform, client), @@ -85,6 +85,8 @@ async def test_injected_client_is_used_and_closed(monkeypatch: pytest.MonkeyPatc assert seen["backend_client"] is client build_kwargs = cast(dict[str, object], seen["build_kwargs"]) assert cast(AnalystDeps, build_kwargs["deps"]).backend is not None + assert build_kwargs["ethos"] is None + assert build_kwargs["ethos_label"] == "ETHOS" assert seen["model_client"] is client model_clients = cast(ConfiguredModelClients, seen["model_clients"]) assert cast(FakeModelClient, model_clients.default).closed @@ -112,7 +114,7 @@ def raising_backend(*, client: FakeClient, insights_output: str | None, local_on with pytest.raises(RuntimeError, match="backend failed"): await run_module.run_analyst( agent="agent", - agent_spec=None, + ethos=None, workspace="workspace", base_url="https://platform", client=cast(AsyncNeMoPlatform, client), @@ -133,7 +135,7 @@ async def raising_model_resolution(client: object, refs: object) -> ConfiguredMo with pytest.raises(RuntimeError, match="model resolution failed"): await run_module.run_analyst( agent="agent", - agent_spec=None, + ethos=None, workspace="workspace", base_url="https://platform", client=cast(AsyncNeMoPlatform, client), @@ -190,7 +192,7 @@ def shutdown(self) -> None: with pytest.raises(RuntimeError, match="shutdown failed"): await run_module.run_analyst( agent="agent", - agent_spec=None, + ethos=None, workspace="workspace", base_url="https://platform", client=cast(AsyncNeMoPlatform, client), diff --git a/plugins/nemo-insights/tests/test_cli_profile.py b/plugins/nemo-insights/tests/test_cli_profile.py index 75224b8134..c450b11708 100644 --- a/plugins/nemo-insights/tests/test_cli_profile.py +++ b/plugins/nemo-insights/tests/test_cli_profile.py @@ -92,7 +92,7 @@ def profile_tree(tmp_path: Path) -> Path: "workspace: flight-workspace\n", encoding="utf-8", ) - (tmp_path / "AGENT-SPEC.md").write_text("# Flight planner", encoding="utf-8") + (tmp_path / "ETHOS.md").write_text("# Flight planner", encoding="utf-8") return tmp_path @@ -107,10 +107,50 @@ def test_analyze_runs_flag_free_from_profile(app: typer.Typer, profile_tree: Pat assert recorder.kwargs is not None assert recorder.kwargs["agent"] == "flight-planner" assert recorder.kwargs["workspace"] == "flight-workspace" - assert recorder.kwargs["agent_spec"] == "# Flight planner" + assert recorder.kwargs["ethos"] == "# Flight planner" + assert recorder.kwargs["ethos_label"] == "ETHOS" assert recorder.kwargs["insights_output"] is None, "a discovered profile must not divert writes off the platform" +def test_analyze_labels_readme_fallback_as_analysis_context( + app: typer.Typer, + profile_tree: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder = AnalystRecorder() + (profile_tree / "ETHOS.md").unlink() + (profile_tree / "README.md").write_text("# Flight planner repository", encoding="utf-8") + monkeypatch.setattr(cli, "run_analyst", recorder) + monkeypatch.chdir(profile_tree) + + result = runner.invoke(app, ["run"]) + + assert result.exit_code == 0, result.output + assert recorder.kwargs is not None + assert recorder.kwargs["ethos"] == "# Flight planner repository" + assert recorder.kwargs["ethos_label"] == "README analysis context (not ETHOS)" + + +def test_ethos_flag_overrides_profile( + app: typer.Typer, + profile_tree: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder = AnalystRecorder() + explicit = tmp_path / "contract.md" + explicit.write_text("# Explicit Ethos", encoding="utf-8") + monkeypatch.setattr(cli, "run_analyst", recorder) + monkeypatch.chdir(profile_tree) + + result = runner.invoke(app, ["run", "--ethos", str(explicit)]) + + assert result.exit_code == 0, result.output + assert recorder.kwargs is not None + assert recorder.kwargs["ethos"] == "# Explicit Ethos" + assert recorder.kwargs["ethos_label"] == "ETHOS" + + def test_no_local_only_flag_is_exposed(app: typer.Typer, profile_tree: Path, monkeypatch) -> None: recorder = AnalystRecorder() monkeypatch.setattr(cli, "run_analyst", recorder) @@ -706,45 +746,45 @@ def test_analyze_accepts_existing_insights_file_without_insights_key( @pytest.mark.parametrize("command", ["doctor", "run"]) -def test_commands_reject_invalid_utf8_agent_spec( +def test_commands_reject_invalid_utf8_ethos( app: typer.Typer, profile_tree: Path, monkeypatch: pytest.MonkeyPatch, command: str, ) -> None: - (profile_tree / "AGENT-SPEC.md").write_bytes(b"\xff\xfe") + (profile_tree / "ETHOS.md").write_bytes(b"\xff\xfe") monkeypatch.chdir(profile_tree) result = runner.invoke(app, [command]) assert result.exit_code == 1 - assert "agent spec" in result.output.lower() + assert "ethos" in result.output.lower() assert "UTF-8" in result.output assert "Traceback" not in result.output @pytest.mark.parametrize("command", ["doctor", "run"]) -def test_commands_reject_unreadable_agent_spec( +def test_commands_reject_unreadable_ethos( app: typer.Typer, profile_tree: Path, monkeypatch: pytest.MonkeyPatch, command: str, ) -> None: - spec = profile_tree / "AGENT-SPEC.md" + ethos = profile_tree / "ETHOS.md" original_read_text = Path.read_text - def deny_spec_read(path: Path, encoding: str | None = None, errors: str | None = None) -> str: - if path == spec: + def deny_ethos_read(path: Path, encoding: str | None = None, errors: str | None = None) -> str: + if path == ethos: raise PermissionError("permission denied") return original_read_text(path, encoding=encoding, errors=errors) - monkeypatch.setattr(Path, "read_text", deny_spec_read) + monkeypatch.setattr(Path, "read_text", deny_ethos_read) monkeypatch.chdir(profile_tree) result = runner.invoke(app, [command]) assert result.exit_code == 1 - assert "agent spec" in result.output.lower() + assert "ethos" in result.output.lower() assert "permission denied" in result.output assert "ensure the file is readable and encoded as UTF-8" in result.output assert "Traceback" not in result.output diff --git a/plugins/nemo-insights/tests/test_periodic_analysis.py b/plugins/nemo-insights/tests/test_periodic_analysis.py index 33fd353c97..4d64df6d9c 100644 --- a/plugins/nemo-insights/tests/test_periodic_analysis.py +++ b/plugins/nemo-insights/tests/test_periodic_analysis.py @@ -681,9 +681,10 @@ def _ctx(tmp_path: Path) -> JobContext: ) -def _analyze_spec(agent: str = "research-agent") -> AnalyzeSpec: +def _analyze_spec(agent: str = "research-agent", ethos: str | None = None) -> AnalyzeSpec: return AnalyzeSpec( agent=agent, + ethos=ethos, default_model="default/gpt-5", fast_model="default/gpt-5-mini", ) @@ -705,7 +706,7 @@ async def fake_run_analyst(**kwargs: object) -> str: sdk = _SyncSdk() result = AnalyzeJob().run( - _analyze_spec().model_dump(mode="json"), + _analyze_spec(ethos="# Research agent Ethos").model_dump(mode="json"), ctx=_ctx(tmp_path), sdk=cast(NeMoPlatform, sdk), ) @@ -723,6 +724,7 @@ async def fake_run_analyst(**kwargs: object) -> str: assert updates[-1]["last_submitted_job"] == "insights-job-1" assert (tmp_path / "persistent" / "analysis-report.txt").read_text() == "analysis report" assert calls[0]["client"] is async_client + assert calls[0]["ethos"] == "# Research agent Ethos" assert calls[0]["model_refs"] == ConfiguredModelRefs( default="default/gpt-5", fast="default/gpt-5-mini", diff --git a/plugins/nemo-insights/tests/test_preflight.py b/plugins/nemo-insights/tests/test_preflight.py index c847151fa1..6a412cecc3 100644 --- a/plugins/nemo-insights/tests/test_preflight.py +++ b/plugins/nemo-insights/tests/test_preflight.py @@ -11,9 +11,10 @@ from nemo_insights_plugin.contracts.checks import format_report, required_failures from nemo_insights_plugin.preflight import ( AnalysisProbes, - check_agent_spec, check_environment, + check_ethos, check_profile, + read_ethos, ) from nemo_insights_plugin.profile import AnalysisProfile from nemo_platform import NeMoPlatformError @@ -120,19 +121,19 @@ def fail_to_construct(base_url: str) -> object: assert asyncio.run(preflight._default_workspace_ok("https://platform.example", "default", "agent")) is False -def test_profile_and_agent_spec_failures_are_required(tmp_path: Path) -> None: +def test_profile_and_ethos_failures_are_required(tmp_path: Path) -> None: profile_results = check_profile(None, None) - spec_results = check_agent_spec(None, "configured agent spec does not exist") + ethos_results = check_ethos(None, "configured Ethos does not exist") assert required_failures(profile_results) == profile_results - assert required_failures(spec_results) == spec_results + assert required_failures(ethos_results) == ethos_results -def test_agent_spec_invalid_utf8_is_required_failure(tmp_path: Path) -> None: - spec = tmp_path / "AGENT-SPEC.md" - spec.write_bytes(b"\xff\xfe") +def test_ethos_invalid_utf8_is_required_failure(tmp_path: Path) -> None: + ethos = tmp_path / "ETHOS.md" + ethos.write_bytes(b"\xff\xfe") - results = check_agent_spec(spec, None) + results = check_ethos(ethos, None) assert results[0].status == "fail" assert results[0].severity == "required" @@ -140,19 +141,19 @@ def test_agent_spec_invalid_utf8_is_required_failure(tmp_path: Path) -> None: assert results[0].hint == "ensure the file is readable and encoded as UTF-8" -def test_agent_spec_unreadable_is_required_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - spec = tmp_path / "AGENT-SPEC.md" - spec.write_text("# Agent", encoding="utf-8") +def test_ethos_unreadable_is_required_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + ethos = tmp_path / "ETHOS.md" + ethos.write_text("# Agent", encoding="utf-8") original_read_text = Path.read_text - def deny_spec_read(path: Path, encoding: str | None = None, errors: str | None = None) -> str: - if path == spec: + def deny_ethos_read(path: Path, encoding: str | None = None, errors: str | None = None) -> str: + if path == ethos: raise PermissionError("permission denied") return original_read_text(path, encoding=encoding, errors=errors) - monkeypatch.setattr(Path, "read_text", deny_spec_read) + monkeypatch.setattr(Path, "read_text", deny_ethos_read) - results = check_agent_spec(spec, None) + results = check_ethos(ethos, None) assert results[0].status == "fail" assert results[0].severity == "required" @@ -160,10 +161,28 @@ def deny_spec_read(path: Path, encoding: str | None = None, errors: str | None = assert results[0].hint == "ensure the file is readable and encoded as UTF-8" +def test_read_ethos_labels_ethos_and_readme_context(tmp_path: Path) -> None: + ethos = tmp_path / "ETHOS.md" + ethos.write_text("# Ethos", encoding="utf-8") + content, label, results = read_ethos(ethos, None) + + assert content == "# Ethos" + assert label == "ETHOS" + assert results[0].status == "pass" + + readme = tmp_path / "README.md" + readme.write_text("# Repository", encoding="utf-8") + content, label, results = read_ethos(readme, None) + + assert content == "# Repository" + assert label == "README analysis context (not ETHOS)" + assert results[0].status == "pass" + + def test_healthy_setup_formats_grouped_report(tmp_path: Path) -> None: profile = AnalysisProfile(agent="a", profile_dir=tmp_path) - (tmp_path / "AGENT-SPEC.md").write_text("# Agent", encoding="utf-8") - results = check_profile(profile, None) + check_agent_spec(tmp_path / "AGENT-SPEC.md", None) + (tmp_path / "ETHOS.md").write_text("# Agent", encoding="utf-8") + results = check_profile(profile, None) + check_ethos(tmp_path / "ETHOS.md", None) results += asyncio.run( check_environment( agent="a", diff --git a/plugins/nemo-insights/tests/test_profile.py b/plugins/nemo-insights/tests/test_profile.py index 1d798d54fb..3d970a3548 100644 --- a/plugins/nemo-insights/tests/test_profile.py +++ b/plugins/nemo-insights/tests/test_profile.py @@ -5,7 +5,7 @@ import pytest from nemo_insights_plugin.contracts.profile import ProfileError -from nemo_insights_plugin.profile import load_profile, pick_agent_spec +from nemo_insights_plugin.profile import load_profile, pick_ethos FULL_PROFILE = """\ agent: flight-planner @@ -18,7 +18,7 @@ rounds: 2 framework_skills: [./skills] workspace: flight-workspace -agent_spec: ./AGENT-SPEC.md +ethos: ./ETHOS.md """ @@ -30,7 +30,7 @@ def test_load_profile_reads_analysis_fields_and_ignores_experiment_fields(tmp_pa assert profile.agent == "flight-planner" assert profile.workspace == "flight-workspace" - assert profile.agent_spec == "./AGENT-SPEC.md" + assert profile.ethos == "./ETHOS.md" assert profile.profile_dir == tmp_path.resolve() @@ -42,10 +42,10 @@ def test_profile_requires_nonempty_agent(tmp_path: Path) -> None: load_profile(path) -def test_pick_agent_spec_is_profile_relative(tmp_path: Path) -> None: +def test_pick_ethos_is_profile_relative(tmp_path: Path) -> None: path = tmp_path / "optimizer.yaml" - path.write_text("agent: a\nagent_spec: ./AGENT-SPEC.md\n", encoding="utf-8") - expected = tmp_path / "AGENT-SPEC.md" + path.write_text("agent: a\nethos: ./ETHOS.md\n", encoding="utf-8") + expected = tmp_path / "ETHOS.md" expected.write_text("# Agent", encoding="utf-8") - assert pick_agent_spec(load_profile(path)) == expected.resolve() + assert pick_ethos(load_profile(path)) == expected.resolve() From 88ece1a10a1fbad26e04847110292ccede4f59b6 Mon Sep 17 00:00:00 2001 From: Alec Khoury Date: Thu, 20 Aug 2026 11:32:21 -0500 Subject: [PATCH 2/2] fix(insights): keep the Ethos migration rename-only Keep Analyst behavior unchanged while adopting Ethos terminology. Signed-off-by: Alec Khoury --- docs/agents/insight-driven-optimization.mdx | 20 ++--- plugins/nemo-insights/README.md | 12 ++- plugins/nemo-insights/evaluation/adapters.py | 1 - plugins/nemo-insights/evaluation/tau2run.py | 4 +- .../src/nemo_insights_plugin/analyst/agent.py | 14 ++- .../src/nemo_insights_plugin/analyst/run.py | 5 +- .../src/nemo_insights_plugin/cli.py | 11 +-- .../nemo_insights_plugin/contracts/profile.py | 2 +- .../src/nemo_insights_plugin/preflight.py | 87 ++++++++----------- .../skills/nemo-analyst/SKILL.md | 7 +- .../tests/contracts/test_profile_contract.py | 4 +- .../tests/evaluation/test_adapters.py | 15 +--- .../nemo-insights/tests/test_analyst_agent.py | 15 ---- .../nemo-insights/tests/test_analyst_run.py | 2 - .../nemo-insights/tests/test_cli_profile.py | 40 --------- .../tests/test_periodic_analysis.py | 6 +- plugins/nemo-insights/tests/test_preflight.py | 19 ---- 17 files changed, 69 insertions(+), 195 deletions(-) diff --git a/docs/agents/insight-driven-optimization.mdx b/docs/agents/insight-driven-optimization.mdx index 5541cecf78..2fea6c4cc0 100644 --- a/docs/agents/insight-driven-optimization.mdx +++ b/docs/agents/insight-driven-optimization.mdx @@ -99,9 +99,9 @@ validates the full schema: ```yaml agent: research-agent -ethos: ETHOS.md # optional; source order is ETHOS.md, then README.md -workspace: default # optional; defaults to "default" -agent_source: . # local dir or git URL with optional @ref +ethos: ETHOS.md # optional; falls back to ETHOS.md, then README.md +workspace: default # optional; defaults to "default" +agent_source: . # local dir or git URL with optional @ref task_template: ./task-template datasets: train: ./harbor_eval/dataset/train @@ -113,10 +113,6 @@ a profile is found, without replacing variables already set in the shell. `NMP_BASE_URL` is the base-URL environment variable for this workflow, and `--base-url` takes precedence over it. -The Analyst labels `README.md` content exactly -`README analysis context (not ETHOS)`. It treats that content as unvalidated -repository context and doesn't validate either file with `parse_ethos()`. - ## How It Works ### The Loop @@ -218,8 +214,8 @@ Its method is to survey sessions broadly, gather evidence (starting from negative feedback and error spans), cluster similar failures across many sessions, check for existing Insights, then emit a single result containing new Insights and evidence appended to existing ones. Give the Analyst an optional -Ethos file (`--ethos ETHOS.md`) so it can flag divergence from intended -behavior. +Ethos (`--ethos ETHOS.md`) so it can flag divergence from +intended behavior. ### Experimenter @@ -358,7 +354,7 @@ nemo agents analyst run \ Useful flags: -- `--ethos ETHOS.md`: Append Ethos content so the Analyst can flag +- `--ethos ETHOS.md`: Append Ethos so the Analyst can flag divergence from intended behavior. - `--insights-file-output tmp/insights.yaml`: Read and write Insights from a specific local YAML file. Trace reads still hit `--base-url`. @@ -545,7 +541,7 @@ Run the Analyst for one analysis of an agent's traces. | Flag | Required | Default | Description | |------|----------|---------|-------------| | `--agent` | yes, unless a profile supplies it | profile `agent` | Agent under test the Analyst should focus on. | -| `--ethos` | no | profile `ethos`, else `ETHOS.md` or `README.md` beside the profile | Path to Ethos or unvalidated README analysis context for the AUT. | +| `--ethos` | no | profile `ethos`, else `ETHOS.md` or `README.md` beside the profile | Path to Ethos Markdown for the AUT. | | `--workspace` | no | profile `workspace`, else `default` | Workspace to operate in. | | `--base-url` | no | `NMP_BASE_URL`, else `http://localhost:8080` | Running platform instance the Analyst's tools call. | | `--profile` | no | discovered by walking up from the current working directory | Path to `optimizer.yaml`. | @@ -574,7 +570,7 @@ Run the local Experimenter loop. | `--insight-id` | no | Not applicable | Select an exact ID, exact title, or zero-based index from a local multi-Insight file. | | `--no-insight` | no | off | Run against a dataset directly rather than guided by an Insight. | | `--agent` | no | profile `agent_source` | Baseline agent override: a local directory or a git URL with optional ref (`...repo.git@main`). A git source records provenance and enables opening a draft PR for the winner. | -| `--ethos` | no | profile `ethos` | Location of Ethos or unvalidated analysis context for the AUT. | +| `--ethos` | no | profile `ethos` | Location of a Markdown file describing the AUT. | | `--train-dataset` | yes, unless the profile supplies it | profile `datasets.train` | Train dataset. Local path for the Harbor evaluator. | | `--validation-dataset` | yes, unless the profile supplies it | profile `datasets.validation` | Validation dataset. Local path for the Harbor evaluator. | | `--task-template` | required with an Insight | profile `task_template` | Evaluator-specific task-template URI, used to build the Insight-specific evaluation suite. | diff --git a/plugins/nemo-insights/README.md b/plugins/nemo-insights/README.md index 94203e5d2b..d1de2ea14f 100644 --- a/plugins/nemo-insights/README.md +++ b/plugins/nemo-insights/README.md @@ -34,17 +34,15 @@ The profile contract consumed by Insights is deliberately small: ```yaml agent: research-agent -ethos: ETHOS.md # optional -workspace: default # optional; defaults to "default" +ethos: ETHOS.md # optional +workspace: default # optional; defaults to "default" ``` Only `agent`, `ethos`, and `workspace` are consumed by Insights. Unknown experiment-owned fields are ignored, while the reserved `profile_dir` -field is rejected. `agent` is required. Relative `ethos` paths resolve from the -profile directory. When `ethos` is omitted, Insights looks for `ETHOS.md`, then -`README.md`, beside the profile. The Analyst labels README content exactly -`README analysis context (not ETHOS)`. It doesn't validate either file with -`parse_ethos()`. +field is rejected. `agent` is required. Relative `ethos` paths are +resolved relative to the profile. When it is omitted, Insights looks for +`ETHOS.md`, then `README.md`, beside the profile. An adjacent `.env` is loaded when a profile is found, without replacing variables already set in the shell. For this shared profile workflow, diff --git a/plugins/nemo-insights/evaluation/adapters.py b/plugins/nemo-insights/evaluation/adapters.py index 43cd3d1df8..3e7eb60716 100644 --- a/plugins/nemo-insights/evaluation/adapters.py +++ b/plugins/nemo-insights/evaluation/adapters.py @@ -298,7 +298,6 @@ async def analyze( return await run_analyst( agent=str(record["agent"]), ethos=policy, - ethos_label="Benchmark policy analysis context (not ETHOS)", workspace=workspace, base_url=str(record["base_url"]), client=make_client(str(record["base_url"])), diff --git a/plugins/nemo-insights/evaluation/tau2run.py b/plugins/nemo-insights/evaluation/tau2run.py index 5b1d727dcf..468bfb6180 100644 --- a/plugins/nemo-insights/evaluation/tau2run.py +++ b/plugins/nemo-insights/evaluation/tau2run.py @@ -133,12 +133,12 @@ def load_tasks(data_dir: Path, domain: str) -> dict[str, dict]: def read_policy(data_dir: Path, domain: str) -> str | None: - """Return the domain policy Markdown as unvalidated Analyst context. + """Return the domain policy markdown (the analyst's Ethos), or None. A tau2 checkout nests domains under ``tau2/domains//``; some data dirs are flat (``domains//``). For each layout, tries ``policy.md`` then ``main_policy.md`` (Telecom uses the latter). Returns the first file - found, else ``None`` (the Analyst then runs without this context). + found, else ``None`` (the analyst then runs without Ethos). """ for base in (data_dir / "tau2" / "domains", data_dir / "domains"): domain_dir = base / domain diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py index e2f4806885..8078ceb12d 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/agent.py @@ -147,11 +147,12 @@ ETHOS_HEADER = """ -## {ethos_label} +## Ethos -Use this content to understand what the agent is supposed to do and what -success looks like. Content labeled ETHOS is the agent's contract. Content -labeled README analysis context (not ETHOS) is unvalidated repository context. +Use this as the contract for what the agent is supposed to do, what +success looks like, and what behavior should be flagged as divergence. +Flag agent divergence from the Ethos. The Ethos was authored by the +developer of the application and should be considered the purpose and goals. """ KICKOFF = ( @@ -170,7 +171,6 @@ def __init__( deps: AnalystDeps, agent: str, ethos: str | None = None, - ethos_label: str = "ETHOS", **kwargs: Any, ) -> None: super().__init__(llm=kwargs.pop("llm", None) or get_default_model(), **kwargs) @@ -184,7 +184,7 @@ def __init__( instructions = INSTRUCTIONS.format(agent=agent) if ethos and ethos.strip(): - instructions = f"{instructions}\n{ETHOS_HEADER.format(ethos_label=ethos_label)}\n\n{ethos.strip()}\n" + instructions = f"{instructions}\n{ETHOS_HEADER}\n\n{ethos.strip()}\n" self.context["analyst_instructions"] = instructions async def fetch_spans( @@ -361,7 +361,6 @@ def build_analyst_agent( deps: AnalystDeps, agent: str, ethos: str | None = None, - ethos_label: str = "ETHOS", llm: UnifiedLLM | None = None, **kwargs: Any, ) -> Analyst: @@ -370,7 +369,6 @@ def build_analyst_agent( deps=deps, agent=agent, ethos=ethos, - ethos_label=ethos_label, llm=llm, **kwargs, ) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py index 64fd47111f..e145cad0dc 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/run.py @@ -52,7 +52,6 @@ async def run_analyst( since: datetime | None = None, evaluation_id: str | None = None, model_refs: ConfiguredModelRefs | None = None, - ethos_label: str = "ETHOS", ) -> str: """Build and run the analyst agent against an agent's telemetry. @@ -61,7 +60,7 @@ async def run_analyst( Args: agent: Agent under test. - ethos: Optional Ethos or analysis-context Markdown for the agent under test. + ethos: Optional Ethos Markdown for the agent under test. workspace: Platform workspace. base_url: Platform base URL. ``None`` uses the active platform context. client: Platform client to use. This function closes it before returning. @@ -75,7 +74,6 @@ async def run_analyst( evaluation_id: Optional run scope; AND-pinned onto every span read. model_refs: Optional explicit default/fast Model Entity IDs. Unset uses the active Platform CLI context. - ethos_label: Source label for the supplied content. """ observability = None model_clients: ConfiguredModelClients | None = None @@ -107,7 +105,6 @@ async def run_analyst( deps=deps, agent=agent, ethos=ethos, - ethos_label=ethos_label, ) result = await _run_agent(analyst, verbose=verbose) return await backend.persist_result(workspace=workspace, agent=agent, result=result) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/cli.py b/plugins/nemo-insights/src/nemo_insights_plugin/cli.py index 471ccf29b7..60f04971c6 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/cli.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/cli.py @@ -52,7 +52,6 @@ class _ResolvedAnalysis: agent: str ethos: str | None - ethos_label: str | None workspace: str base_url: str insights_output: Path | None @@ -129,7 +128,7 @@ def _resolve_analysis( ethos_path = pick_ethos(profile) except ProfileError as exc: ethos_error = str(exc) - ethos_content, ethos_label, ethos_checks = read_ethos(ethos_path, ethos_error) + ethos_content, ethos_checks = read_ethos(ethos_path, ethos_error) resolved_base_url = resolve_base_url(base_url) validate_insights_file(insights_output) @@ -137,7 +136,6 @@ def _resolve_analysis( return _ResolvedAnalysis( agent=resolved_agent, ethos=ethos_content, - ethos_label=ethos_label, workspace=resolved_workspace, base_url=resolved_base_url, insights_output=insights_output, @@ -183,7 +181,6 @@ async def _run_analysis(analysis: _ResolvedAnalysis, *, verbose: bool) -> str: return await run_analyst( agent=analysis.agent, ethos=analysis.ethos, - ethos_label=analysis.ethos_label or "ETHOS", workspace=analysis.workspace, base_url=analysis.base_url, client=client, @@ -218,7 +215,7 @@ def analyze( ethos: Path | None = typer.Option( None, "--ethos", - help="Path to Markdown content that describes the agent under test.", + help="Path to a markdown file describing the agent under test (its Ethos).", exists=True, readable=True, ), @@ -262,7 +259,7 @@ def analyze( ) -> None: """Run the analyst agent against a running NMP instance. - Builds the analyst agent with ``--agent`` and optional ``--ethos`` content + Builds the analyst agent with ``--agent`` (and optional ``--ethos``) formatted into its instructions and tools scoped to ``--agent`` / ``--workspace`` / ``--base-url``, runs it, and prints whatever the agent returns. Insights are written to the @@ -312,7 +309,7 @@ def doctor( ethos_path = pick_ethos(profile) except ProfileError as exc: ethos_error = str(exc) - _, _, ethos_results = read_ethos(ethos_path, ethos_error) + _, ethos_results = read_ethos(ethos_path, ethos_error) async def _flow() -> list[CheckResult]: results = check_profile(profile, profile_error) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py b/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py index 8ba5e0c012..972f2eb63f 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/contracts/profile.py @@ -93,7 +93,7 @@ def load_env_file(path: Path, env: MutableMapping[str, str] = os.environ) -> lis def resolve_ethos_path(profile_dir: Path, configured: str | None) -> Path | None: - """Resolve configured Ethos content or a conventional profile-local file.""" + """Resolve a configured Ethos or the conventional profile-local file.""" if configured is not None: path = resolve_profile_path(configured, profile_dir) if not path.is_file(): diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py b/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py index 4a1bb47ce5..c02a83a7dc 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/preflight.py @@ -100,73 +100,56 @@ def check_ethos( ethos_error: str | None, ) -> list[CheckResult]: """Check the optional Ethos content, including explicit UTF-8 readability.""" - return read_ethos(ethos_path, ethos_error)[2] + return read_ethos(ethos_path, ethos_error)[1] def read_ethos( ethos_path: Path | None, ethos_error: str | None, -) -> tuple[str | None, str | None, list[CheckResult]]: - """Read optional Ethos or README context and return its source label and check.""" +) -> tuple[str | None, list[CheckResult]]: + """Read the optional Ethos as UTF-8 and return its readiness check.""" if ethos_error is not None: - return ( - None, - None, - [ - CheckResult( - name="ethos", - group="artifacts", - status="fail", - severity="required", - message=ethos_error, - ) - ], - ) + return None, [ + CheckResult( + name="ethos", + group="artifacts", + status="fail", + severity="required", + message=ethos_error, + ) + ] if ethos_path is None: - return ( - None, - None, - [ - CheckResult( - name="ethos", - group="artifacts", - status="pass", - severity="advisory", - message="Ethos omitted (optional)", - ) - ], - ) + return None, [ + CheckResult( + name="ethos", + group="artifacts", + status="pass", + severity="advisory", + message="Ethos omitted (optional)", + ) + ] try: content = ethos_path.read_text(encoding="utf-8") except (OSError, UnicodeError) as exc: - return ( - None, - None, - [ - CheckResult( - name="ethos", - group="artifacts", - status="fail", - severity="required", - message=f"Could not read Ethos content {ethos_path} as UTF-8: {exc}", - hint="ensure the file is readable and encoded as UTF-8", - ) - ], - ) - label = "README analysis context (not ETHOS)" if ethos_path.name == "README.md" else "ETHOS" - return ( - content, - label, - [ + return None, [ CheckResult( name="ethos", group="artifacts", - status="pass", + status="fail", severity="required", - message=f"{label} readable at {ethos_path}", + message=f"Could not read Ethos {ethos_path} as UTF-8: {exc}", + hint="ensure the file is readable and encoded as UTF-8", ) - ], - ) + ] + return content, [ + CheckResult( + name="ethos", + group="artifacts", + status="pass", + severity="required", + message=f"Ethos readable at {ethos_path}", + ) + ] def check_models() -> list[CheckResult]: diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md b/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md index 1c5eade70a..36f761b783 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md +++ b/plugins/nemo-insights/src/nemo_insights_plugin/skills/nemo-analyst/SKILL.md @@ -72,10 +72,7 @@ The Analyst reads telemetry; it cannot create it. Confirm all three: An `ETHOS.md` file is optional but makes the Analyst materially better. It carries the intent behind the agent — what it is for, its constraints, what counts as success — none of which is recoverable from code or traces, so -without it the Analyst can only judge an agent against itself. If the profile -has no `ETHOS.md` file, the Analyst uses `README.md` as unvalidated -`README analysis context (not ETHOS)`. It doesn't validate either file with -`parse_ethos()`. +without it the Analyst can only judge an agent against itself. ## Pre-flight @@ -101,7 +98,7 @@ and `--verbose` to stream its tool calls and reasoning to stderr. Expect several minutes; it surveys many sessions before drilling into any of them. From an agent directory, an `optimizer.yaml` profile supplies `agent`, -`workspace`, and `ethos`, so these flags become optional: +`workspace`, and `ethos`, so the flags above become optional: ```bash nemo agents analyst run diff --git a/plugins/nemo-insights/tests/contracts/test_profile_contract.py b/plugins/nemo-insights/tests/contracts/test_profile_contract.py index 90e14b3d48..f083ba98f8 100644 --- a/plugins/nemo-insights/tests/contracts/test_profile_contract.py +++ b/plugins/nemo-insights/tests/contracts/test_profile_contract.py @@ -134,10 +134,10 @@ def test_load_env_file_wraps_permission_errors( path.write_text("KEY=value\n", encoding="utf-8") original_read_text = Path.read_text - def deny(candidate: Path, encoding: str | None = None, errors: str | None = None) -> str: + def deny(candidate: Path, *args: object, **kwargs: object) -> str: if candidate == path: raise PermissionError("permission denied") - return original_read_text(candidate, encoding=encoding, errors=errors) + return original_read_text(candidate, *args, **kwargs) monkeypatch.setattr(Path, "read_text", deny) diff --git a/plugins/nemo-insights/tests/evaluation/test_adapters.py b/plugins/nemo-insights/tests/evaluation/test_adapters.py index 02ae893a12..a869bdc811 100644 --- a/plugins/nemo-insights/tests/evaluation/test_adapters.py +++ b/plugins/nemo-insights/tests/evaluation/test_adapters.py @@ -354,23 +354,11 @@ async def test_benchmark_analyze_uses_record(monkeypatch, tmp_path): seen: dict[str, object] = {} async def fake_run_analyst( - *, - agent, - ethos, - ethos_label, - workspace, - base_url, - client, - insights_output, - local_only, - verbose, - since, - evaluation_id, + *, agent, ethos, workspace, base_url, client, insights_output, local_only, verbose, since, evaluation_id ): seen.update( agent=agent, ethos=ethos, - ethos_label=ethos_label, workspace=workspace, base_url=base_url, evaluation_id=evaluation_id, @@ -397,7 +385,6 @@ async def fake_run_analyst( assert ran_tau2["v"] is False # analyze never runs tau2 assert seen["agent"] == "tau2-airline" assert seen["ethos"] == "POLICY" - assert seen["ethos_label"] == "Benchmark policy analysis context (not ETHOS)" assert seen["workspace"] == "tau2-airline" # the stable REALISTIC workspace, never the oracle one assert seen["evaluation_id"] == "tau2-airline-20260626-000000-abcd" # run-scoped assert seen["base_url"] == "http://localhost:8080" diff --git a/plugins/nemo-insights/tests/test_analyst_agent.py b/plugins/nemo-insights/tests/test_analyst_agent.py index 5cec3ca465..bf7287c4db 100644 --- a/plugins/nemo-insights/tests/test_analyst_agent.py +++ b/plugins/nemo-insights/tests/test_analyst_agent.py @@ -63,7 +63,6 @@ async def test_nooa_codeact_returns_typed_analyst_result_and_receives_prompt() - assert result.updated_insights == [] rendered_messages = json.dumps(fake.last_messages) assert "target-agent" in rendered_messages - assert "## ETHOS" in rendered_messages assert "Expected behavior" in rendered_messages assert "One method, two modes" in rendered_messages assert "private-workspace" not in rendered_messages @@ -71,20 +70,6 @@ async def test_nooa_codeact_returns_typed_analyst_result_and_receives_prompt() - assert {tool.name for tool in fake.last_tools} == {"execute_python", "return_result"} -def test_readme_context_header_does_not_claim_ethos() -> None: - analyst = build_analyst_agent( - deps=AnalystDeps(agent="target-agent", workspace="workspace"), - agent="target-agent", - ethos="# Repository context", - ethos_label="README analysis context (not ETHOS)", - llm=FakeLLMClient(), - ) - - instructions = cast(Any, analyst.context)["analyst_instructions"] - assert "## README analysis context (not ETHOS)" in instructions - assert "## ETHOS\n" not in instructions - - class _SpanBackend: def __init__(self) -> None: self.kwargs: dict[str, object] | None = None diff --git a/plugins/nemo-insights/tests/test_analyst_run.py b/plugins/nemo-insights/tests/test_analyst_run.py index 0adf6d7cd4..8dc1936c5a 100644 --- a/plugins/nemo-insights/tests/test_analyst_run.py +++ b/plugins/nemo-insights/tests/test_analyst_run.py @@ -85,8 +85,6 @@ async def test_injected_client_is_used_and_closed(monkeypatch: pytest.MonkeyPatc assert seen["backend_client"] is client build_kwargs = cast(dict[str, object], seen["build_kwargs"]) assert cast(AnalystDeps, build_kwargs["deps"]).backend is not None - assert build_kwargs["ethos"] is None - assert build_kwargs["ethos_label"] == "ETHOS" assert seen["model_client"] is client model_clients = cast(ConfiguredModelClients, seen["model_clients"]) assert cast(FakeModelClient, model_clients.default).closed diff --git a/plugins/nemo-insights/tests/test_cli_profile.py b/plugins/nemo-insights/tests/test_cli_profile.py index c450b11708..0f5d5db7f9 100644 --- a/plugins/nemo-insights/tests/test_cli_profile.py +++ b/plugins/nemo-insights/tests/test_cli_profile.py @@ -108,49 +108,9 @@ def test_analyze_runs_flag_free_from_profile(app: typer.Typer, profile_tree: Pat assert recorder.kwargs["agent"] == "flight-planner" assert recorder.kwargs["workspace"] == "flight-workspace" assert recorder.kwargs["ethos"] == "# Flight planner" - assert recorder.kwargs["ethos_label"] == "ETHOS" assert recorder.kwargs["insights_output"] is None, "a discovered profile must not divert writes off the platform" -def test_analyze_labels_readme_fallback_as_analysis_context( - app: typer.Typer, - profile_tree: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - recorder = AnalystRecorder() - (profile_tree / "ETHOS.md").unlink() - (profile_tree / "README.md").write_text("# Flight planner repository", encoding="utf-8") - monkeypatch.setattr(cli, "run_analyst", recorder) - monkeypatch.chdir(profile_tree) - - result = runner.invoke(app, ["run"]) - - assert result.exit_code == 0, result.output - assert recorder.kwargs is not None - assert recorder.kwargs["ethos"] == "# Flight planner repository" - assert recorder.kwargs["ethos_label"] == "README analysis context (not ETHOS)" - - -def test_ethos_flag_overrides_profile( - app: typer.Typer, - profile_tree: Path, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - recorder = AnalystRecorder() - explicit = tmp_path / "contract.md" - explicit.write_text("# Explicit Ethos", encoding="utf-8") - monkeypatch.setattr(cli, "run_analyst", recorder) - monkeypatch.chdir(profile_tree) - - result = runner.invoke(app, ["run", "--ethos", str(explicit)]) - - assert result.exit_code == 0, result.output - assert recorder.kwargs is not None - assert recorder.kwargs["ethos"] == "# Explicit Ethos" - assert recorder.kwargs["ethos_label"] == "ETHOS" - - def test_no_local_only_flag_is_exposed(app: typer.Typer, profile_tree: Path, monkeypatch) -> None: recorder = AnalystRecorder() monkeypatch.setattr(cli, "run_analyst", recorder) diff --git a/plugins/nemo-insights/tests/test_periodic_analysis.py b/plugins/nemo-insights/tests/test_periodic_analysis.py index 4d64df6d9c..33fd353c97 100644 --- a/plugins/nemo-insights/tests/test_periodic_analysis.py +++ b/plugins/nemo-insights/tests/test_periodic_analysis.py @@ -681,10 +681,9 @@ def _ctx(tmp_path: Path) -> JobContext: ) -def _analyze_spec(agent: str = "research-agent", ethos: str | None = None) -> AnalyzeSpec: +def _analyze_spec(agent: str = "research-agent") -> AnalyzeSpec: return AnalyzeSpec( agent=agent, - ethos=ethos, default_model="default/gpt-5", fast_model="default/gpt-5-mini", ) @@ -706,7 +705,7 @@ async def fake_run_analyst(**kwargs: object) -> str: sdk = _SyncSdk() result = AnalyzeJob().run( - _analyze_spec(ethos="# Research agent Ethos").model_dump(mode="json"), + _analyze_spec().model_dump(mode="json"), ctx=_ctx(tmp_path), sdk=cast(NeMoPlatform, sdk), ) @@ -724,7 +723,6 @@ async def fake_run_analyst(**kwargs: object) -> str: assert updates[-1]["last_submitted_job"] == "insights-job-1" assert (tmp_path / "persistent" / "analysis-report.txt").read_text() == "analysis report" assert calls[0]["client"] is async_client - assert calls[0]["ethos"] == "# Research agent Ethos" assert calls[0]["model_refs"] == ConfiguredModelRefs( default="default/gpt-5", fast="default/gpt-5-mini", diff --git a/plugins/nemo-insights/tests/test_preflight.py b/plugins/nemo-insights/tests/test_preflight.py index 6a412cecc3..52fa427c16 100644 --- a/plugins/nemo-insights/tests/test_preflight.py +++ b/plugins/nemo-insights/tests/test_preflight.py @@ -14,7 +14,6 @@ check_environment, check_ethos, check_profile, - read_ethos, ) from nemo_insights_plugin.profile import AnalysisProfile from nemo_platform import NeMoPlatformError @@ -161,24 +160,6 @@ def deny_ethos_read(path: Path, encoding: str | None = None, errors: str | None assert results[0].hint == "ensure the file is readable and encoded as UTF-8" -def test_read_ethos_labels_ethos_and_readme_context(tmp_path: Path) -> None: - ethos = tmp_path / "ETHOS.md" - ethos.write_text("# Ethos", encoding="utf-8") - content, label, results = read_ethos(ethos, None) - - assert content == "# Ethos" - assert label == "ETHOS" - assert results[0].status == "pass" - - readme = tmp_path / "README.md" - readme.write_text("# Repository", encoding="utf-8") - content, label, results = read_ethos(readme, None) - - assert content == "# Repository" - assert label == "README analysis context (not ETHOS)" - assert results[0].status == "pass" - - def test_healthy_setup_formats_grouped_report(tmp_path: Path) -> None: profile = AnalysisProfile(agent="a", profile_dir=tmp_path) (tmp_path / "ETHOS.md").write_text("# Agent", encoding="utf-8")