From 4a89710d4979ef6487fabf4d19ddcc7ae6215b9f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:14:15 -0800 Subject: [PATCH 1/5] feat(protocol): the ATIF validator, its CLI and the adapter kit (Y1b, FD-6075) agentenv_protocol.atif.validate checks a document against ATIF's vendored models, the profile's models, and what neither checks: unknown profile keys and versions, vocabularies, keys outside extra.agentenv, unique tool_call_ids, references that resolve, final_metrics totals over the root and the sub-agents it references, total_steps, cache counts within prompt_tokens, media paths and bytes, timestamps, Harbor's save and all-text messages as strings. Each finding carries a rule id PROFILE.md names; a producer is held to profile 1.0, a reader only warned where a producer would fail. validate_live checks the pages a live reader was served: each a prefix of the next and of the final read, with no unanswered call served before its task ends. python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader] prints the findings. check_adapter runs an adapter over recorded tasks for an agent's CI: the live rule after every record, the validator on the document, the expected document, unmatched records and from_atif. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/atif/PROFILE.md | 43 ++ .../src/agentenv_protocol/atif/__init__.py | 10 +- .../src/agentenv_protocol/atif/__main__.py | 41 ++ .../src/agentenv_protocol/atif/kit.py | 104 ++++ .../src/agentenv_protocol/atif/validator.py | 497 ++++++++++++++++++ .../agentenv-protocol/tests/test_atif_kit.py | 96 ++++ .../tests/test_atif_validator.py | 204 +++++++ 7 files changed, 992 insertions(+), 3 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py create mode 100644 packages/agentenv-protocol/tests/test_atif_kit.py create mode 100644 packages/agentenv-protocol/tests/test_atif_validator.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md index 717283ba5..e2b6da52d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md @@ -146,6 +146,49 @@ field added in a later minor names the version it arrived in. Proposed for a lat `input_includes_cache` on the root and `cost_status` (`reported`, `priced` or `unpriced`), with the usage cross-check that reads them; per-part origins, when a reader needs them; and an always-on harness's fields. +## Validator + +`validate(document)` returns findings, each with a rule id below, a severity and where in the document it is. It holds a +producer's output to profile 1.0. `reader=True` checks someone else's document for reading instead: what a producer +must not write only warns, and the document fails only where agentenv couldn't read it. `media` is the folder the +document's media paths are relative to, the one holding `media/`, or the files by path; without it only the paths' form +is checked. `validate_live(pages, final)` checks what a live reader was served, page by page, with the final read. On +the command line: `python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]`, which prints each finding +and exits 1 when one is an error. + +| Rule | Holds | Producer | Reader | +| --- | --- | --- | --- | +| `json` | The document is a JSON object: UTF-8, with no NaN or infinity | error | error | +| `atif` | ATIF's own models, vendored from Harbor, accept it | error | error | +| `profile` | Each `extra.agentenv` parses with its profile model | error | error | +| `profile-key` | Each `extra.agentenv` holds only the fields profile 1.0 defines | error | warning | +| `profile-version` | The root's `extra.agentenv.version`, when given, is `1.0` | error | warning | +| `vocabulary` | A vocabulary field holds one of its values | error | warning | +| `outside-key` | No `extra` holds a key outside `agentenv` but `ToolCall.extra.raw_arguments` | error | warning | +| `call-id` | Each `tool_call_id` is unique across the document and its sub-agents | error | warning | +| `reference` | Each `subagent_trajectory_ref` names an embedded sub-agent by `trajectory_id` | error | warning | +| `totals` | A `final_metrics` total equals its summed model steps, when every one has the count | error | warning | +| `total-steps` | `final_metrics.total_steps` counts the root's steps, or the root's `notes` explain the difference | error | warning | +| `usage` | A step's cache reads and writes fit within its `prompt_tokens` | error | warning | +| `media-path` | A media path is `media/.` | error | warning | +| `media-bytes` | Given the media, each file the document names exists, with the sha256 its name and profile give and the profile's size | error | error | +| `timestamp` | A timestamp is UTC in the builder's form: `YYYY-MM-DDTHH:MM:SSZ`, with no, 3 or 6 fractional digits | error | not checked | +| `harbor-save` | Harbor's `to_json_dict` gives the document back unchanged | error | warning | +| `text-message` | A step message whose parts are all text is one string | error | warning | +| `live-prefix` | Every live page is a prefix of the next page and of the final read | error | not checked | +| `live-unanswered` | No live page serves a step with an unanswered call before its task ends | error | not checked | + +The adapter kit, `check_adapter(adapter, recorded, expected=, from_atif=)`, is for an agent's CI. It runs an adapter +over a context's recorded tasks as the SDK runs it live, applies the live rule after every record and the validator to +the final document, and adds its own rules. `kit-from-atif` only warns when the adapter has no `from_atif`. + +| Rule | Holds | Severity | +| --- | --- | --- | +| `kit-run` | The adapter takes every recorded record without raising | error | +| `kit-expected` | The document equals the checked-in expected document | error | +| `kit-unmatched` | Every native record has a handler; one that none takes is only counted | warning | +| `kit-from-atif` | `from_atif` turns a document without the profile into a native log | error | + ## Fixtures `fixtures/` holds the profile's canonical examples, built through the adapter framework from a toy format's records: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py index 78e3c8a36..1d24bd9e5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py @@ -44,16 +44,20 @@ profile_of, read_profile, ) +from .kit import KIT_RULES, KitResult, check_adapter from .reader import Task, check, is_prefix, load, media_paths, strip_profile, tasks from .runner import AdapterRun, Fed, RecordedTask, replay +from .validator import RULES, Finding, Rule, errors, validate, validate_live __all__ = [ "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "AtifError", "AtifTurn", "AudioSource", "ContentPart", "Document", "FORMAT", "Fed", "FinalMetrics", "FinalMetricsProfile", "ImageSource", - "Media", "MediaInfo", "Metrics", "MetricsProfile", "Native", "Observation", "ObservationResult", + "Finding", "KIT_RULES", "KitResult", "Media", "MediaInfo", "Metrics", "MetricsProfile", "Native", "Observation", + "ObservationResult", "RULES", "Rule", "ObservationResultProfile", "PROFILES", "PROFILE_KEY", "PROFILE_VERSION", "RAW_ARGUMENTS", "ReasoningBlock", "RecordedTask", "SCHEMA_VERSION", "Selector", "Step", "StepError", "StepProfile", "SubagentTrajectoryRef", "SubagentTrajectoryRefProfile", "Task", "ToolCall", "ToolCallProfile", "Trajectory", "TrajectoryProfile", - "TurnContext", "VOCABULARIES", "atif_adapter", "aux", "check", "dump_profile", "finish", "is_prefix", "load", - "media_paths", "native", "profile_of", "read_profile", "replay", "strip_profile", "tasks", "timestamp", + "TurnContext", "VOCABULARIES", "atif_adapter", "aux", "check", "check_adapter", "dump_profile", "errors", "finish", + "is_prefix", "load", "media_paths", "native", "profile_of", "read_profile", "replay", "strip_profile", "tasks", + "timestamp", "validate", "validate_live", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py new file mode 100644 index 000000000..f22e5b65e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py @@ -0,0 +1,41 @@ +"""``python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]``: check an ATIF document, printing each +finding; exit 1 when any is an error, 2 when the file can't be read.""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Sequence + +from .validator import errors, parse, validate + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.atif", + description="Check ATIF documents against ATIF and the agentenv profile.") + commands = parser.add_subparsers(dest="command", required=True) + check = commands.add_parser("validate", help="check one document; exit 1 when it has an error") + check.add_argument("file", type=Path, help="the trajectory.json to check") + check.add_argument("--media", type=Path, metavar="DIR", + help="the folder the document's media paths are relative to, which holds media/; " + "given, each file is checked against its name and profile") + check.add_argument("--reader", action="store_true", + help="check someone else's document for reading: what a producer must not write warns") + args = parser.parse_args(argv) + try: + document, findings = parse(args.file) + except OSError as exc: + print(f"can't read {args.file}: {exc.strerror or exc}", file=sys.stderr) + return 2 + if not findings: + findings = validate(document, reader=args.reader, media=args.media) + for finding in findings: + print(finding) + failed = errors(findings) + print(f"{args.file}: {'invalid' if failed else 'valid'}, {len(failed)} errors, " + f"{len(findings) - len(failed)} warnings") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py new file mode 100644 index 000000000..3fff208f8 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py @@ -0,0 +1,104 @@ +"""The adapter kit, for an agent's CI: run an adapter over recorded tasks, as the SDK runs it live, and check what it +writes. The live rule holds after every record, the final document passes the validator as a producer's, it equals +the checked-in document when one is given, and ``from_atif`` turns ATIF without the profile back into a native log.""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .adapter import Adapter, Native +from .profile import profile_of +from .reader import strip_profile +from .runner import AdapterRun, RecordedTask +from .validator import ERROR, WARNING, Finding, Rule, _difference, errors, validate_live + +KIT_RULES: dict[str, Rule] = {rule.id: rule for rule in ( + Rule("kit-run", ERROR, None, "The adapter takes every recorded record without raising"), + Rule("kit-expected", ERROR, None, "The document equals the checked-in expected document"), + Rule("kit-unmatched", WARNING, None, "Every native record has a handler; one that none takes is only counted"), + Rule("kit-from-atif", ERROR, None, "`from_atif` turns a document without the profile into a native log"), +)} + + +@dataclass +class KitResult: + """What the kit found, with the document and every live page the adapter's run served.""" + + document: dict[str, Any] | None + pages: list[dict[str, Any] | None] = field(default_factory=list) + media: dict[str, bytes] = field(default_factory=dict) + findings: list[Finding] = field(default_factory=list) + + @property + def errors(self) -> list[Finding]: + return errors(self.findings) + + def raise_for_errors(self) -> None: + """Raise AssertionError listing every finding when any is an error.""" + if self.errors: + raise AssertionError("the adapter kit found:\n" + "\n".join(str(f) for f in self.findings)) + + +def check_adapter(adapter: type[Adapter], recorded: Sequence[RecordedTask], *, + expected: Mapping[str, Any] | Path | None = None, + from_atif: Sequence[Mapping[str, Any]] | None = None) -> KitResult: + """Run ``adapter`` over ``recorded``, one A2A context's tasks in order, and check it. ``expected`` is the document + the run should write, or the file holding it. ``from_atif`` are the documents its ``from_atif`` is tried on; by + default, the run's own document with the profile stripped.""" + run = AdapterRun(adapter) + result = KitResult(document=None) + where = "" + try: + for t, task in enumerate(recorded): + where = f"task {t}, its start" + run.start_task(task.context) + result.pages.append(run.page()) + for r, fed in enumerate(task.records): + where = f"task {t}, record {r}" + (f" of aux {fed.aux!r}" if fed.aux is not None else "") + if fed.aux is None: + run.native(fed.record) + else: + run.aux(fed.aux, fed.record) + result.pages.append(run.page()) + where = f"task {t}, its end" + run.end_task(state=task.state, final=task.final, ended_at=task.ended_at) + result.pages.append(run.page()) + except Exception as exc: + result.findings.append(Finding("kit-run", ERROR, where, f"{type(exc).__name__}: {exc}")) + return result + result.document = run.document() + result.media = run.media_files + if result.document is None: + result.findings.append(Finding("kit-run", ERROR, "", "the run wrote no step")) + return result + result.findings.extend(validate_live(result.pages, result.document, media=result.media)) + unmatched = profile_of(result.document).get("unmatched") + if unmatched: + counts = ", ".join(f"{kind} ({count})" for kind, count in sorted(unmatched.items())) + result.findings.append(Finding("kit-unmatched", WARNING, "extra.agentenv.unmatched", + f"no handler takes these native records: {counts}")) + if expected is not None: + want = json.loads(expected.read_text(encoding="utf-8")) if isinstance(expected, Path) else expected + if result.document != want: + result.findings.append(Finding("kit-expected", ERROR, _difference(want, result.document), + "the document differs from the expected one here")) + for index, document in enumerate(from_atif if from_atif is not None else [strip_profile(result.document)]): + _from_atif(adapter, document, index, result) + return result + + +def _from_atif(adapter: type[Adapter], document: Mapping[str, Any], index: int, result: KitResult) -> None: + where = f"from_atif document {index}" + try: + native = adapter.from_atif(document) + except NotImplementedError as exc: + result.findings.append(Finding("kit-from-atif", WARNING, where, str(exc))) + return + except Exception as exc: + result.findings.append(Finding("kit-from-atif", ERROR, where, f"raised {type(exc).__name__}: {exc}")) + return + if not isinstance(native, Native) or not isinstance(native.log, list): + result.findings.append(Finding("kit-from-atif", ERROR, where, + f"returned {type(native).__name__}, not a Native holding a list log")) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py new file mode 100644 index 000000000..2d29ad577 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -0,0 +1,497 @@ +"""Checking an ATIF document against ATIF and the agentenv profile: the vendored models, the profile's models, and the +rules neither checks, each a finding with a rule id PROFILE.md names. A producer is held to profile 1.0; a reader of +someone else's document is warned where a producer would fail, and fails only where agentenv couldn't read it.""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +from pydantic import BaseModel, ValidationError + +from .models import Trajectory +from .profile import ( + PROFILE_KEY, + PROFILE_VERSION, + PROFILES, + RAW_ARGUMENTS, + VOCABULARIES, + AdapterRef, + Arrival, + MediaInfo, + ReasoningBlock, + StepError, + profile_of, +) +from .reader import is_prefix, tasks + +ERROR = "error" +WARNING = "warning" + +MEDIA_PATH = re.compile(r"^media/(?P[0-9a-f]{64})\.[A-Za-z0-9]+$") +TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3}|\.\d{6})?Z$") +_COUNTS = (("prompt_tokens", "total_prompt_tokens"), ("completion_tokens", "total_completion_tokens"), + ("cached_tokens", "total_cached_tokens")) +#: A profile field that holds a nested profile object, by its owner and name. +_NESTED: dict[tuple[str, str], type[BaseModel]] = { + ("Trajectory", "adapter"): AdapterRef, ("Step", "error"): StepError, + ("ObservationResult", "arrived"): Arrival, +} +_LISTS: dict[tuple[str, str], type[BaseModel]] = {("Step", "reasoning"): ReasoningBlock} +_MAPS: dict[tuple[str, str], type[BaseModel]] = { + ("Step", "media"): MediaInfo, ("ObservationResult", "media"): MediaInfo, +} +_VOCABULARY_AT = { + ("Step", "state"): "StepProfile.state", ("ObservationResult", "status"): "ObservationResultProfile.status", + ("Metrics", "cost_source"): "MetricsProfile.cost_source", + ("FinalMetrics", "cost_source"): "FinalMetricsProfile.cost_source", +} + + +@dataclass(frozen=True) +class Rule: + """A check: its id, its severity for a producer and for a reader (None: not checked there), and what it holds.""" + + id: str + producer: str + reader: str | None + holds: str + + +RULES: dict[str, Rule] = {rule.id: rule for rule in ( + Rule("json", ERROR, ERROR, "The document is a JSON object: UTF-8, with no NaN or infinity"), + Rule("atif", ERROR, ERROR, "ATIF's own models, vendored from Harbor, accept it"), + Rule("profile", ERROR, ERROR, "Each `extra.agentenv` parses with its profile model"), + Rule("profile-key", ERROR, WARNING, "Each `extra.agentenv` holds only the fields profile 1.0 defines"), + Rule("profile-version", ERROR, WARNING, "The root's `extra.agentenv.version`, when given, is `1.0`"), + Rule("vocabulary", ERROR, WARNING, "A vocabulary field holds one of its values"), + Rule("outside-key", ERROR, WARNING, "No `extra` holds a key outside `agentenv` but `ToolCall.extra.raw_arguments`"), + Rule("call-id", ERROR, WARNING, "Each `tool_call_id` is unique across the document and its sub-agents"), + Rule("reference", ERROR, WARNING, "Each `subagent_trajectory_ref` names an embedded sub-agent by `trajectory_id`"), + Rule("totals", ERROR, WARNING, "A `final_metrics` total equals its summed model steps, when every one has the " + "count"), + Rule("total-steps", ERROR, WARNING, "`final_metrics.total_steps` counts the root's steps, or the root's `notes` " + "explain the difference"), + Rule("usage", ERROR, WARNING, "A step's cache reads and writes fit within its `prompt_tokens`"), + Rule("media-path", ERROR, WARNING, "A media path is `media/.`"), + Rule("media-bytes", ERROR, ERROR, "Given the media, each file the document names exists, with the sha256 its " + "name and profile give and the profile's size"), + Rule("timestamp", ERROR, None, "A timestamp is UTC in the builder's form: `YYYY-MM-DDTHH:MM:SSZ`, with no, 3 or " + "6 fractional digits"), + Rule("harbor-save", ERROR, WARNING, "Harbor's `to_json_dict` gives the document back unchanged"), + Rule("text-message", ERROR, WARNING, "A step message whose parts are all text is one string"), + Rule("live-prefix", ERROR, None, "Every live page is a prefix of the next page and of the final read"), + Rule("live-unanswered", ERROR, None, "No live page serves a step with an unanswered call before its task ends"), +)} + + +@dataclass(frozen=True) +class Finding: + """What a rule found, and where: a path into the document, such as ``steps[3].tool_calls[0]``.""" + + rule: str + severity: str + where: str + message: str + + def __str__(self) -> str: + return f"{self.severity} {self.rule} {self.where or 'document'}: {self.message}" + + +class _Findings: + def __init__(self, reader: bool, prefix: str = "") -> None: + self.reader = reader + self.prefix = prefix + self.items: list[Finding] = [] + + def add(self, rule: str, where: str, message: str) -> None: + severity = RULES[rule].reader if self.reader else RULES[rule].producer + if severity is not None: + self.items.append(Finding(rule, severity, f"{self.prefix}{where}", message)) + + +def errors(findings: Sequence[Finding]) -> list[Finding]: + """The findings that fail a check.""" + return [finding for finding in findings if finding.severity == ERROR] + + +def parse(source: str | bytes | Path) -> tuple[Any, list[Finding]]: + """JSON text or a file read as the ``json`` rule reads it: the value, or None with the finding.""" + def no_constant(name: str) -> Any: + raise ValueError(f"{name} isn't JSON") + try: + if isinstance(source, Path): + source = source.read_bytes() + text = source.decode("utf-8") if isinstance(source, (bytes, bytearray)) else source + return json.loads(text, parse_constant=no_constant), [] + except (UnicodeDecodeError, ValueError) as exc: + return None, [Finding("json", ERROR, "", str(exc))] + + +def validate(document: Any, *, reader: bool = False, + media: Path | Mapping[str, bytes] | None = None) -> list[Finding]: + """Every finding for ``document``. ``reader`` checks someone else's document for reading rather than a producer's + output. ``media`` is the folder the document's media paths are relative to, or the files by path; without it, only + the paths' form is checked.""" + found = _Findings(reader) + _check(document, found, media) + return found.items + + +def _check(document: Any, found: _Findings, media: Path | Mapping[str, bytes] | None) -> None: + if not isinstance(document, Mapping): + found.add("json", "", "a document is a JSON object") + return + try: + json.dumps(document, allow_nan=False) + except (TypeError, ValueError) as exc: + found.add("json", "", str(exc)) + return + try: + parsed = Trajectory.model_validate(document) + except ValidationError as exc: + for error in exc.errors(include_url=False): + found.add("atif", _loc(error["loc"]), error["msg"]) + return + objects = list(_objects(document)) + _profiles(objects, found) + _outside_keys(objects, found) + _calls_and_references(document, found) + _totals(document, found) + _usage(document, found) + _media(document, found, media) + _timestamps(document, found) + saved = parsed.to_json_dict() + if saved != document: + found.add("harbor-save", _difference(document, saved), "Harbor's to_json_dict gives it back changed") + for where, step in _steps(document): + message = step.get("message") + if isinstance(message, list) and message and all( + isinstance(p, Mapping) and p.get("type") == "text" for p in message): + found.add("text-message", f"{where}.message", "every part is text, so the message is one string") + + +def validate_live(pages: Sequence[Mapping[str, Any] | None], final: Mapping[str, Any], *, + media: Path | Mapping[str, bytes] | None = None) -> list[Finding]: + """The live rule, for a producer: each page a reader was served, in order, validates, is a prefix of the next and + of the final read, and serves no step whose call is unanswered before its task ends. The final read is checked + too.""" + findings: list[Finding] = [] + served: tuple[int, Mapping[str, Any]] | None = None + for index, page in enumerate(pages): + if page is None: + if served is not None: + findings.append(Finding("live-prefix", ERROR, f"page {index}: ", + f"serves nothing after page {served[0]} served steps")) + continue + found = _Findings(False, f"page {index}: ") + _check(page, found, media) + _unanswered(page, found) + findings.extend(found.items) + if served is not None and not is_prefix(served[1], page): + findings.append(Finding("live-prefix", ERROR, f"page {index}: ", + f"page {served[0]} isn't a prefix of it: {_difference(served[1], page)}")) + served = (index, page) + found = _Findings(False, "final: ") + _check(final, found, media) + findings.extend(found.items) + if served is not None and not is_prefix(served[1], final): + findings.append(Finding("live-prefix", ERROR, "final: ", + f"page {served[0]} isn't a prefix of it: {_difference(served[1], final)}")) + return findings + + +# What the document holds + +def _trajectories(document: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: + yield "", document + for index, sub in enumerate(document.get("subagent_trajectories") or []): + yield f"subagent_trajectories[{index}].", sub + + +def _steps(document: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: + for prefix, trajectory in _trajectories(document): + for index, step in enumerate(trajectory.get("steps") or []): + yield f"{prefix}steps[{index}]", step + + +def _results(step: Mapping[str, Any]) -> list[Mapping[str, Any]]: + return list((step.get("observation") or {}).get("results") or []) + + +def _objects(document: Mapping[str, Any]) -> Iterator[tuple[str, str, Mapping[str, Any]]]: + """Every ATIF object that may have an ``extra``: its kind, as PROFILES names it, its path and the object.""" + for prefix, trajectory in _trajectories(document): + yield "Trajectory", prefix.rstrip(".") or "", trajectory + yield "Agent", f"{prefix}agent", trajectory.get("agent") or {} + if trajectory.get("final_metrics"): + yield "FinalMetrics", f"{prefix}final_metrics", trajectory["final_metrics"] + for index, step in enumerate(trajectory.get("steps") or []): + where = f"{prefix}steps[{index}]" + yield "Step", where, step + if step.get("metrics"): + yield "Metrics", f"{where}.metrics", step["metrics"] + for n, call in enumerate(step.get("tool_calls") or []): + yield "ToolCall", f"{where}.tool_calls[{n}]", call + for n, result in enumerate(_results(step)): + yield "ObservationResult", f"{where}.observation.results[{n}]", result + for m, ref in enumerate(result.get("subagent_trajectory_ref") or []): + yield "SubagentTrajectoryRef", f"{where}.observation.results[{n}].subagent_trajectory_ref[{m}]", ref + + +def _profiles(objects: list[tuple[str, str, Mapping[str, Any]]], found: _Findings) -> None: + for kind, where, obj in objects: + extra = obj.get("extra") + if not isinstance(extra, Mapping) or PROFILE_KEY not in extra: + continue + at = f"{where}.extra.{PROFILE_KEY}".lstrip(".") + profile = extra[PROFILE_KEY] + if not isinstance(profile, Mapping): + found.add("profile", at, "the profile is an object") + continue + try: + PROFILES[kind].model_validate(profile) + except ValidationError as exc: + for error in exc.errors(include_url=False): + found.add("profile", f"{at}.{_loc(error['loc'])}", error["msg"]) + continue + _known_keys(kind, at, profile, found) + for (owner, name), vocabulary in _VOCABULARY_AT.items(): + if owner == kind and profile.get(name) is not None: + _vocabulary(vocabulary, f"{at}.{name}", profile[name], found) + error = profile.get("error") if kind == "Step" else None + if isinstance(error, Mapping) and error.get("kind") is not None: + _vocabulary("StepError.kind", f"{at}.error.kind", error["kind"], found) + for n, block in enumerate(profile.get("reasoning") or [] if kind == "Step" else []): + if isinstance(block, Mapping) and block.get("visibility") is not None: + _vocabulary("ReasoningBlock.visibility", f"{at}.reasoning[{n}].visibility", block["visibility"], found) + if kind == "Trajectory" and where == "" and profile.get("version") not in (None, PROFILE_VERSION): + found.add("profile-version", f"{at}.version", + f"this is profile {profile['version']!r}; the validator checks profile {PROFILE_VERSION}") + + +def _known_keys(kind: str, at: str, profile: Mapping[str, Any], found: _Findings) -> None: + def unknown(model: type[BaseModel], value: Mapping[str, Any], path: str) -> None: + for key in value: + if key not in model.model_fields: + found.add("profile-key", f"{path}.{key}", f"profile {PROFILE_VERSION} defines no {key!r} here") + unknown(PROFILES[kind], profile, at) + for (owner, name), model in _NESTED.items(): + if owner == kind and isinstance(profile.get(name), Mapping): + unknown(model, profile[name], f"{at}.{name}") + for (owner, name), model in _LISTS.items(): + if owner == kind: + for n, item in enumerate(profile.get(name) or []): + if isinstance(item, Mapping): + unknown(model, item, f"{at}.{name}[{n}]") + for (owner, name), model in _MAPS.items(): + if owner == kind and isinstance(profile.get(name), Mapping): + for key, item in profile[name].items(): + if isinstance(item, Mapping): + unknown(model, item, f"{at}.{name}[{key}]") + + +def _vocabulary(name: str, where: str, value: Any, found: _Findings) -> None: + if value not in VOCABULARIES[name]: + found.add("vocabulary", where, f"{value!r} isn't one of {', '.join(VOCABULARIES[name])}") + + +def _outside_keys(objects: list[tuple[str, str, Mapping[str, Any]]], found: _Findings) -> None: + for kind, where, obj in objects: + extra = obj.get("extra") + for key in extra if isinstance(extra, Mapping) else (): + if key != PROFILE_KEY and not (kind == "ToolCall" and key == RAW_ARGUMENTS): + found.add("outside-key", f"{where}.extra.{key}".lstrip("."), + f"the profile names no key outside extra.{PROFILE_KEY} here") + + +def _calls_and_references(document: Mapping[str, Any], found: _Findings) -> None: + embedded = {sub.get("trajectory_id") for sub in document.get("subagent_trajectories") or []} + seen: dict[str, str] = {} + for where, step in _steps(document): + for n, call in enumerate(step.get("tool_calls") or []): + call_id = call.get("tool_call_id") + if call_id in seen: + found.add("call-id", f"{where}.tool_calls[{n}]", f"{call_id!r} is also the id at {seen[call_id]}") + else: + seen[call_id] = f"{where}.tool_calls[{n}]" + for n, result in enumerate(_results(step)): + for m, ref in enumerate(result.get("subagent_trajectory_ref") or []): + at = f"{where}.observation.results[{n}].subagent_trajectory_ref[{m}]" + trajectory_id = ref.get("trajectory_id") + if trajectory_id is None: + if not found.reader: + found.add("reference", at, "names no embedded sub-agent: the profile embeds every sub-agent") + elif trajectory_id not in embedded: + found.add("reference", at, f"no embedded sub-agent has trajectory_id {trajectory_id!r}") + + +def _referenced(document: Mapping[str, Any]) -> list[Mapping[str, Any]]: + """The root and every embedded sub-agent its steps reference, directly or through another sub-agent.""" + subs = {sub.get("trajectory_id"): sub for sub in document.get("subagent_trajectories") or []} + out: list[Mapping[str, Any]] = [document] + reached: set[Any] = set() + for trajectory in out: + for step in trajectory.get("steps") or []: + for result in _results(step): + for ref in result.get("subagent_trajectory_ref") or []: + trajectory_id = ref.get("trajectory_id") + if trajectory_id in subs and trajectory_id not in reached: + reached.add(trajectory_id) + out.append(subs[trajectory_id]) + return out + + +def _totals(document: Mapping[str, Any], found: _Findings) -> None: + totals = document.get("final_metrics") + if not isinstance(totals, Mapping): + return + model_steps = [step.get("metrics") or {} for trajectory in _referenced(document) + for step in trajectory.get("steps") or [] + if step.get("source") == "agent" and step.get("llm_call_count") != 0] + for count, total in _COUNTS: + if total in totals and all(count in metrics for metrics in model_steps): + expected = sum(metrics[count] for metrics in model_steps) + if totals[total] != expected: + found.add("totals", f"final_metrics.{total}", + f"is {totals[total]}, but its model steps sum to {expected}") + writes = profile_of(totals).get("total_cache_write_tokens") + step_writes = [profile_of(metrics).get("cache_write_tokens") for metrics in model_steps] + if writes is not None and all(isinstance(w, int) for w in step_writes): + if writes != sum(step_writes): + found.add("totals", f"final_metrics.extra.{PROFILE_KEY}.total_cache_write_tokens", + f"is {writes}, but its model steps sum to {sum(step_writes)}") + steps = len(document.get("steps") or []) + if totals.get("total_steps") is not None and totals["total_steps"] != steps and not document.get("notes"): + found.add("total-steps", "final_metrics.total_steps", + f"is {totals['total_steps']}, but the root has {steps} steps and its notes don't say why") + + +def _usage(document: Mapping[str, Any], found: _Findings) -> None: + for where, step in _steps(document): + metrics = step.get("metrics") or {} + if metrics.get("prompt_tokens") is None: + continue + writes = profile_of(metrics).get("cache_write_tokens") or 0 + cache = (metrics.get("cached_tokens") or 0) + (writes if isinstance(writes, int) else 0) + if cache > metrics["prompt_tokens"]: + found.add("usage", f"{where}.metrics", + f"cache reads and writes ({cache}) exceed prompt_tokens ({metrics['prompt_tokens']}), " + "which include them") + + +def _media(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[str, bytes] | None) -> None: + for where, path, info in _media_paths(document): + if "://" in path: + found.add("media-path", where, f"{path!r} is a URL; the profile's media are files beside the document") + continue + named = MEDIA_PATH.match(path) + if not named: + found.add("media-path", where, f"{path!r} isn't media/.") + if media is None: + continue + data = _media_bytes(media, path) + if data is None: + found.add("media-bytes", where, f"{path} isn't in the media given") + continue + digest = hashlib.sha256(data).hexdigest() + if named and named["sha256"] != digest: + found.add("media-bytes", where, f"{path}'s bytes have sha256 {digest}") + if info.get("sha256") is not None and info["sha256"] != digest: + found.add("media-bytes", where, f"the profile gives sha256 {info['sha256']}, but the bytes have {digest}") + if info.get("bytes") is not None and info["bytes"] != len(data): + found.add("media-bytes", where, f"the profile gives {info['bytes']} bytes, but the file has {len(data)}") + + +def _media_paths(document: Mapping[str, Any]) -> Iterator[tuple[str, str, Mapping[str, Any]]]: + """Each media path the document names, where, and the profile's details for it.""" + for where, step in _steps(document): + holders = [(f"{where}.message", step, step.get("message"))] + [ + (f"{where}.observation.results[{n}].content", result, result.get("content")) + for n, result in enumerate(_results(step))] + for at, holder, parts in holders: + details = profile_of(holder).get("media") or {} + details = details if isinstance(details, Mapping) else {} + paths = [(f"{at}[{n}].source.path", part["source"]["path"]) + for n, part in enumerate(parts if isinstance(parts, list) else []) + if isinstance(part, Mapping) and isinstance((part.get("source") or {}).get("path"), str)] + listed = {path for _, path in paths} + owner = at.rsplit(".", 1)[0] + paths += [(f"{owner}.extra.{PROFILE_KEY}.media[{path}]", path) for path in details if path not in listed] + for path_where, path in paths: + info = details.get(path) + yield path_where, path, info if isinstance(info, Mapping) else {} + + +def _media_bytes(media: Path | Mapping[str, bytes], path: str) -> bytes | None: + if isinstance(media, Mapping): + return media.get(path) + root = media.resolve() + target = (root / path).resolve() + if not target.is_relative_to(root) or not target.is_file(): + return None + return target.read_bytes() + + +def _timestamps(document: Mapping[str, Any], found: _Findings) -> None: + for where, step in _steps(document): + if isinstance(step.get("timestamp"), str) and not TIMESTAMP.match(step["timestamp"]): + found.add("timestamp", f"{where}.timestamp", f"{step['timestamp']!r} isn't in the builder's form") + for n, result in enumerate(_results(step)): + arrived = profile_of(result).get("arrived") + at = arrived.get("at") if isinstance(arrived, Mapping) else None + if isinstance(at, str) and not TIMESTAMP.match(at): + found.add("timestamp", f"{where}.observation.results[{n}].extra.{PROFILE_KEY}.arrived.at", + f"{at!r} isn't in the builder's form") + + +def _unanswered(page: Mapping[str, Any], found: _Findings) -> None: + """A page's steps whose calls have no result yet, served before their task ended.""" + ended_steps: set[int] = set() + offset = 0 + for task in tasks(page): + if task.state is not None: + ended_steps.update(range(offset, offset + len(task.steps))) + offset += len(task.steps) + spawner: dict[Any, int] = {} + for index, step in enumerate(page.get("steps") or []): + for result in _results(step): + for ref in result.get("subagent_trajectory_ref") or []: + spawner.setdefault(ref.get("trajectory_id"), index) + for prefix, trajectory in _trajectories(page): + root_index = None if not prefix else spawner.get(trajectory.get("trajectory_id")) + for index, step in enumerate(trajectory.get("steps") or []): + if (index if not prefix else root_index) in ended_steps: + continue + answered = {result.get("source_call_id") for result in _results(step)} + for n, call in enumerate(step.get("tool_calls") or []): + if call.get("tool_call_id") not in answered: + found.add("live-unanswered", f"{prefix}steps[{index}].tool_calls[{n}]", + f"call {call.get('tool_call_id')!r} has no result, and its task hasn't ended") + + +def _loc(loc: Sequence[Any]) -> str: + out = "" + for part in loc: + out += f"[{part}]" if isinstance(part, int) else (f".{part}" if out else str(part)) + return out + + +def _difference(a: Any, b: Any, where: str = "") -> str: + """The first path at which two JSON values differ.""" + if isinstance(a, Mapping) and isinstance(b, Mapping): + for key in list(a) + [k for k in b if k not in a]: + if key not in a or key not in b: + return f"{where}.{key}".lstrip(".") + (" is missing" if key not in b else " is added") + if a[key] != b[key]: + return _difference(a[key], b[key], f"{where}.{key}".lstrip(".")) + if isinstance(a, list) and isinstance(b, list): + for index, (x, y) in enumerate(zip(a, b)): + if x != y: + return _difference(x, y, f"{where}[{index}]") + if len(a) != len(b): + return f"{where} has {len(a)} items, then {len(b)}" + return where or "the document" diff --git a/packages/agentenv-protocol/tests/test_atif_kit.py b/packages/agentenv-protocol/tests/test_atif_kit.py new file mode 100644 index 000000000..aa89656cc --- /dev/null +++ b/packages/agentenv-protocol/tests/test_atif_kit.py @@ -0,0 +1,96 @@ +"""The adapter kit: the toy adapter passes against its own fixtures, and the kit names what's wrong with an adapter that +writes a duplicate tool_call_id, a document other than the expected one, or a from_atif that gives no native log.""" +import json +from typing import Any + +import pytest + +from agentenv_protocol.atif import ( + Fed, + Native, + RecordedTask, + TurnContext, + atif_adapter, + check_adapter, + native, + strip_profile, +) +from atif_cases import CASES, FIXTURES, Toy + +REPLAYABLE = sorted(name for name, tasks in CASES.items() if not any(task.inbound_media for task in tasks)) +DECLARED = {"native_format": "toy-events/1", "version": "0.1.0", "input_includes_cache": False, + "tested_with": ("toy 1.0",)} + + +@atif_adapter(**DECLARED) +class Reversible(Toy): + @classmethod + def from_atif(cls, document: Any) -> Native: + return Native(log=[{"type": "say", "step": f"s{step['step_id']}", "text": step["message"]} + for step in document["steps"] if step["source"] == "agent" + and isinstance(step.get("message"), str)]) + + +@atif_adapter(**DECLARED) +class Duplicating(Reversible): + @native(type="call") + def call(self, r: dict, turn: Any) -> None: + turn.tool_call(step=r["step"], tool_call_id="toolu_01", function_name=r["name"], arguments=r.get("args")) + + +@atif_adapter(**DECLARED) +class NoLog(Toy): + @classmethod + def from_atif(cls, document: Any) -> Any: + return {"log": []} + + +def recorded(name: str) -> list[RecordedTask]: + return [RecordedTask(context=TurnContext(**task.context), records=[Fed(r) for r in task.records], + state=task.state, final=task.final, ended_at=task.ended_at) for task in CASES[name]] + + +def _fixture(name: str) -> dict[str, Any]: + return json.loads((FIXTURES / f"{name}.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("name", REPLAYABLE) +def test_the_kit_passes_the_toy_adapter_against_its_fixture(name): + stripped = json.loads((FIXTURES / "coding.stripped.json").read_text(encoding="utf-8")) + result = check_adapter(Reversible, recorded(name), expected=FIXTURES / f"{name}.json", + from_atif=[stripped, strip_profile(_fixture(name))]) + assert result.errors == [] + assert result.document == _fixture(name) + assert len(result.pages) == sum(len(task.records) + 2 for task in CASES[name]) + unmatched = "unmatched" in _fixture(name)["extra"]["agentenv"] + assert {f.rule for f in result.findings} == ({"kit-unmatched"} if unmatched else set()) + + +def test_the_kit_warns_of_native_records_no_handler_takes(): + (finding,) = check_adapter(Reversible, recorded("coding")).findings + assert (finding.rule, finding.severity, finding.message) == ( + "kit-unmatched", "warning", "no handler takes these native records: system (1)") + + +def test_the_kit_fails_an_adapter_that_writes_a_duplicate_tool_call_id(): + result = check_adapter(Duplicating, recorded("coding")) + (finding,) = result.errors + assert (finding.rule, finding.where) == ("kit-run", "task 0, record 4") + assert "tool_call_id 'toolu_01' is already in this document" in finding.message + with pytest.raises(AssertionError, match="kit-run"): + result.raise_for_errors() + + +def test_the_kit_names_where_the_document_differs_from_the_expected_one(): + expected = _fixture("coding") + expected["steps"][5]["message"] = "something else" + (finding,) = check_adapter(Reversible, recorded("coding"), expected=expected).errors + assert (finding.rule, finding.where) == ("kit-expected", "steps[5].message") + + +def test_the_kit_fails_a_from_atif_that_gives_no_native_log_and_warns_when_there_is_none(): + (failed,) = check_adapter(NoLog, recorded("late_result")).errors + assert (failed.rule, failed.message) == ("kit-from-atif", "returned dict, not a Native holding a list log") + result = check_adapter(Toy, recorded("late_result")) + assert result.errors == [] + assert [(f.rule, f.severity) for f in result.findings] == [("kit-from-atif", "warning")] diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py new file mode 100644 index 000000000..26d8be2e5 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -0,0 +1,204 @@ +"""The validator: the fixtures and every live page they were served are clean, a negative fixture for each rule fails +with that rule's id at the severity PROFILE.md gives it, and the CLI reports it.""" +import copy +import json +import subprocess +import sys +from importlib.resources import files +from typing import Any, Callable + +import pytest + +from agentenv_protocol.atif import KIT_RULES, RULES, errors, validate, validate_live +from agentenv_protocol.atif.__main__ import main +from atif_cases import CASES, FIXTURES, build + +SPEC = files("agentenv_protocol.atif").joinpath("PROFILE.md").read_text(encoding="utf-8") + + +def _fixture(name: str) -> dict[str, Any]: + return json.loads((FIXTURES / f"{name}.json").read_text(encoding="utf-8")) + + +def _rules(findings, severity: str | None = None) -> set[str]: + return {f.rule for f in findings if severity is None or f.severity == severity} + + +def _parts(document: dict[str, Any]) -> list[dict[str, Any]]: + return [part for step in document["steps"] for result in (step.get("observation") or {}).get("results", []) + for part in result.get("content") or [] if isinstance(part, dict) and part.get("type") == "image"] + + +def _duplicate_call_id(document: dict[str, Any]) -> None: + step = document["steps"][2] + step["tool_calls"][0]["tool_call_id"] = "toolu_01" + step["observation"]["results"][0]["source_call_id"] = "toolu_01" + + +def _dangling_reference(document: dict[str, Any]) -> None: + for step in document["steps"]: + for result in (step.get("observation") or {}).get("results", []): + for ref in result.get("subagent_trajectory_ref") or []: + ref["trajectory_id"] = "nobody" + + +def _rename_media(document: dict[str, Any]) -> None: + _parts(document)[0]["source"]["path"] = "screens/1.png" + + +def _one_more(field: str) -> Callable[[dict[str, Any]], None]: + def mutate(document: dict[str, Any]) -> None: + document["final_metrics"][field] += 1 + return mutate + + +def _over_cached(document: dict[str, Any]) -> None: + metrics = document["steps"][1]["metrics"] + metrics["cached_tokens"] = metrics["prompt_tokens"] + 1 + + +def _text_parts(document: dict[str, Any]) -> None: + step = document["steps"][-1] + step["message"] = [{"type": "text", "text": step["message"]}] + + +#: A fixture and the change that breaks one rule. +NEGATIVE: dict[str, tuple[str, Callable[[dict[str, Any]], None]]] = { + "json": ("coding", lambda d: d["steps"][1]["tool_calls"][0]["arguments"].update(x=float("nan"))), + "atif": ("coding", lambda d: d["steps"][1].update(step_id=9)), + "profile": ("coding", lambda d: d["steps"][0]["extra"]["agentenv"].update(turn="zero")), + "profile-key": ("coding", lambda d: d["steps"][0]["extra"]["agentenv"].update(later=1)), + "profile-version": ("coding", lambda d: d["extra"]["agentenv"].update(version="2.0")), + "vocabulary": ("coding", lambda d: d["steps"][-1]["extra"]["agentenv"].update(state="paused")), + "outside-key": ("coding", lambda d: d["steps"][0]["extra"].update(note="x")), + "call-id": ("coding", _duplicate_call_id), + "reference": ("subagent", _dangling_reference), + "totals": ("coding", _one_more("total_prompt_tokens")), + "total-steps": ("coding", _one_more("total_steps")), + "usage": ("coding", _over_cached), + "media-path": ("computer_use", _rename_media), + "timestamp": ("coding", lambda d: d["steps"][0].update(timestamp="2026-11-04T17:02:11.204+00:00")), + "harbor-save": ("coding", lambda d: d["steps"][1].update(model_name=None)), + "text-message": ("coding", _text_parts), +} + + +def _negative(rule: str) -> dict[str, Any]: + name, mutate = NEGATIVE[rule] + document = _fixture(name) + mutate(document) + return document + + +@pytest.mark.parametrize("path", sorted(FIXTURES.glob("*.json")), ids=lambda p: p.stem) +def test_every_fixture_is_clean_for_a_producer_and_for_a_reader(path): + document = json.loads(path.read_text(encoding="utf-8")) + assert validate(document, media=FIXTURES) == [] + assert validate(document, reader=True, media=FIXTURES) == [] + + +@pytest.mark.parametrize("name", sorted(CASES)) +def test_every_page_a_case_served_passes_the_live_rule(name): + document, pages, media = build(name) + assert validate_live(pages, document, media=media) == [] + + +def test_every_rule_has_a_negative_fixture(): + assert set(NEGATIVE) | {"media-bytes", "live-prefix", "live-unanswered"} == set(RULES) + + +@pytest.mark.parametrize("rule", sorted(NEGATIVE)) +def test_a_negative_fixture_fails_with_its_rules_id_and_the_cli_reports_it(rule, tmp_path, capsys): + document = _negative(rule) + assert rule in _rules(errors(validate(document))) + path = tmp_path / "trajectory.json" + path.write_text(json.dumps(document), encoding="utf-8") + assert main(["validate", str(path)]) == 1 + assert f"error {rule} " in capsys.readouterr().out + + +@pytest.mark.parametrize("rule", sorted(NEGATIVE)) +def test_a_reader_sees_each_rule_at_its_reader_severity(rule): + found = {f.severity for f in validate(_negative(rule), reader=True) if f.rule == rule} + assert found == ({RULES[rule].reader} if RULES[rule].reader else set()) + + +def test_an_unknown_profile_key_fails_a_producer_and_only_warns_a_reader(tmp_path, capsys): + document = _negative("profile-key") + assert [(f.rule, f.severity, f.where) for f in validate(document)] == [ + ("profile-key", "error", "steps[0].extra.agentenv.later")] + assert [(f.rule, f.severity) for f in validate(document, reader=True)] == [("profile-key", "warning")] + path = tmp_path / "trajectory.json" + path.write_text(json.dumps(document), encoding="utf-8") + assert main(["validate", "--reader", str(path)]) == 0 + assert "warning profile-key steps[0].extra.agentenv.later" in capsys.readouterr().out + + +def test_media_bytes_are_checked_against_their_name_and_profile(): + document = _fixture("computer_use") + path = _parts(document)[0]["source"]["path"] + assert validate(document, media=FIXTURES) == [] + assert "media-bytes" in _rules(errors(validate(document, media={}))) + assert "media-bytes" in _rules(errors(validate(document, media={path: b"not the screenshot"}))) + + +def test_totals_sum_the_sub_agents_steps_and_skip_a_count_a_step_lacks(): + document = _fixture("subagent") + root = sum(s["metrics"]["prompt_tokens"] for s in document["steps"] if s.get("metrics")) + assert root < document["final_metrics"]["total_prompt_tokens"] + assert validate(document) == [] + sub_step = next(s for s in document["subagent_trajectories"][0]["steps"] if s.get("metrics")) + del sub_step["metrics"]["cached_tokens"] + assert validate(document) == [] + document["final_metrics"]["total_prompt_tokens"] += 1 + assert [f.where for f in validate(document)] == ["final_metrics.total_prompt_tokens"] + + +def _open_page(name: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, bytes]]: + """A case's final document and a page served while its task ran, with two closed steps.""" + document, pages, media = build(name) + page = next(p for p in pages[:-1] if p is not None and len(p["steps"]) >= 3) + return document, copy.deepcopy(page), media + + +def test_the_live_check_flags_a_page_that_differs_from_the_final_read_by_one_step(): + document, page, media = _open_page("coding") + assert validate_live([page], document, media=media) == [] + page["steps"][1]["message"] = "a step served, then changed" + assert [(f.rule, f.where) for f in errors(validate_live([page], document, media=media))] == [ + ("live-prefix", "final: ")] + + +def test_the_live_check_flags_a_page_serving_a_call_with_no_result_before_its_task_ends(): + document, page, media = _open_page("coding") + del page["steps"][1]["observation"] + assert "live-unanswered" in _rules(errors(validate_live([page], document, media=media))) + + +def test_a_page_that_serves_nothing_after_serving_steps_breaks_the_live_rule(): + document, page, media = _open_page("coding") + assert "live-prefix" in _rules(errors(validate_live([page, None], document, media=media))) + + +def test_python_m_runs_the_cli(tmp_path): + clean = subprocess.run([sys.executable, "-m", "agentenv_protocol.atif", "validate", "--media", str(FIXTURES), + str(FIXTURES / "computer_use.json")], capture_output=True, text=True) + assert (clean.returncode, clean.stdout.splitlines()[-1].split(": ")[-1]) == (0, "valid, 0 errors, 0 warnings") + path = tmp_path / "trajectory.json" + path.write_text(json.dumps(_negative("call-id")), encoding="utf-8") + broken = subprocess.run([sys.executable, "-m", "agentenv_protocol.atif", "validate", str(path)], + capture_output=True, text=True) + assert broken.returncode == 1 + assert "error call-id steps[2].tool_calls[0]: 'toolu_01' is also the id at steps[1].tool_calls[0]" in broken.stdout + + +def test_the_cli_exits_2_on_a_file_it_cant_read(tmp_path, capsys): + assert main(["validate", str(tmp_path / "missing.json")]) == 2 + assert "can't read" in capsys.readouterr().err + + +def test_profile_md_lists_every_rule_with_its_severities(): + for rule in RULES.values(): + assert f"| `{rule.id}` | {rule.holds} | {rule.producer} | {rule.reader or 'not checked'} |" in SPEC + for rule in KIT_RULES.values(): + assert f"| `{rule.id}` | {rule.holds} | {rule.producer} |" in SPEC From 21a4da1f7c5f72cb3c524473da93305bcf18e0c0 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:20:29 -0800 Subject: [PATCH 2/5] fix(protocol): the validator sums every embedded sub-agent, places a sub-agent's task through its spawn, and reads a null count as absent Chaos on the validator found three things. final_metrics totals summed only the sub-agents a result references, while the builder, and ATIF's "across all steps", sum every embedded one: a sub-agent whose spawning call never answered is embedded but referenced by nothing. live-unanswered couldn't place such a sub-agent's steps in a task, and flagged them; a sub-agent's task now comes through its chain of spawning calls, nested ones included, and one no served step references isn't checked. A count present as null, which ATIF allows, crashed the totals sum. Difference paths are now bare paths. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/atif/PROFILE.md | 4 +- .../src/agentenv_protocol/atif/validator.py | 67 ++++++++++--------- .../tests/test_atif_validator.py | 2 + 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md index e2b6da52d..35cd851cf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md @@ -152,7 +152,7 @@ cross-check that reads them; per-part origins, when a reader needs them; and an producer's output to profile 1.0. `reader=True` checks someone else's document for reading instead: what a producer must not write only warns, and the document fails only where agentenv couldn't read it. `media` is the folder the document's media paths are relative to, the one holding `media/`, or the files by path; without it only the paths' form -is checked. `validate_live(pages, final)` checks what a live reader was served, page by page, with the final read. On +is checked. `validate_live(pages, final)` checks what a live reader was served, page by page, with the final read. A sub-agent's task is its spawning call's, found through the references, so an embedded sub-agent that no served step references isn't held to `live-unanswered`. On the command line: `python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]`, which prints each finding and exits 1 when one is an error. @@ -167,7 +167,7 @@ and exits 1 when one is an error. | `outside-key` | No `extra` holds a key outside `agentenv` but `ToolCall.extra.raw_arguments` | error | warning | | `call-id` | Each `tool_call_id` is unique across the document and its sub-agents | error | warning | | `reference` | Each `subagent_trajectory_ref` names an embedded sub-agent by `trajectory_id` | error | warning | -| `totals` | A `final_metrics` total equals its summed model steps, when every one has the count | error | warning | +| `totals` | A `final_metrics` total equals the sum over the model steps of the root and its embedded sub-agents, when every one has the count | error | warning | | `total-steps` | `final_metrics.total_steps` counts the root's steps, or the root's `notes` explain the difference | error | warning | | `usage` | A step's cache reads and writes fit within its `prompt_tokens` | error | warning | | `media-path` | A media path is `media/.` | error | warning | diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index 2d29ad577..c8b066e62 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -71,8 +71,8 @@ class Rule: Rule("outside-key", ERROR, WARNING, "No `extra` holds a key outside `agentenv` but `ToolCall.extra.raw_arguments`"), Rule("call-id", ERROR, WARNING, "Each `tool_call_id` is unique across the document and its sub-agents"), Rule("reference", ERROR, WARNING, "Each `subagent_trajectory_ref` names an embedded sub-agent by `trajectory_id`"), - Rule("totals", ERROR, WARNING, "A `final_metrics` total equals its summed model steps, when every one has the " - "count"), + Rule("totals", ERROR, WARNING, "A `final_metrics` total equals the sum over the model steps of the root and its " + "embedded sub-agents, when every one has the count"), Rule("total-steps", ERROR, WARNING, "`final_metrics.total_steps` counts the root's steps, or the root's `notes` " "explain the difference"), Rule("usage", ERROR, WARNING, "A step's cache reads and writes fit within its `prompt_tokens`"), @@ -166,7 +166,7 @@ def _check(document: Any, found: _Findings, media: Path | Mapping[str, bytes] | _timestamps(document, found) saved = parsed.to_json_dict() if saved != document: - found.add("harbor-save", _difference(document, saved), "Harbor's to_json_dict gives it back changed") + found.add("harbor-save", _difference(document, saved), "Harbor's to_json_dict gives this back changed") for where, step in _steps(document): message = step.get("message") if isinstance(message, list) and message and all( @@ -193,14 +193,16 @@ def validate_live(pages: Sequence[Mapping[str, Any] | None], final: Mapping[str, findings.extend(found.items) if served is not None and not is_prefix(served[1], page): findings.append(Finding("live-prefix", ERROR, f"page {index}: ", - f"page {served[0]} isn't a prefix of it: {_difference(served[1], page)}")) + f"page {served[0]} isn't a prefix of it; they first differ at " + f"{_difference(served[1], page) or 'the document'}")) served = (index, page) found = _Findings(False, "final: ") _check(final, found, media) findings.extend(found.items) if served is not None and not is_prefix(served[1], final): findings.append(Finding("live-prefix", ERROR, "final: ", - f"page {served[0]} isn't a prefix of it: {_difference(served[1], final)}")) + f"page {served[0]} isn't a prefix of it; they first differ at " + f"{_difference(served[1], final) or 'the document'}")) return findings @@ -329,31 +331,15 @@ def _calls_and_references(document: Mapping[str, Any], found: _Findings) -> None found.add("reference", at, f"no embedded sub-agent has trajectory_id {trajectory_id!r}") -def _referenced(document: Mapping[str, Any]) -> list[Mapping[str, Any]]: - """The root and every embedded sub-agent its steps reference, directly or through another sub-agent.""" - subs = {sub.get("trajectory_id"): sub for sub in document.get("subagent_trajectories") or []} - out: list[Mapping[str, Any]] = [document] - reached: set[Any] = set() - for trajectory in out: - for step in trajectory.get("steps") or []: - for result in _results(step): - for ref in result.get("subagent_trajectory_ref") or []: - trajectory_id = ref.get("trajectory_id") - if trajectory_id in subs and trajectory_id not in reached: - reached.add(trajectory_id) - out.append(subs[trajectory_id]) - return out - - def _totals(document: Mapping[str, Any], found: _Findings) -> None: totals = document.get("final_metrics") if not isinstance(totals, Mapping): return - model_steps = [step.get("metrics") or {} for trajectory in _referenced(document) + model_steps = [step.get("metrics") or {} for _, trajectory in _trajectories(document) for step in trajectory.get("steps") or [] if step.get("source") == "agent" and step.get("llm_call_count") != 0] for count, total in _COUNTS: - if total in totals and all(count in metrics for metrics in model_steps): + if isinstance(totals.get(total), int) and all(isinstance(m.get(count), int) for m in model_steps): expected = sum(metrics[count] for metrics in model_steps) if totals[total] != expected: found.add("totals", f"final_metrics.{total}", @@ -449,20 +435,35 @@ def _timestamps(document: Mapping[str, Any], found: _Findings) -> None: def _unanswered(page: Mapping[str, Any], found: _Findings) -> None: - """A page's steps whose calls have no result yet, served before their task ended.""" + """A page's steps whose calls have no result yet, served before their task ended. A sub-agent's task is its + spawning call's, found through the references; one no served step references can't be placed, so isn't checked.""" ended_steps: set[int] = set() offset = 0 for task in tasks(page): if task.state is not None: ended_steps.update(range(offset, offset + len(task.steps))) offset += len(task.steps) - spawner: dict[Any, int] = {} - for index, step in enumerate(page.get("steps") or []): - for result in _results(step): - for ref in result.get("subagent_trajectory_ref") or []: - spawner.setdefault(ref.get("trajectory_id"), index) + spawner: dict[Any, tuple[Any, int]] = {} + for _, trajectory in _trajectories(page): + owner = trajectory.get("trajectory_id") if trajectory is not page else None + for index, step in enumerate(trajectory.get("steps") or []): + for result in _results(step): + for ref in result.get("subagent_trajectory_ref") or []: + spawner.setdefault(ref.get("trajectory_id"), (owner, index)) + + def root_step(trajectory_id: Any) -> int | None: + seen = set() + while trajectory_id in spawner and trajectory_id not in seen: + seen.add(trajectory_id) + trajectory_id, index = spawner[trajectory_id] + if trajectory_id is None: + return index + return None + for prefix, trajectory in _trajectories(page): - root_index = None if not prefix else spawner.get(trajectory.get("trajectory_id")) + root_index = root_step(trajectory.get("trajectory_id")) if prefix else None + if prefix and root_index is None: + continue for index, step in enumerate(trajectory.get("steps") or []): if (index if not prefix else root_index) in ended_steps: continue @@ -485,7 +486,7 @@ def _difference(a: Any, b: Any, where: str = "") -> str: if isinstance(a, Mapping) and isinstance(b, Mapping): for key in list(a) + [k for k in b if k not in a]: if key not in a or key not in b: - return f"{where}.{key}".lstrip(".") + (" is missing" if key not in b else " is added") + return f"{where}.{key}".lstrip(".") if a[key] != b[key]: return _difference(a[key], b[key], f"{where}.{key}".lstrip(".")) if isinstance(a, list) and isinstance(b, list): @@ -493,5 +494,5 @@ def _difference(a: Any, b: Any, where: str = "") -> str: if x != y: return _difference(x, y, f"{where}[{index}]") if len(a) != len(b): - return f"{where} has {len(a)} items, then {len(b)}" - return where or "the document" + return f"{where}[{min(len(a), len(b))}]" + return where diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py index 26d8be2e5..793430832 100644 --- a/packages/agentenv-protocol/tests/test_atif_validator.py +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -152,6 +152,8 @@ def test_totals_sum_the_sub_agents_steps_and_skip_a_count_a_step_lacks(): assert validate(document) == [] document["final_metrics"]["total_prompt_tokens"] += 1 assert [f.where for f in validate(document)] == ["final_metrics.total_prompt_tokens"] + sub_step["metrics"]["prompt_tokens"] = None + assert "totals" not in _rules(validate(document)) def _open_page(name: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, bytes]]: From 919884d07aa277de29d0cb3f6ac52cfa14262a47 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:26:12 -0800 Subject: [PATCH 3/5] fix(protocol): the validator walks sub-agents at any depth, counts as ATIF's models read them, and the kit takes recorded media Greptile's first review: nested subagent_trajectories, which ATIF allows, escaped every check past the first level; the walk now goes to any depth with full paths, and a reference resolves to a sub-agent embedded in its trajectory or one around it. A count given as a numeric string, which the models accept, reached arithmetic in usage; counts are now read through the models' own int coercion. check_adapter takes media= for the files of Media in the recorded tasks' inbound or final content, so a case with attached files, odd_parts, replays through the kit. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/atif/kit.py | 8 +- .../src/agentenv_protocol/atif/validator.py | 109 +++++++++++------- .../agentenv-protocol/tests/test_atif_kit.py | 34 ++++-- .../tests/test_atif_validator.py | 36 ++++++ 4 files changed, 129 insertions(+), 58 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py index 3fff208f8..874ea5f2c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py @@ -43,10 +43,12 @@ def raise_for_errors(self) -> None: def check_adapter(adapter: type[Adapter], recorded: Sequence[RecordedTask], *, expected: Mapping[str, Any] | Path | None = None, - from_atif: Sequence[Mapping[str, Any]] | None = None) -> KitResult: + from_atif: Sequence[Mapping[str, Any]] | None = None, + media: Mapping[str, bytes] | None = None) -> KitResult: """Run ``adapter`` over ``recorded``, one A2A context's tasks in order, and check it. ``expected`` is the document the run should write, or the file holding it. ``from_atif`` are the documents its ``from_atif`` is tried on; by - default, the run's own document with the profile stripped.""" + default, the run's own document with the profile stripped. ``media`` holds, by path, the files of any Media in the + recorded tasks' inbound or final content; the adapter's own media come from the run.""" run = AdapterRun(adapter) result = KitResult(document=None) where = "" @@ -69,7 +71,7 @@ def check_adapter(adapter: type[Adapter], recorded: Sequence[RecordedTask], *, result.findings.append(Finding("kit-run", ERROR, where, f"{type(exc).__name__}: {exc}")) return result result.document = run.document() - result.media = run.media_files + result.media = {**(media or {}), **run.media_files} if result.document is None: result.findings.append(Finding("kit-run", ERROR, "", "the run wrote no step")) return result diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index c8b066e62..8f751f7db 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any, Iterator, Mapping, Sequence -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from .models import Trajectory from .profile import ( @@ -35,6 +35,7 @@ TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3}|\.\d{6})?Z$") _COUNTS = (("prompt_tokens", "total_prompt_tokens"), ("completion_tokens", "total_completion_tokens"), ("cached_tokens", "total_cached_tokens")) +_INT = TypeAdapter(int) #: A profile field that holds a nested profile object, by its owner and name. _NESTED: dict[tuple[str, str], type[BaseModel]] = { ("Trajectory", "adapter"): AdapterRef, ("Step", "error"): StepError, @@ -208,10 +209,19 @@ def validate_live(pages: Sequence[Mapping[str, Any] | None], final: Mapping[str, # What the document holds -def _trajectories(document: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: - yield "", document +def _trajectories(document: Mapping[str, Any], prefix: str = "") -> Iterator[tuple[str, Mapping[str, Any]]]: + """The document and every sub-agent embedded in it, at any depth, with the path prefix of each.""" + yield prefix, document for index, sub in enumerate(document.get("subagent_trajectories") or []): - yield f"subagent_trajectories[{index}].", sub + yield from _trajectories(sub, f"{prefix}subagent_trajectories[{index}].") + + +def _count(value: Any) -> int | None: + """A token count as ATIF's models read it, a numeric string included, or None when there is none.""" + try: + return None if value is None else _INT.validate_python(value) + except ValidationError: + return None def _steps(document: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: @@ -310,25 +320,41 @@ def _outside_keys(objects: list[tuple[str, str, Mapping[str, Any]]], found: _Fin f"the profile names no key outside extra.{PROFILE_KEY} here") +def _scopes(document: Mapping[str, Any], prefix: str = "", outer: frozenset = frozenset()) -> dict[str, frozenset]: + """The sub-agent ids a reference in each trajectory may name: those embedded in it or in a trajectory around it, + since the builder embeds every sub-agent in the root while ATIF also lets a sub-agent embed its own.""" + scope = outer | {sub.get("trajectory_id") for sub in document.get("subagent_trajectories") or []} + scopes = {prefix: scope} + for index, sub in enumerate(document.get("subagent_trajectories") or []): + scopes.update(_scopes(sub, f"{prefix}subagent_trajectories[{index}].", scope)) + return scopes + + def _calls_and_references(document: Mapping[str, Any], found: _Findings) -> None: - embedded = {sub.get("trajectory_id") for sub in document.get("subagent_trajectories") or []} + scopes = _scopes(document) seen: dict[str, str] = {} - for where, step in _steps(document): - for n, call in enumerate(step.get("tool_calls") or []): - call_id = call.get("tool_call_id") - if call_id in seen: - found.add("call-id", f"{where}.tool_calls[{n}]", f"{call_id!r} is also the id at {seen[call_id]}") - else: - seen[call_id] = f"{where}.tool_calls[{n}]" - for n, result in enumerate(_results(step)): - for m, ref in enumerate(result.get("subagent_trajectory_ref") or []): - at = f"{where}.observation.results[{n}].subagent_trajectory_ref[{m}]" - trajectory_id = ref.get("trajectory_id") - if trajectory_id is None: - if not found.reader: - found.add("reference", at, "names no embedded sub-agent: the profile embeds every sub-agent") - elif trajectory_id not in embedded: - found.add("reference", at, f"no embedded sub-agent has trajectory_id {trajectory_id!r}") + for prefix, trajectory in _trajectories(document): + for index, step in enumerate(trajectory.get("steps") or []): + _step_calls(f"{prefix}steps[{index}]", step, scopes[prefix], seen, found) + + +def _step_calls(where: str, step: Mapping[str, Any], embedded: frozenset, seen: dict[str, str], + found: _Findings) -> None: + for n, call in enumerate(step.get("tool_calls") or []): + call_id = call.get("tool_call_id") + if call_id in seen: + found.add("call-id", f"{where}.tool_calls[{n}]", f"{call_id!r} is also the id at {seen[call_id]}") + else: + seen[call_id] = f"{where}.tool_calls[{n}]" + for n, result in enumerate(_results(step)): + for m, ref in enumerate(result.get("subagent_trajectory_ref") or []): + at = f"{where}.observation.results[{n}].subagent_trajectory_ref[{m}]" + trajectory_id = ref.get("trajectory_id") + if trajectory_id is None: + if not found.reader: + found.add("reference", at, "names no embedded sub-agent: the profile embeds every sub-agent") + elif trajectory_id not in embedded: + found.add("reference", at, f"no embedded sub-agent has trajectory_id {trajectory_id!r}") def _totals(document: Mapping[str, Any], found: _Findings) -> None: @@ -338,35 +364,32 @@ def _totals(document: Mapping[str, Any], found: _Findings) -> None: model_steps = [step.get("metrics") or {} for _, trajectory in _trajectories(document) for step in trajectory.get("steps") or [] if step.get("source") == "agent" and step.get("llm_call_count") != 0] - for count, total in _COUNTS: - if isinstance(totals.get(total), int) and all(isinstance(m.get(count), int) for m in model_steps): - expected = sum(metrics[count] for metrics in model_steps) - if totals[total] != expected: - found.add("totals", f"final_metrics.{total}", - f"is {totals[total]}, but its model steps sum to {expected}") - writes = profile_of(totals).get("total_cache_write_tokens") - step_writes = [profile_of(metrics).get("cache_write_tokens") for metrics in model_steps] - if writes is not None and all(isinstance(w, int) for w in step_writes): - if writes != sum(step_writes): - found.add("totals", f"final_metrics.extra.{PROFILE_KEY}.total_cache_write_tokens", - f"is {writes}, but its model steps sum to {sum(step_writes)}") + sums = [(f"final_metrics.{total}", _count(totals.get(total)), [_count(m.get(count)) for m in model_steps]) + for count, total in _COUNTS] + sums.append((f"final_metrics.extra.{PROFILE_KEY}.total_cache_write_tokens", + _count(profile_of(totals).get("total_cache_write_tokens")), + [_count(profile_of(m).get("cache_write_tokens")) for m in model_steps])) + for where, total, counts in sums: + if total is not None and None not in counts and total != sum(counts): + found.add("totals", where, f"is {total}, but its model steps sum to {sum(counts)}") steps = len(document.get("steps") or []) - if totals.get("total_steps") is not None and totals["total_steps"] != steps and not document.get("notes"): + total_steps = _count(totals.get("total_steps")) + if total_steps is not None and total_steps != steps and not document.get("notes"): found.add("total-steps", "final_metrics.total_steps", - f"is {totals['total_steps']}, but the root has {steps} steps and its notes don't say why") + f"is {total_steps}, but the root has {steps} steps and its notes don't say why") def _usage(document: Mapping[str, Any], found: _Findings) -> None: for where, step in _steps(document): metrics = step.get("metrics") or {} - if metrics.get("prompt_tokens") is None: + prompt = _count(metrics.get("prompt_tokens")) + if prompt is None: continue - writes = profile_of(metrics).get("cache_write_tokens") or 0 - cache = (metrics.get("cached_tokens") or 0) + (writes if isinstance(writes, int) else 0) - if cache > metrics["prompt_tokens"]: + writes = _count(profile_of(metrics).get("cache_write_tokens")) + cache = (_count(metrics.get("cached_tokens")) or 0) + (writes or 0) + if cache > prompt: found.add("usage", f"{where}.metrics", - f"cache reads and writes ({cache}) exceed prompt_tokens ({metrics['prompt_tokens']}), " - "which include them") + f"cache reads and writes ({cache}) exceed prompt_tokens ({prompt}), which include them") def _media(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[str, bytes] | None) -> None: @@ -444,8 +467,8 @@ def _unanswered(page: Mapping[str, Any], found: _Findings) -> None: ended_steps.update(range(offset, offset + len(task.steps))) offset += len(task.steps) spawner: dict[Any, tuple[Any, int]] = {} - for _, trajectory in _trajectories(page): - owner = trajectory.get("trajectory_id") if trajectory is not page else None + for prefix, trajectory in _trajectories(page): + owner = trajectory.get("trajectory_id") if prefix else None for index, step in enumerate(trajectory.get("steps") or []): for result in _results(step): for ref in result.get("subagent_trajectory_ref") or []: diff --git a/packages/agentenv-protocol/tests/test_atif_kit.py b/packages/agentenv-protocol/tests/test_atif_kit.py index aa89656cc..45054f6aa 100644 --- a/packages/agentenv-protocol/tests/test_atif_kit.py +++ b/packages/agentenv-protocol/tests/test_atif_kit.py @@ -6,6 +6,7 @@ import pytest from agentenv_protocol.atif import ( + AdapterRun, Fed, Native, RecordedTask, @@ -17,7 +18,6 @@ ) from atif_cases import CASES, FIXTURES, Toy -REPLAYABLE = sorted(name for name, tasks in CASES.items() if not any(task.inbound_media for task in tasks)) DECLARED = {"native_format": "toy-events/1", "version": "0.1.0", "input_includes_cache": False, "tested_with": ("toy 1.0",)} @@ -45,20 +45,30 @@ def from_atif(cls, document: Any) -> Any: return {"log": []} -def recorded(name: str) -> list[RecordedTask]: - return [RecordedTask(context=TurnContext(**task.context), records=[Fed(r) for r in task.records], - state=task.state, final=task.final, ended_at=task.ended_at) for task in CASES[name]] +def recorded(name: str) -> tuple[list[RecordedTask], dict[str, bytes]]: + """A case's tasks as the SDK records them, and the files of the media their inbound content attaches.""" + store, out = AdapterRun(Toy), [] + for task in CASES[name]: + context = dict(task.context) + inbound = list(context.pop("inbound", [])) + inbound += [store.media(label.encode(), media_type, **info) for label, media_type, info in task.inbound_media] + if task.inbound_media and name == "odd_parts": + inbound.append({"type": "text", "text": json.dumps({"ticket": 4411, "priority": "high"})}) + out.append(RecordedTask(context=TurnContext(inbound=inbound, **context), records=[Fed(r) for r in task.records], + state=task.state, final=task.final, ended_at=task.ended_at)) + return out, store.media_files def _fixture(name: str) -> dict[str, Any]: return json.loads((FIXTURES / f"{name}.json").read_text(encoding="utf-8")) -@pytest.mark.parametrize("name", REPLAYABLE) +@pytest.mark.parametrize("name", sorted(CASES)) def test_the_kit_passes_the_toy_adapter_against_its_fixture(name): stripped = json.loads((FIXTURES / "coding.stripped.json").read_text(encoding="utf-8")) - result = check_adapter(Reversible, recorded(name), expected=FIXTURES / f"{name}.json", - from_atif=[stripped, strip_profile(_fixture(name))]) + tasks, media = recorded(name) + result = check_adapter(Reversible, tasks, expected=FIXTURES / f"{name}.json", + from_atif=[stripped, strip_profile(_fixture(name))], media=media) assert result.errors == [] assert result.document == _fixture(name) assert len(result.pages) == sum(len(task.records) + 2 for task in CASES[name]) @@ -67,13 +77,13 @@ def test_the_kit_passes_the_toy_adapter_against_its_fixture(name): def test_the_kit_warns_of_native_records_no_handler_takes(): - (finding,) = check_adapter(Reversible, recorded("coding")).findings + (finding,) = check_adapter(Reversible, recorded("coding")[0]).findings assert (finding.rule, finding.severity, finding.message) == ( "kit-unmatched", "warning", "no handler takes these native records: system (1)") def test_the_kit_fails_an_adapter_that_writes_a_duplicate_tool_call_id(): - result = check_adapter(Duplicating, recorded("coding")) + result = check_adapter(Duplicating, recorded("coding")[0]) (finding,) = result.errors assert (finding.rule, finding.where) == ("kit-run", "task 0, record 4") assert "tool_call_id 'toolu_01' is already in this document" in finding.message @@ -84,13 +94,13 @@ def test_the_kit_fails_an_adapter_that_writes_a_duplicate_tool_call_id(): def test_the_kit_names_where_the_document_differs_from_the_expected_one(): expected = _fixture("coding") expected["steps"][5]["message"] = "something else" - (finding,) = check_adapter(Reversible, recorded("coding"), expected=expected).errors + (finding,) = check_adapter(Reversible, recorded("coding")[0], expected=expected).errors assert (finding.rule, finding.where) == ("kit-expected", "steps[5].message") def test_the_kit_fails_a_from_atif_that_gives_no_native_log_and_warns_when_there_is_none(): - (failed,) = check_adapter(NoLog, recorded("late_result")).errors + (failed,) = check_adapter(NoLog, recorded("late_result")[0]).errors assert (failed.rule, failed.message) == ("kit-from-atif", "returned dict, not a Native holding a list log") - result = check_adapter(Toy, recorded("late_result")) + result = check_adapter(Toy, recorded("late_result")[0]) assert result.errors == [] assert [(f.rule, f.severity) for f in result.findings] == [("kit-from-atif", "warning")] diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py index 793430832..a93b19f79 100644 --- a/packages/agentenv-protocol/tests/test_atif_validator.py +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -156,6 +156,42 @@ def test_totals_sum_the_sub_agents_steps_and_skip_a_count_a_step_lacks(): assert "totals" not in _rules(validate(document)) +def test_a_count_given_as_a_numeric_string_is_read_as_the_models_read_it(): + document = _fixture("coding") + metrics = document["steps"][1]["metrics"] + metrics["prompt_tokens"] = str(metrics["prompt_tokens"]) + assert _rules(validate(document)) == {"harbor-save"} + metrics["cached_tokens"] = str(int(metrics["prompt_tokens"]) + 1) + assert "usage" in _rules(errors(validate(document))) + + +def test_a_sub_agent_embedded_in_a_sub_agent_is_checked_at_its_full_path(): + document = _fixture("subagent") + sub = document["subagent_trajectories"][0] + nested = copy.deepcopy(sub) + nested["trajectory_id"] = "deeper" + for step in nested["steps"]: + for call in step.get("tool_calls") or []: + call["tool_call_id"] += "-deeper" + for result in (step.get("observation") or {}).get("results", []): + result["source_call_id"] += "-deeper" + nested["steps"][0].setdefault("extra", {}).setdefault("agentenv", {})["turn"] = "zero" + sub["subagent_trajectories"] = [nested] + assert ("profile", "subagent_trajectories[0].subagent_trajectories[0].steps[0].extra.agentenv.turn") in { + (f.rule, f.where) for f in errors(validate(document))} + + +def test_a_reference_resolves_to_a_sub_agent_embedded_around_it(): + document = _fixture("subagent") + sub = document["subagent_trajectories"][0] + step = next(s for s in sub["steps"] if s.get("tool_calls")) + result = next(r for r in step["observation"]["results"]) + result["subagent_trajectory_ref"] = [{"trajectory_id": sub["trajectory_id"]}] + assert "reference" not in _rules(validate(document)) + result["subagent_trajectory_ref"] = [{"trajectory_id": "nobody"}] + assert "reference" in _rules(errors(validate(document))) + + def _open_page(name: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, bytes]]: """A case's final document and a page served while its task ran, with two closed steps.""" document, pages, media = build(name) From 017904217d3618a53ab0ea23e6ea5c15361a5bd8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:49:16 -0800 Subject: [PATCH 4/5] fix(protocol): the live check places each sub-agent by its scoped path, not its bare id ATIF makes a trajectory_id unique only within its parent's array, so two sub-agents may each embed a child of the same id. References now resolve to a sub-agent's path, its own embedded children first and then those around it, and the live check walks the spawning chain by path, so each child keeps its own task. The reference rule uses the same resolver. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/atif/validator.py | 46 +++++++++++-------- .../tests/test_atif_validator.py | 36 +++++++++++++++ 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index 8f751f7db..76fe11314 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -320,25 +320,29 @@ def _outside_keys(objects: list[tuple[str, str, Mapping[str, Any]]], found: _Fin f"the profile names no key outside extra.{PROFILE_KEY} here") -def _scopes(document: Mapping[str, Any], prefix: str = "", outer: frozenset = frozenset()) -> dict[str, frozenset]: - """The sub-agent ids a reference in each trajectory may name: those embedded in it or in a trajectory around it, - since the builder embeds every sub-agent in the root while ATIF also lets a sub-agent embed its own.""" - scope = outer | {sub.get("trajectory_id") for sub in document.get("subagent_trajectories") or []} - scopes = {prefix: scope} - for index, sub in enumerate(document.get("subagent_trajectories") or []): - scopes.update(_scopes(sub, f"{prefix}subagent_trajectories[{index}].", scope)) - return scopes +def _resolver(document: Mapping[str, Any], prefix: str = "", + outer: Mapping[Any, str] | None = None) -> dict[str, dict[Any, str]]: + """For each trajectory, by its path prefix, the sub-agent each id a reference in it names: one embedded in it, + else the nearest one embedded around it, by path prefix. ATIF makes an id unique only within its parent's array, + and the builder embeds every sub-agent in the root.""" + subs = list(document.get("subagent_trajectories") or []) + visible = {**(outer or {}), **{sub.get("trajectory_id"): f"{prefix}subagent_trajectories[{index}]." + for index, sub in enumerate(subs)}} + out = {prefix: visible} + for index, sub in enumerate(subs): + out.update(_resolver(sub, f"{prefix}subagent_trajectories[{index}].", visible)) + return out def _calls_and_references(document: Mapping[str, Any], found: _Findings) -> None: - scopes = _scopes(document) + resolve = _resolver(document) seen: dict[str, str] = {} for prefix, trajectory in _trajectories(document): for index, step in enumerate(trajectory.get("steps") or []): - _step_calls(f"{prefix}steps[{index}]", step, scopes[prefix], seen, found) + _step_calls(f"{prefix}steps[{index}]", step, resolve[prefix], seen, found) -def _step_calls(where: str, step: Mapping[str, Any], embedded: frozenset, seen: dict[str, str], +def _step_calls(where: str, step: Mapping[str, Any], embedded: Mapping[Any, str], seen: dict[str, str], found: _Findings) -> None: for n, call in enumerate(step.get("tool_calls") or []): call_id = call.get("tool_call_id") @@ -466,25 +470,27 @@ def _unanswered(page: Mapping[str, Any], found: _Findings) -> None: if task.state is not None: ended_steps.update(range(offset, offset + len(task.steps))) offset += len(task.steps) - spawner: dict[Any, tuple[Any, int]] = {} + resolve = _resolver(page) + spawner: dict[str, tuple[str, int]] = {} for prefix, trajectory in _trajectories(page): - owner = trajectory.get("trajectory_id") if prefix else None for index, step in enumerate(trajectory.get("steps") or []): for result in _results(step): for ref in result.get("subagent_trajectory_ref") or []: - spawner.setdefault(ref.get("trajectory_id"), (owner, index)) + target = resolve[prefix].get(ref.get("trajectory_id")) + if target is not None: + spawner.setdefault(target, (prefix, index)) - def root_step(trajectory_id: Any) -> int | None: + def root_step(prefix: str) -> int | None: seen = set() - while trajectory_id in spawner and trajectory_id not in seen: - seen.add(trajectory_id) - trajectory_id, index = spawner[trajectory_id] - if trajectory_id is None: + while prefix in spawner and prefix not in seen: + seen.add(prefix) + prefix, index = spawner[prefix] + if not prefix: return index return None for prefix, trajectory in _trajectories(page): - root_index = root_step(trajectory.get("trajectory_id")) if prefix else None + root_index = root_step(prefix) if prefix else None if prefix and root_index is None: continue for index, step in enumerate(trajectory.get("steps") or []): diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py index a93b19f79..d6fc936df 100644 --- a/packages/agentenv-protocol/tests/test_atif_validator.py +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -213,6 +213,42 @@ def test_the_live_check_flags_a_page_serving_a_call_with_no_result_before_its_ta assert "live-unanswered" in _rules(errors(validate_live([page], document, media=media))) +def _agent(step_id: int, call: str, *, result: bool = True, spawns: str | None = None, + profile: dict[str, Any] | None = None) -> dict[str, Any]: + step: dict[str, Any] = {"step_id": step_id, "source": "agent", "message": "", + "tool_calls": [{"tool_call_id": call, "function_name": "run", "arguments": {}}]} + if result: + answer: dict[str, Any] = {"source_call_id": call, "content": "done"} + if spawns: + answer["subagent_trajectory_ref"] = [{"trajectory_id": spawns}] + step["observation"] = {"results": [answer]} + if profile: + step["extra"] = {"agentenv": profile} + return step + + +def _sub(trajectory_id: str, steps: list[dict[str, Any]], **more: Any) -> dict[str, Any]: + return {"schema_version": "ATIF-v1.8", "trajectory_id": trajectory_id, "agent": {"name": "a", "version": "1"}, + "steps": steps, **more} + + +def test_children_that_share_an_id_under_different_parents_keep_their_own_tasks(): + user = {"step_id": 1, "source": "user", "message": "go", "extra": {"agentenv": {"task_id": "t0", "turn": 0}}} + later = {"step_id": 3, "source": "user", "message": "again", "extra": {"agentenv": {"task_id": "t1", "turn": 1}}} + page = {"schema_version": "ATIF-v1.8", "session_id": "ctx", "agent": {"name": "a", "version": "1"}, + "steps": [user, _agent(2, "c0", spawns="A", profile={"state": "completed"}), later, + _agent(4, "c1", spawns="B")], + "subagent_trajectories": [ + _sub("A", [_agent(1, "a0", spawns="worker")], + subagent_trajectories=[_sub("worker", [_agent(1, "w0", result=False)])]), + _sub("B", [_agent(1, "b0", spawns="worker")], + subagent_trajectories=[_sub("worker", [_agent(1, "w1", result=False)])]), + ]} + unanswered = [f.where for f in validate_live([page], page) if f.rule == "live-unanswered"] + assert unanswered == ["page 0: subagent_trajectories[1].subagent_trajectories[0].steps[0].tool_calls[0]"] + assert "reference" not in _rules(validate(page)) + + def test_a_page_that_serves_nothing_after_serving_steps_breaks_the_live_rule(): document, page, media = _open_page("coding") assert "live-prefix" in _rules(errors(validate_live([page, None], document, media=media))) From 5ebd6ee6813a8b0624347bbf9da81696b5ebac4f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 16:43:12 -0800 Subject: [PATCH 5/5] perf(protocol): the live check validates a repeated page once, and errors is errors_in validate_live skips a page equal to the one before it: most records close no step, so a recorded session's pages repeat. The adapter kit over a 400-step session drops from 5.6 s to 2.6 s. The helper that filters findings to errors is renamed errors_in, before a release makes its generic name hard to change. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/atif/__init__.py | 21 ++++++++--------- .../src/agentenv_protocol/atif/__main__.py | 4 ++-- .../src/agentenv_protocol/atif/kit.py | 4 ++-- .../src/agentenv_protocol/atif/validator.py | 6 +++-- .../tests/test_atif_validator.py | 23 +++++++++++-------- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py index 1d24bd9e5..c8c2fb340 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py @@ -47,17 +47,16 @@ from .kit import KIT_RULES, KitResult, check_adapter from .reader import Task, check, is_prefix, load, media_paths, strip_profile, tasks from .runner import AdapterRun, Fed, RecordedTask, replay -from .validator import RULES, Finding, Rule, errors, validate, validate_live +from .validator import RULES, Finding, Rule, errors_in, validate, validate_live __all__ = [ - "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "AtifError", "AtifTurn", - "AudioSource", "ContentPart", "Document", "FORMAT", "Fed", "FinalMetrics", "FinalMetricsProfile", "ImageSource", - "Finding", "KIT_RULES", "KitResult", "Media", "MediaInfo", "Metrics", "MetricsProfile", "Native", "Observation", - "ObservationResult", "RULES", "Rule", - "ObservationResultProfile", "PROFILES", "PROFILE_KEY", "PROFILE_VERSION", "RAW_ARGUMENTS", "ReasoningBlock", - "RecordedTask", "SCHEMA_VERSION", "Selector", "Step", "StepError", "StepProfile", "SubagentTrajectoryRef", - "SubagentTrajectoryRefProfile", "Task", "ToolCall", "ToolCallProfile", "Trajectory", "TrajectoryProfile", - "TurnContext", "VOCABULARIES", "atif_adapter", "aux", "check", "check_adapter", "dump_profile", "errors", "finish", - "is_prefix", "load", "media_paths", "native", "profile_of", "read_profile", "replay", "strip_profile", "tasks", - "timestamp", "validate", "validate_live", + "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "atif_adapter", + "AtifError", "AtifTurn", "AudioSource", "aux", "check", "check_adapter", "ContentPart", "Document", "dump_profile", + "errors_in", "Fed", "FinalMetrics", "FinalMetricsProfile", "Finding", "finish", "FORMAT", "ImageSource", + "is_prefix", "KIT_RULES", "KitResult", "load", "Media", "media_paths", "MediaInfo", "Metrics", "MetricsProfile", + "Native", "native", "Observation", "ObservationResult", "ObservationResultProfile", "PROFILE_KEY", "profile_of", + "PROFILE_VERSION", "PROFILES", "RAW_ARGUMENTS", "read_profile", "ReasoningBlock", "RecordedTask", "replay", "Rule", + "RULES", "SCHEMA_VERSION", "Selector", "Step", "StepError", "StepProfile", "strip_profile", "SubagentTrajectoryRef", + "SubagentTrajectoryRefProfile", "Task", "tasks", "timestamp", "ToolCall", "ToolCallProfile", "Trajectory", + "TrajectoryProfile", "TurnContext", "validate", "validate_live", "VOCABULARIES", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py index f22e5b65e..651bedc13 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Sequence -from .validator import errors, parse, validate +from .validator import errors_in, parse, validate def main(argv: Sequence[str] | None = None) -> int: @@ -31,7 +31,7 @@ def main(argv: Sequence[str] | None = None) -> int: findings = validate(document, reader=args.reader, media=args.media) for finding in findings: print(finding) - failed = errors(findings) + failed = errors_in(findings) print(f"{args.file}: {'invalid' if failed else 'valid'}, {len(failed)} errors, " f"{len(findings) - len(failed)} warnings") return 1 if failed else 0 diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py index 874ea5f2c..3bf01aa31 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py @@ -12,7 +12,7 @@ from .profile import profile_of from .reader import strip_profile from .runner import AdapterRun, RecordedTask -from .validator import ERROR, WARNING, Finding, Rule, _difference, errors, validate_live +from .validator import ERROR, WARNING, Finding, Rule, _difference, errors_in, validate_live KIT_RULES: dict[str, Rule] = {rule.id: rule for rule in ( Rule("kit-run", ERROR, None, "The adapter takes every recorded record without raising"), @@ -33,7 +33,7 @@ class KitResult: @property def errors(self) -> list[Finding]: - return errors(self.findings) + return errors_in(self.findings) def raise_for_errors(self) -> None: """Raise AssertionError listing every finding when any is an error.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index 76fe11314..2a18737c5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -114,7 +114,7 @@ def add(self, rule: str, where: str, message: str) -> None: self.items.append(Finding(rule, severity, f"{self.prefix}{where}", message)) -def errors(findings: Sequence[Finding]) -> list[Finding]: +def errors_in(findings: Sequence[Finding]) -> list[Finding]: """The findings that fail a check.""" return [finding for finding in findings if finding.severity == ERROR] @@ -179,10 +179,12 @@ def validate_live(pages: Sequence[Mapping[str, Any] | None], final: Mapping[str, media: Path | Mapping[str, bytes] | None = None) -> list[Finding]: """The live rule, for a producer: each page a reader was served, in order, validates, is a prefix of the next and of the final read, and serves no step whose call is unanswered before its task ends. The final read is checked - too.""" + too. A page equal to the one before it is checked once, since most records close no step.""" findings: list[Finding] = [] served: tuple[int, Mapping[str, Any]] | None = None for index, page in enumerate(pages): + if served is not None and page == served[1]: + continue if page is None: if served is not None: findings.append(Finding("live-prefix", ERROR, f"page {index}: ", diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py index d6fc936df..5f61970bb 100644 --- a/packages/agentenv-protocol/tests/test_atif_validator.py +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -9,7 +9,7 @@ import pytest -from agentenv_protocol.atif import KIT_RULES, RULES, errors, validate, validate_live +from agentenv_protocol.atif import KIT_RULES, RULES, errors_in, validate, validate_live from agentenv_protocol.atif.__main__ import main from atif_cases import CASES, FIXTURES, build @@ -110,7 +110,7 @@ def test_every_rule_has_a_negative_fixture(): @pytest.mark.parametrize("rule", sorted(NEGATIVE)) def test_a_negative_fixture_fails_with_its_rules_id_and_the_cli_reports_it(rule, tmp_path, capsys): document = _negative(rule) - assert rule in _rules(errors(validate(document))) + assert rule in _rules(errors_in(validate(document))) path = tmp_path / "trajectory.json" path.write_text(json.dumps(document), encoding="utf-8") assert main(["validate", str(path)]) == 1 @@ -138,8 +138,8 @@ def test_media_bytes_are_checked_against_their_name_and_profile(): document = _fixture("computer_use") path = _parts(document)[0]["source"]["path"] assert validate(document, media=FIXTURES) == [] - assert "media-bytes" in _rules(errors(validate(document, media={}))) - assert "media-bytes" in _rules(errors(validate(document, media={path: b"not the screenshot"}))) + assert "media-bytes" in _rules(errors_in(validate(document, media={}))) + assert "media-bytes" in _rules(errors_in(validate(document, media={path: b"not the screenshot"}))) def test_totals_sum_the_sub_agents_steps_and_skip_a_count_a_step_lacks(): @@ -162,7 +162,7 @@ def test_a_count_given_as_a_numeric_string_is_read_as_the_models_read_it(): metrics["prompt_tokens"] = str(metrics["prompt_tokens"]) assert _rules(validate(document)) == {"harbor-save"} metrics["cached_tokens"] = str(int(metrics["prompt_tokens"]) + 1) - assert "usage" in _rules(errors(validate(document))) + assert "usage" in _rules(errors_in(validate(document))) def test_a_sub_agent_embedded_in_a_sub_agent_is_checked_at_its_full_path(): @@ -178,7 +178,7 @@ def test_a_sub_agent_embedded_in_a_sub_agent_is_checked_at_its_full_path(): nested["steps"][0].setdefault("extra", {}).setdefault("agentenv", {})["turn"] = "zero" sub["subagent_trajectories"] = [nested] assert ("profile", "subagent_trajectories[0].subagent_trajectories[0].steps[0].extra.agentenv.turn") in { - (f.rule, f.where) for f in errors(validate(document))} + (f.rule, f.where) for f in errors_in(validate(document))} def test_a_reference_resolves_to_a_sub_agent_embedded_around_it(): @@ -189,7 +189,7 @@ def test_a_reference_resolves_to_a_sub_agent_embedded_around_it(): result["subagent_trajectory_ref"] = [{"trajectory_id": sub["trajectory_id"]}] assert "reference" not in _rules(validate(document)) result["subagent_trajectory_ref"] = [{"trajectory_id": "nobody"}] - assert "reference" in _rules(errors(validate(document))) + assert "reference" in _rules(errors_in(validate(document))) def _open_page(name: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, bytes]]: @@ -203,14 +203,17 @@ def test_the_live_check_flags_a_page_that_differs_from_the_final_read_by_one_ste document, page, media = _open_page("coding") assert validate_live([page], document, media=media) == [] page["steps"][1]["message"] = "a step served, then changed" - assert [(f.rule, f.where) for f in errors(validate_live([page], document, media=media))] == [ + assert [(f.rule, f.where) for f in errors_in(validate_live([page], document, media=media))] == [ ("live-prefix", "final: ")] def test_the_live_check_flags_a_page_serving_a_call_with_no_result_before_its_task_ends(): document, page, media = _open_page("coding") del page["steps"][1]["observation"] - assert "live-unanswered" in _rules(errors(validate_live([page], document, media=media))) + assert "live-unanswered" in _rules(errors_in(validate_live([page], document, media=media))) + repeated = [f.where for f in validate_live([page, copy.deepcopy(page)], document, media=media) + if f.rule == "live-unanswered"] + assert repeated == ["page 0: steps[1].tool_calls[0]"] def _agent(step_id: int, call: str, *, result: bool = True, spawns: str | None = None, @@ -251,7 +254,7 @@ def test_children_that_share_an_id_under_different_parents_keep_their_own_tasks( def test_a_page_that_serves_nothing_after_serving_steps_breaks_the_live_rule(): document, page, media = _open_page("coding") - assert "live-prefix" in _rules(errors(validate_live([page, None], document, media=media))) + assert "live-prefix" in _rules(errors_in(validate_live([page, None], document, media=media))) def test_python_m_runs_the_cli(tmp_path):