diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md index 717283ba5..35cd851cf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md @@ -146,6 +146,49 @@ field added in a later minor names the version it arrived in. Proposed for a lat `input_includes_cache` on the root and `cost_status` (`reported`, `priced` or `unpriced`), with the usage cross-check that reads them; per-part origins, when a reader needs them; and an always-on harness's fields. +## Validator + +`validate(document)` returns findings, each with a rule id below, a severity and where in the document it is. It holds a +producer's output to profile 1.0. `reader=True` checks someone else's document for reading instead: what a producer +must not write only warns, and the document fails only where agentenv couldn't read it. `media` is the folder the +document's media paths are relative to, the one holding `media/`, or the files by path; without it only the paths' form +is checked. `validate_live(pages, final)` checks what a live reader was served, page by page, with the final read. A sub-agent's task is its spawning call's, found through the references, so an embedded sub-agent that no served step references isn't held to `live-unanswered`. On +the command line: `python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]`, which prints each finding +and exits 1 when one is an error. + +| Rule | Holds | Producer | Reader | +| --- | --- | --- | --- | +| `json` | The document is a JSON object: UTF-8, with no NaN or infinity | error | error | +| `atif` | ATIF's own models, vendored from Harbor, accept it | error | error | +| `profile` | Each `extra.agentenv` parses with its profile model | error | error | +| `profile-key` | Each `extra.agentenv` holds only the fields profile 1.0 defines | error | warning | +| `profile-version` | The root's `extra.agentenv.version`, when given, is `1.0` | error | warning | +| `vocabulary` | A vocabulary field holds one of its values | error | warning | +| `outside-key` | No `extra` holds a key outside `agentenv` but `ToolCall.extra.raw_arguments` | error | warning | +| `call-id` | Each `tool_call_id` is unique across the document and its sub-agents | error | warning | +| `reference` | Each `subagent_trajectory_ref` names an embedded sub-agent by `trajectory_id` | error | warning | +| `totals` | A `final_metrics` total equals the sum over the model steps of the root and its embedded sub-agents, when every one has the count | error | warning | +| `total-steps` | `final_metrics.total_steps` counts the root's steps, or the root's `notes` explain the difference | error | warning | +| `usage` | A step's cache reads and writes fit within its `prompt_tokens` | error | warning | +| `media-path` | A media path is `media/.` | error | warning | +| `media-bytes` | Given the media, each file the document names exists, with the sha256 its name and profile give and the profile's size | error | error | +| `timestamp` | A timestamp is UTC in the builder's form: `YYYY-MM-DDTHH:MM:SSZ`, with no, 3 or 6 fractional digits | error | not checked | +| `harbor-save` | Harbor's `to_json_dict` gives the document back unchanged | error | warning | +| `text-message` | A step message whose parts are all text is one string | error | warning | +| `live-prefix` | Every live page is a prefix of the next page and of the final read | error | not checked | +| `live-unanswered` | No live page serves a step with an unanswered call before its task ends | error | not checked | + +The adapter kit, `check_adapter(adapter, recorded, expected=, from_atif=)`, is for an agent's CI. It runs an adapter +over a context's recorded tasks as the SDK runs it live, applies the live rule after every record and the validator to +the final document, and adds its own rules. `kit-from-atif` only warns when the adapter has no `from_atif`. + +| Rule | Holds | Severity | +| --- | --- | --- | +| `kit-run` | The adapter takes every recorded record without raising | error | +| `kit-expected` | The document equals the checked-in expected document | error | +| `kit-unmatched` | Every native record has a handler; one that none takes is only counted | warning | +| `kit-from-atif` | `from_atif` turns a document without the profile into a native log | error | + ## Fixtures `fixtures/` holds the profile's canonical examples, built through the adapter framework from a toy format's records: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py index 78e3c8a36..c8c2fb340 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py @@ -44,16 +44,19 @@ profile_of, read_profile, ) +from .kit import KIT_RULES, KitResult, check_adapter from .reader import Task, check, is_prefix, load, media_paths, strip_profile, tasks from .runner import AdapterRun, Fed, RecordedTask, replay +from .validator import RULES, Finding, Rule, errors_in, validate, validate_live __all__ = [ - "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "AtifError", "AtifTurn", - "AudioSource", "ContentPart", "Document", "FORMAT", "Fed", "FinalMetrics", "FinalMetricsProfile", "ImageSource", - "Media", "MediaInfo", "Metrics", "MetricsProfile", "Native", "Observation", "ObservationResult", - "ObservationResultProfile", "PROFILES", "PROFILE_KEY", "PROFILE_VERSION", "RAW_ARGUMENTS", "ReasoningBlock", - "RecordedTask", "SCHEMA_VERSION", "Selector", "Step", "StepError", "StepProfile", "SubagentTrajectoryRef", - "SubagentTrajectoryRefProfile", "Task", "ToolCall", "ToolCallProfile", "Trajectory", "TrajectoryProfile", - "TurnContext", "VOCABULARIES", "atif_adapter", "aux", "check", "dump_profile", "finish", "is_prefix", "load", - "media_paths", "native", "profile_of", "read_profile", "replay", "strip_profile", "tasks", "timestamp", + "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "atif_adapter", + "AtifError", "AtifTurn", "AudioSource", "aux", "check", "check_adapter", "ContentPart", "Document", "dump_profile", + "errors_in", "Fed", "FinalMetrics", "FinalMetricsProfile", "Finding", "finish", "FORMAT", "ImageSource", + "is_prefix", "KIT_RULES", "KitResult", "load", "Media", "media_paths", "MediaInfo", "Metrics", "MetricsProfile", + "Native", "native", "Observation", "ObservationResult", "ObservationResultProfile", "PROFILE_KEY", "profile_of", + "PROFILE_VERSION", "PROFILES", "RAW_ARGUMENTS", "read_profile", "ReasoningBlock", "RecordedTask", "replay", "Rule", + "RULES", "SCHEMA_VERSION", "Selector", "Step", "StepError", "StepProfile", "strip_profile", "SubagentTrajectoryRef", + "SubagentTrajectoryRefProfile", "Task", "tasks", "timestamp", "ToolCall", "ToolCallProfile", "Trajectory", + "TrajectoryProfile", "TurnContext", "validate", "validate_live", "VOCABULARIES", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py new file mode 100644 index 000000000..651bedc13 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py @@ -0,0 +1,41 @@ +"""``python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]``: check an ATIF document, printing each +finding; exit 1 when any is an error, 2 when the file can't be read.""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Sequence + +from .validator import errors_in, parse, validate + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.atif", + description="Check ATIF documents against ATIF and the agentenv profile.") + commands = parser.add_subparsers(dest="command", required=True) + check = commands.add_parser("validate", help="check one document; exit 1 when it has an error") + check.add_argument("file", type=Path, help="the trajectory.json to check") + check.add_argument("--media", type=Path, metavar="DIR", + help="the folder the document's media paths are relative to, which holds media/; " + "given, each file is checked against its name and profile") + check.add_argument("--reader", action="store_true", + help="check someone else's document for reading: what a producer must not write warns") + args = parser.parse_args(argv) + try: + document, findings = parse(args.file) + except OSError as exc: + print(f"can't read {args.file}: {exc.strerror or exc}", file=sys.stderr) + return 2 + if not findings: + findings = validate(document, reader=args.reader, media=args.media) + for finding in findings: + print(finding) + failed = errors_in(findings) + print(f"{args.file}: {'invalid' if failed else 'valid'}, {len(failed)} errors, " + f"{len(findings) - len(failed)} warnings") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py new file mode 100644 index 000000000..3bf01aa31 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py @@ -0,0 +1,106 @@ +"""The adapter kit, for an agent's CI: run an adapter over recorded tasks, as the SDK runs it live, and check what it +writes. The live rule holds after every record, the final document passes the validator as a producer's, it equals +the checked-in document when one is given, and ``from_atif`` turns ATIF without the profile back into a native log.""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .adapter import Adapter, Native +from .profile import profile_of +from .reader import strip_profile +from .runner import AdapterRun, RecordedTask +from .validator import ERROR, WARNING, Finding, Rule, _difference, errors_in, validate_live + +KIT_RULES: dict[str, Rule] = {rule.id: rule for rule in ( + Rule("kit-run", ERROR, None, "The adapter takes every recorded record without raising"), + Rule("kit-expected", ERROR, None, "The document equals the checked-in expected document"), + Rule("kit-unmatched", WARNING, None, "Every native record has a handler; one that none takes is only counted"), + Rule("kit-from-atif", ERROR, None, "`from_atif` turns a document without the profile into a native log"), +)} + + +@dataclass +class KitResult: + """What the kit found, with the document and every live page the adapter's run served.""" + + document: dict[str, Any] | None + pages: list[dict[str, Any] | None] = field(default_factory=list) + media: dict[str, bytes] = field(default_factory=dict) + findings: list[Finding] = field(default_factory=list) + + @property + def errors(self) -> list[Finding]: + return errors_in(self.findings) + + def raise_for_errors(self) -> None: + """Raise AssertionError listing every finding when any is an error.""" + if self.errors: + raise AssertionError("the adapter kit found:\n" + "\n".join(str(f) for f in self.findings)) + + +def check_adapter(adapter: type[Adapter], recorded: Sequence[RecordedTask], *, + expected: Mapping[str, Any] | Path | None = None, + from_atif: Sequence[Mapping[str, Any]] | None = None, + media: Mapping[str, bytes] | None = None) -> KitResult: + """Run ``adapter`` over ``recorded``, one A2A context's tasks in order, and check it. ``expected`` is the document + the run should write, or the file holding it. ``from_atif`` are the documents its ``from_atif`` is tried on; by + default, the run's own document with the profile stripped. ``media`` holds, by path, the files of any Media in the + recorded tasks' inbound or final content; the adapter's own media come from the run.""" + run = AdapterRun(adapter) + result = KitResult(document=None) + where = "" + try: + for t, task in enumerate(recorded): + where = f"task {t}, its start" + run.start_task(task.context) + result.pages.append(run.page()) + for r, fed in enumerate(task.records): + where = f"task {t}, record {r}" + (f" of aux {fed.aux!r}" if fed.aux is not None else "") + if fed.aux is None: + run.native(fed.record) + else: + run.aux(fed.aux, fed.record) + result.pages.append(run.page()) + where = f"task {t}, its end" + run.end_task(state=task.state, final=task.final, ended_at=task.ended_at) + result.pages.append(run.page()) + except Exception as exc: + result.findings.append(Finding("kit-run", ERROR, where, f"{type(exc).__name__}: {exc}")) + return result + result.document = run.document() + result.media = {**(media or {}), **run.media_files} + if result.document is None: + result.findings.append(Finding("kit-run", ERROR, "", "the run wrote no step")) + return result + result.findings.extend(validate_live(result.pages, result.document, media=result.media)) + unmatched = profile_of(result.document).get("unmatched") + if unmatched: + counts = ", ".join(f"{kind} ({count})" for kind, count in sorted(unmatched.items())) + result.findings.append(Finding("kit-unmatched", WARNING, "extra.agentenv.unmatched", + f"no handler takes these native records: {counts}")) + if expected is not None: + want = json.loads(expected.read_text(encoding="utf-8")) if isinstance(expected, Path) else expected + if result.document != want: + result.findings.append(Finding("kit-expected", ERROR, _difference(want, result.document), + "the document differs from the expected one here")) + for index, document in enumerate(from_atif if from_atif is not None else [strip_profile(result.document)]): + _from_atif(adapter, document, index, result) + return result + + +def _from_atif(adapter: type[Adapter], document: Mapping[str, Any], index: int, result: KitResult) -> None: + where = f"from_atif document {index}" + try: + native = adapter.from_atif(document) + except NotImplementedError as exc: + result.findings.append(Finding("kit-from-atif", WARNING, where, str(exc))) + return + except Exception as exc: + result.findings.append(Finding("kit-from-atif", ERROR, where, f"raised {type(exc).__name__}: {exc}")) + return + if not isinstance(native, Native) or not isinstance(native.log, list): + result.findings.append(Finding("kit-from-atif", ERROR, where, + f"returned {type(native).__name__}, not a Native holding a list log")) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py new file mode 100644 index 000000000..2a18737c5 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -0,0 +1,529 @@ +"""Checking an ATIF document against ATIF and the agentenv profile: the vendored models, the profile's models, and the +rules neither checks, each a finding with a rule id PROFILE.md names. A producer is held to profile 1.0; a reader of +someone else's document is warned where a producer would fail, and fails only where agentenv couldn't read it.""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from .models import Trajectory +from .profile import ( + PROFILE_KEY, + PROFILE_VERSION, + PROFILES, + RAW_ARGUMENTS, + VOCABULARIES, + AdapterRef, + Arrival, + MediaInfo, + ReasoningBlock, + StepError, + profile_of, +) +from .reader import is_prefix, tasks + +ERROR = "error" +WARNING = "warning" + +MEDIA_PATH = re.compile(r"^media/(?P[0-9a-f]{64})\.[A-Za-z0-9]+$") +TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3}|\.\d{6})?Z$") +_COUNTS = (("prompt_tokens", "total_prompt_tokens"), ("completion_tokens", "total_completion_tokens"), + ("cached_tokens", "total_cached_tokens")) +_INT = TypeAdapter(int) +#: A profile field that holds a nested profile object, by its owner and name. +_NESTED: dict[tuple[str, str], type[BaseModel]] = { + ("Trajectory", "adapter"): AdapterRef, ("Step", "error"): StepError, + ("ObservationResult", "arrived"): Arrival, +} +_LISTS: dict[tuple[str, str], type[BaseModel]] = {("Step", "reasoning"): ReasoningBlock} +_MAPS: dict[tuple[str, str], type[BaseModel]] = { + ("Step", "media"): MediaInfo, ("ObservationResult", "media"): MediaInfo, +} +_VOCABULARY_AT = { + ("Step", "state"): "StepProfile.state", ("ObservationResult", "status"): "ObservationResultProfile.status", + ("Metrics", "cost_source"): "MetricsProfile.cost_source", + ("FinalMetrics", "cost_source"): "FinalMetricsProfile.cost_source", +} + + +@dataclass(frozen=True) +class Rule: + """A check: its id, its severity for a producer and for a reader (None: not checked there), and what it holds.""" + + id: str + producer: str + reader: str | None + holds: str + + +RULES: dict[str, Rule] = {rule.id: rule for rule in ( + Rule("json", ERROR, ERROR, "The document is a JSON object: UTF-8, with no NaN or infinity"), + Rule("atif", ERROR, ERROR, "ATIF's own models, vendored from Harbor, accept it"), + Rule("profile", ERROR, ERROR, "Each `extra.agentenv` parses with its profile model"), + Rule("profile-key", ERROR, WARNING, "Each `extra.agentenv` holds only the fields profile 1.0 defines"), + Rule("profile-version", ERROR, WARNING, "The root's `extra.agentenv.version`, when given, is `1.0`"), + Rule("vocabulary", ERROR, WARNING, "A vocabulary field holds one of its values"), + Rule("outside-key", ERROR, WARNING, "No `extra` holds a key outside `agentenv` but `ToolCall.extra.raw_arguments`"), + Rule("call-id", ERROR, WARNING, "Each `tool_call_id` is unique across the document and its sub-agents"), + Rule("reference", ERROR, WARNING, "Each `subagent_trajectory_ref` names an embedded sub-agent by `trajectory_id`"), + Rule("totals", ERROR, WARNING, "A `final_metrics` total equals the sum over the model steps of the root and its " + "embedded sub-agents, when every one has the count"), + Rule("total-steps", ERROR, WARNING, "`final_metrics.total_steps` counts the root's steps, or the root's `notes` " + "explain the difference"), + Rule("usage", ERROR, WARNING, "A step's cache reads and writes fit within its `prompt_tokens`"), + Rule("media-path", ERROR, WARNING, "A media path is `media/.`"), + Rule("media-bytes", ERROR, ERROR, "Given the media, each file the document names exists, with the sha256 its " + "name and profile give and the profile's size"), + Rule("timestamp", ERROR, None, "A timestamp is UTC in the builder's form: `YYYY-MM-DDTHH:MM:SSZ`, with no, 3 or " + "6 fractional digits"), + Rule("harbor-save", ERROR, WARNING, "Harbor's `to_json_dict` gives the document back unchanged"), + Rule("text-message", ERROR, WARNING, "A step message whose parts are all text is one string"), + Rule("live-prefix", ERROR, None, "Every live page is a prefix of the next page and of the final read"), + Rule("live-unanswered", ERROR, None, "No live page serves a step with an unanswered call before its task ends"), +)} + + +@dataclass(frozen=True) +class Finding: + """What a rule found, and where: a path into the document, such as ``steps[3].tool_calls[0]``.""" + + rule: str + severity: str + where: str + message: str + + def __str__(self) -> str: + return f"{self.severity} {self.rule} {self.where or 'document'}: {self.message}" + + +class _Findings: + def __init__(self, reader: bool, prefix: str = "") -> None: + self.reader = reader + self.prefix = prefix + self.items: list[Finding] = [] + + def add(self, rule: str, where: str, message: str) -> None: + severity = RULES[rule].reader if self.reader else RULES[rule].producer + if severity is not None: + self.items.append(Finding(rule, severity, f"{self.prefix}{where}", message)) + + +def errors_in(findings: Sequence[Finding]) -> list[Finding]: + """The findings that fail a check.""" + return [finding for finding in findings if finding.severity == ERROR] + + +def parse(source: str | bytes | Path) -> tuple[Any, list[Finding]]: + """JSON text or a file read as the ``json`` rule reads it: the value, or None with the finding.""" + def no_constant(name: str) -> Any: + raise ValueError(f"{name} isn't JSON") + try: + if isinstance(source, Path): + source = source.read_bytes() + text = source.decode("utf-8") if isinstance(source, (bytes, bytearray)) else source + return json.loads(text, parse_constant=no_constant), [] + except (UnicodeDecodeError, ValueError) as exc: + return None, [Finding("json", ERROR, "", str(exc))] + + +def validate(document: Any, *, reader: bool = False, + media: Path | Mapping[str, bytes] | None = None) -> list[Finding]: + """Every finding for ``document``. ``reader`` checks someone else's document for reading rather than a producer's + output. ``media`` is the folder the document's media paths are relative to, or the files by path; without it, only + the paths' form is checked.""" + found = _Findings(reader) + _check(document, found, media) + return found.items + + +def _check(document: Any, found: _Findings, media: Path | Mapping[str, bytes] | None) -> None: + if not isinstance(document, Mapping): + found.add("json", "", "a document is a JSON object") + return + try: + json.dumps(document, allow_nan=False) + except (TypeError, ValueError) as exc: + found.add("json", "", str(exc)) + return + try: + parsed = Trajectory.model_validate(document) + except ValidationError as exc: + for error in exc.errors(include_url=False): + found.add("atif", _loc(error["loc"]), error["msg"]) + return + objects = list(_objects(document)) + _profiles(objects, found) + _outside_keys(objects, found) + _calls_and_references(document, found) + _totals(document, found) + _usage(document, found) + _media(document, found, media) + _timestamps(document, found) + saved = parsed.to_json_dict() + if saved != document: + found.add("harbor-save", _difference(document, saved), "Harbor's to_json_dict gives this back changed") + for where, step in _steps(document): + message = step.get("message") + if isinstance(message, list) and message and all( + isinstance(p, Mapping) and p.get("type") == "text" for p in message): + found.add("text-message", f"{where}.message", "every part is text, so the message is one string") + + +def validate_live(pages: Sequence[Mapping[str, Any] | None], final: Mapping[str, Any], *, + media: Path | Mapping[str, bytes] | None = None) -> list[Finding]: + """The live rule, for a producer: each page a reader was served, in order, validates, is a prefix of the next and + of the final read, and serves no step whose call is unanswered before its task ends. The final read is checked + too. A page equal to the one before it is checked once, since most records close no step.""" + findings: list[Finding] = [] + served: tuple[int, Mapping[str, Any]] | None = None + for index, page in enumerate(pages): + if served is not None and page == served[1]: + continue + if page is None: + if served is not None: + findings.append(Finding("live-prefix", ERROR, f"page {index}: ", + f"serves nothing after page {served[0]} served steps")) + continue + found = _Findings(False, f"page {index}: ") + _check(page, found, media) + _unanswered(page, found) + findings.extend(found.items) + if served is not None and not is_prefix(served[1], page): + findings.append(Finding("live-prefix", ERROR, f"page {index}: ", + f"page {served[0]} isn't a prefix of it; they first differ at " + f"{_difference(served[1], page) or 'the document'}")) + served = (index, page) + found = _Findings(False, "final: ") + _check(final, found, media) + findings.extend(found.items) + if served is not None and not is_prefix(served[1], final): + findings.append(Finding("live-prefix", ERROR, "final: ", + f"page {served[0]} isn't a prefix of it; they first differ at " + f"{_difference(served[1], final) or 'the document'}")) + return findings + + +# What the document holds + +def _trajectories(document: Mapping[str, Any], prefix: str = "") -> Iterator[tuple[str, Mapping[str, Any]]]: + """The document and every sub-agent embedded in it, at any depth, with the path prefix of each.""" + yield prefix, document + for index, sub in enumerate(document.get("subagent_trajectories") or []): + yield from _trajectories(sub, f"{prefix}subagent_trajectories[{index}].") + + +def _count(value: Any) -> int | None: + """A token count as ATIF's models read it, a numeric string included, or None when there is none.""" + try: + return None if value is None else _INT.validate_python(value) + except ValidationError: + return None + + +def _steps(document: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: + for prefix, trajectory in _trajectories(document): + for index, step in enumerate(trajectory.get("steps") or []): + yield f"{prefix}steps[{index}]", step + + +def _results(step: Mapping[str, Any]) -> list[Mapping[str, Any]]: + return list((step.get("observation") or {}).get("results") or []) + + +def _objects(document: Mapping[str, Any]) -> Iterator[tuple[str, str, Mapping[str, Any]]]: + """Every ATIF object that may have an ``extra``: its kind, as PROFILES names it, its path and the object.""" + for prefix, trajectory in _trajectories(document): + yield "Trajectory", prefix.rstrip(".") or "", trajectory + yield "Agent", f"{prefix}agent", trajectory.get("agent") or {} + if trajectory.get("final_metrics"): + yield "FinalMetrics", f"{prefix}final_metrics", trajectory["final_metrics"] + for index, step in enumerate(trajectory.get("steps") or []): + where = f"{prefix}steps[{index}]" + yield "Step", where, step + if step.get("metrics"): + yield "Metrics", f"{where}.metrics", step["metrics"] + for n, call in enumerate(step.get("tool_calls") or []): + yield "ToolCall", f"{where}.tool_calls[{n}]", call + for n, result in enumerate(_results(step)): + yield "ObservationResult", f"{where}.observation.results[{n}]", result + for m, ref in enumerate(result.get("subagent_trajectory_ref") or []): + yield "SubagentTrajectoryRef", f"{where}.observation.results[{n}].subagent_trajectory_ref[{m}]", ref + + +def _profiles(objects: list[tuple[str, str, Mapping[str, Any]]], found: _Findings) -> None: + for kind, where, obj in objects: + extra = obj.get("extra") + if not isinstance(extra, Mapping) or PROFILE_KEY not in extra: + continue + at = f"{where}.extra.{PROFILE_KEY}".lstrip(".") + profile = extra[PROFILE_KEY] + if not isinstance(profile, Mapping): + found.add("profile", at, "the profile is an object") + continue + try: + PROFILES[kind].model_validate(profile) + except ValidationError as exc: + for error in exc.errors(include_url=False): + found.add("profile", f"{at}.{_loc(error['loc'])}", error["msg"]) + continue + _known_keys(kind, at, profile, found) + for (owner, name), vocabulary in _VOCABULARY_AT.items(): + if owner == kind and profile.get(name) is not None: + _vocabulary(vocabulary, f"{at}.{name}", profile[name], found) + error = profile.get("error") if kind == "Step" else None + if isinstance(error, Mapping) and error.get("kind") is not None: + _vocabulary("StepError.kind", f"{at}.error.kind", error["kind"], found) + for n, block in enumerate(profile.get("reasoning") or [] if kind == "Step" else []): + if isinstance(block, Mapping) and block.get("visibility") is not None: + _vocabulary("ReasoningBlock.visibility", f"{at}.reasoning[{n}].visibility", block["visibility"], found) + if kind == "Trajectory" and where == "" and profile.get("version") not in (None, PROFILE_VERSION): + found.add("profile-version", f"{at}.version", + f"this is profile {profile['version']!r}; the validator checks profile {PROFILE_VERSION}") + + +def _known_keys(kind: str, at: str, profile: Mapping[str, Any], found: _Findings) -> None: + def unknown(model: type[BaseModel], value: Mapping[str, Any], path: str) -> None: + for key in value: + if key not in model.model_fields: + found.add("profile-key", f"{path}.{key}", f"profile {PROFILE_VERSION} defines no {key!r} here") + unknown(PROFILES[kind], profile, at) + for (owner, name), model in _NESTED.items(): + if owner == kind and isinstance(profile.get(name), Mapping): + unknown(model, profile[name], f"{at}.{name}") + for (owner, name), model in _LISTS.items(): + if owner == kind: + for n, item in enumerate(profile.get(name) or []): + if isinstance(item, Mapping): + unknown(model, item, f"{at}.{name}[{n}]") + for (owner, name), model in _MAPS.items(): + if owner == kind and isinstance(profile.get(name), Mapping): + for key, item in profile[name].items(): + if isinstance(item, Mapping): + unknown(model, item, f"{at}.{name}[{key}]") + + +def _vocabulary(name: str, where: str, value: Any, found: _Findings) -> None: + if value not in VOCABULARIES[name]: + found.add("vocabulary", where, f"{value!r} isn't one of {', '.join(VOCABULARIES[name])}") + + +def _outside_keys(objects: list[tuple[str, str, Mapping[str, Any]]], found: _Findings) -> None: + for kind, where, obj in objects: + extra = obj.get("extra") + for key in extra if isinstance(extra, Mapping) else (): + if key != PROFILE_KEY and not (kind == "ToolCall" and key == RAW_ARGUMENTS): + found.add("outside-key", f"{where}.extra.{key}".lstrip("."), + f"the profile names no key outside extra.{PROFILE_KEY} here") + + +def _resolver(document: Mapping[str, Any], prefix: str = "", + outer: Mapping[Any, str] | None = None) -> dict[str, dict[Any, str]]: + """For each trajectory, by its path prefix, the sub-agent each id a reference in it names: one embedded in it, + else the nearest one embedded around it, by path prefix. ATIF makes an id unique only within its parent's array, + and the builder embeds every sub-agent in the root.""" + subs = list(document.get("subagent_trajectories") or []) + visible = {**(outer or {}), **{sub.get("trajectory_id"): f"{prefix}subagent_trajectories[{index}]." + for index, sub in enumerate(subs)}} + out = {prefix: visible} + for index, sub in enumerate(subs): + out.update(_resolver(sub, f"{prefix}subagent_trajectories[{index}].", visible)) + return out + + +def _calls_and_references(document: Mapping[str, Any], found: _Findings) -> None: + resolve = _resolver(document) + seen: dict[str, str] = {} + for prefix, trajectory in _trajectories(document): + for index, step in enumerate(trajectory.get("steps") or []): + _step_calls(f"{prefix}steps[{index}]", step, resolve[prefix], seen, found) + + +def _step_calls(where: str, step: Mapping[str, Any], embedded: Mapping[Any, str], seen: dict[str, str], + found: _Findings) -> None: + for n, call in enumerate(step.get("tool_calls") or []): + call_id = call.get("tool_call_id") + if call_id in seen: + found.add("call-id", f"{where}.tool_calls[{n}]", f"{call_id!r} is also the id at {seen[call_id]}") + else: + seen[call_id] = f"{where}.tool_calls[{n}]" + for n, result in enumerate(_results(step)): + for m, ref in enumerate(result.get("subagent_trajectory_ref") or []): + at = f"{where}.observation.results[{n}].subagent_trajectory_ref[{m}]" + trajectory_id = ref.get("trajectory_id") + if trajectory_id is None: + if not found.reader: + found.add("reference", at, "names no embedded sub-agent: the profile embeds every sub-agent") + elif trajectory_id not in embedded: + found.add("reference", at, f"no embedded sub-agent has trajectory_id {trajectory_id!r}") + + +def _totals(document: Mapping[str, Any], found: _Findings) -> None: + totals = document.get("final_metrics") + if not isinstance(totals, Mapping): + return + model_steps = [step.get("metrics") or {} for _, trajectory in _trajectories(document) + for step in trajectory.get("steps") or [] + if step.get("source") == "agent" and step.get("llm_call_count") != 0] + sums = [(f"final_metrics.{total}", _count(totals.get(total)), [_count(m.get(count)) for m in model_steps]) + for count, total in _COUNTS] + sums.append((f"final_metrics.extra.{PROFILE_KEY}.total_cache_write_tokens", + _count(profile_of(totals).get("total_cache_write_tokens")), + [_count(profile_of(m).get("cache_write_tokens")) for m in model_steps])) + for where, total, counts in sums: + if total is not None and None not in counts and total != sum(counts): + found.add("totals", where, f"is {total}, but its model steps sum to {sum(counts)}") + steps = len(document.get("steps") or []) + total_steps = _count(totals.get("total_steps")) + if total_steps is not None and total_steps != steps and not document.get("notes"): + found.add("total-steps", "final_metrics.total_steps", + f"is {total_steps}, but the root has {steps} steps and its notes don't say why") + + +def _usage(document: Mapping[str, Any], found: _Findings) -> None: + for where, step in _steps(document): + metrics = step.get("metrics") or {} + prompt = _count(metrics.get("prompt_tokens")) + if prompt is None: + continue + writes = _count(profile_of(metrics).get("cache_write_tokens")) + cache = (_count(metrics.get("cached_tokens")) or 0) + (writes or 0) + if cache > prompt: + found.add("usage", f"{where}.metrics", + f"cache reads and writes ({cache}) exceed prompt_tokens ({prompt}), which include them") + + +def _media(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[str, bytes] | None) -> None: + for where, path, info in _media_paths(document): + if "://" in path: + found.add("media-path", where, f"{path!r} is a URL; the profile's media are files beside the document") + continue + named = MEDIA_PATH.match(path) + if not named: + found.add("media-path", where, f"{path!r} isn't media/.") + if media is None: + continue + data = _media_bytes(media, path) + if data is None: + found.add("media-bytes", where, f"{path} isn't in the media given") + continue + digest = hashlib.sha256(data).hexdigest() + if named and named["sha256"] != digest: + found.add("media-bytes", where, f"{path}'s bytes have sha256 {digest}") + if info.get("sha256") is not None and info["sha256"] != digest: + found.add("media-bytes", where, f"the profile gives sha256 {info['sha256']}, but the bytes have {digest}") + if info.get("bytes") is not None and info["bytes"] != len(data): + found.add("media-bytes", where, f"the profile gives {info['bytes']} bytes, but the file has {len(data)}") + + +def _media_paths(document: Mapping[str, Any]) -> Iterator[tuple[str, str, Mapping[str, Any]]]: + """Each media path the document names, where, and the profile's details for it.""" + for where, step in _steps(document): + holders = [(f"{where}.message", step, step.get("message"))] + [ + (f"{where}.observation.results[{n}].content", result, result.get("content")) + for n, result in enumerate(_results(step))] + for at, holder, parts in holders: + details = profile_of(holder).get("media") or {} + details = details if isinstance(details, Mapping) else {} + paths = [(f"{at}[{n}].source.path", part["source"]["path"]) + for n, part in enumerate(parts if isinstance(parts, list) else []) + if isinstance(part, Mapping) and isinstance((part.get("source") or {}).get("path"), str)] + listed = {path for _, path in paths} + owner = at.rsplit(".", 1)[0] + paths += [(f"{owner}.extra.{PROFILE_KEY}.media[{path}]", path) for path in details if path not in listed] + for path_where, path in paths: + info = details.get(path) + yield path_where, path, info if isinstance(info, Mapping) else {} + + +def _media_bytes(media: Path | Mapping[str, bytes], path: str) -> bytes | None: + if isinstance(media, Mapping): + return media.get(path) + root = media.resolve() + target = (root / path).resolve() + if not target.is_relative_to(root) or not target.is_file(): + return None + return target.read_bytes() + + +def _timestamps(document: Mapping[str, Any], found: _Findings) -> None: + for where, step in _steps(document): + if isinstance(step.get("timestamp"), str) and not TIMESTAMP.match(step["timestamp"]): + found.add("timestamp", f"{where}.timestamp", f"{step['timestamp']!r} isn't in the builder's form") + for n, result in enumerate(_results(step)): + arrived = profile_of(result).get("arrived") + at = arrived.get("at") if isinstance(arrived, Mapping) else None + if isinstance(at, str) and not TIMESTAMP.match(at): + found.add("timestamp", f"{where}.observation.results[{n}].extra.{PROFILE_KEY}.arrived.at", + f"{at!r} isn't in the builder's form") + + +def _unanswered(page: Mapping[str, Any], found: _Findings) -> None: + """A page's steps whose calls have no result yet, served before their task ended. A sub-agent's task is its + spawning call's, found through the references; one no served step references can't be placed, so isn't checked.""" + ended_steps: set[int] = set() + offset = 0 + for task in tasks(page): + if task.state is not None: + ended_steps.update(range(offset, offset + len(task.steps))) + offset += len(task.steps) + resolve = _resolver(page) + spawner: dict[str, tuple[str, int]] = {} + for prefix, trajectory in _trajectories(page): + for index, step in enumerate(trajectory.get("steps") or []): + for result in _results(step): + for ref in result.get("subagent_trajectory_ref") or []: + target = resolve[prefix].get(ref.get("trajectory_id")) + if target is not None: + spawner.setdefault(target, (prefix, index)) + + def root_step(prefix: str) -> int | None: + seen = set() + while prefix in spawner and prefix not in seen: + seen.add(prefix) + prefix, index = spawner[prefix] + if not prefix: + return index + return None + + for prefix, trajectory in _trajectories(page): + root_index = root_step(prefix) if prefix else None + if prefix and root_index is None: + continue + for index, step in enumerate(trajectory.get("steps") or []): + if (index if not prefix else root_index) in ended_steps: + continue + answered = {result.get("source_call_id") for result in _results(step)} + for n, call in enumerate(step.get("tool_calls") or []): + if call.get("tool_call_id") not in answered: + found.add("live-unanswered", f"{prefix}steps[{index}].tool_calls[{n}]", + f"call {call.get('tool_call_id')!r} has no result, and its task hasn't ended") + + +def _loc(loc: Sequence[Any]) -> str: + out = "" + for part in loc: + out += f"[{part}]" if isinstance(part, int) else (f".{part}" if out else str(part)) + return out + + +def _difference(a: Any, b: Any, where: str = "") -> str: + """The first path at which two JSON values differ.""" + if isinstance(a, Mapping) and isinstance(b, Mapping): + for key in list(a) + [k for k in b if k not in a]: + if key not in a or key not in b: + return f"{where}.{key}".lstrip(".") + if a[key] != b[key]: + return _difference(a[key], b[key], f"{where}.{key}".lstrip(".")) + if isinstance(a, list) and isinstance(b, list): + for index, (x, y) in enumerate(zip(a, b)): + if x != y: + return _difference(x, y, f"{where}[{index}]") + if len(a) != len(b): + return f"{where}[{min(len(a), len(b))}]" + return where diff --git a/packages/agentenv-protocol/tests/test_atif_kit.py b/packages/agentenv-protocol/tests/test_atif_kit.py new file mode 100644 index 000000000..45054f6aa --- /dev/null +++ b/packages/agentenv-protocol/tests/test_atif_kit.py @@ -0,0 +1,106 @@ +"""The adapter kit: the toy adapter passes against its own fixtures, and the kit names what's wrong with an adapter that +writes a duplicate tool_call_id, a document other than the expected one, or a from_atif that gives no native log.""" +import json +from typing import Any + +import pytest + +from agentenv_protocol.atif import ( + AdapterRun, + Fed, + Native, + RecordedTask, + TurnContext, + atif_adapter, + check_adapter, + native, + strip_profile, +) +from atif_cases import CASES, FIXTURES, Toy + +DECLARED = {"native_format": "toy-events/1", "version": "0.1.0", "input_includes_cache": False, + "tested_with": ("toy 1.0",)} + + +@atif_adapter(**DECLARED) +class Reversible(Toy): + @classmethod + def from_atif(cls, document: Any) -> Native: + return Native(log=[{"type": "say", "step": f"s{step['step_id']}", "text": step["message"]} + for step in document["steps"] if step["source"] == "agent" + and isinstance(step.get("message"), str)]) + + +@atif_adapter(**DECLARED) +class Duplicating(Reversible): + @native(type="call") + def call(self, r: dict, turn: Any) -> None: + turn.tool_call(step=r["step"], tool_call_id="toolu_01", function_name=r["name"], arguments=r.get("args")) + + +@atif_adapter(**DECLARED) +class NoLog(Toy): + @classmethod + def from_atif(cls, document: Any) -> Any: + return {"log": []} + + +def recorded(name: str) -> tuple[list[RecordedTask], dict[str, bytes]]: + """A case's tasks as the SDK records them, and the files of the media their inbound content attaches.""" + store, out = AdapterRun(Toy), [] + for task in CASES[name]: + context = dict(task.context) + inbound = list(context.pop("inbound", [])) + inbound += [store.media(label.encode(), media_type, **info) for label, media_type, info in task.inbound_media] + if task.inbound_media and name == "odd_parts": + inbound.append({"type": "text", "text": json.dumps({"ticket": 4411, "priority": "high"})}) + out.append(RecordedTask(context=TurnContext(inbound=inbound, **context), records=[Fed(r) for r in task.records], + state=task.state, final=task.final, ended_at=task.ended_at)) + return out, store.media_files + + +def _fixture(name: str) -> dict[str, Any]: + return json.loads((FIXTURES / f"{name}.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("name", sorted(CASES)) +def test_the_kit_passes_the_toy_adapter_against_its_fixture(name): + stripped = json.loads((FIXTURES / "coding.stripped.json").read_text(encoding="utf-8")) + tasks, media = recorded(name) + result = check_adapter(Reversible, tasks, expected=FIXTURES / f"{name}.json", + from_atif=[stripped, strip_profile(_fixture(name))], media=media) + assert result.errors == [] + assert result.document == _fixture(name) + assert len(result.pages) == sum(len(task.records) + 2 for task in CASES[name]) + unmatched = "unmatched" in _fixture(name)["extra"]["agentenv"] + assert {f.rule for f in result.findings} == ({"kit-unmatched"} if unmatched else set()) + + +def test_the_kit_warns_of_native_records_no_handler_takes(): + (finding,) = check_adapter(Reversible, recorded("coding")[0]).findings + assert (finding.rule, finding.severity, finding.message) == ( + "kit-unmatched", "warning", "no handler takes these native records: system (1)") + + +def test_the_kit_fails_an_adapter_that_writes_a_duplicate_tool_call_id(): + result = check_adapter(Duplicating, recorded("coding")[0]) + (finding,) = result.errors + assert (finding.rule, finding.where) == ("kit-run", "task 0, record 4") + assert "tool_call_id 'toolu_01' is already in this document" in finding.message + with pytest.raises(AssertionError, match="kit-run"): + result.raise_for_errors() + + +def test_the_kit_names_where_the_document_differs_from_the_expected_one(): + expected = _fixture("coding") + expected["steps"][5]["message"] = "something else" + (finding,) = check_adapter(Reversible, recorded("coding")[0], expected=expected).errors + assert (finding.rule, finding.where) == ("kit-expected", "steps[5].message") + + +def test_the_kit_fails_a_from_atif_that_gives_no_native_log_and_warns_when_there_is_none(): + (failed,) = check_adapter(NoLog, recorded("late_result")[0]).errors + assert (failed.rule, failed.message) == ("kit-from-atif", "returned dict, not a Native holding a list log") + result = check_adapter(Toy, recorded("late_result")[0]) + assert result.errors == [] + assert [(f.rule, f.severity) for f in result.findings] == [("kit-from-atif", "warning")] diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py new file mode 100644 index 000000000..5f61970bb --- /dev/null +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -0,0 +1,281 @@ +"""The validator: the fixtures and every live page they were served are clean, a negative fixture for each rule fails +with that rule's id at the severity PROFILE.md gives it, and the CLI reports it.""" +import copy +import json +import subprocess +import sys +from importlib.resources import files +from typing import Any, Callable + +import pytest + +from agentenv_protocol.atif import KIT_RULES, RULES, errors_in, validate, validate_live +from agentenv_protocol.atif.__main__ import main +from atif_cases import CASES, FIXTURES, build + +SPEC = files("agentenv_protocol.atif").joinpath("PROFILE.md").read_text(encoding="utf-8") + + +def _fixture(name: str) -> dict[str, Any]: + return json.loads((FIXTURES / f"{name}.json").read_text(encoding="utf-8")) + + +def _rules(findings, severity: str | None = None) -> set[str]: + return {f.rule for f in findings if severity is None or f.severity == severity} + + +def _parts(document: dict[str, Any]) -> list[dict[str, Any]]: + return [part for step in document["steps"] for result in (step.get("observation") or {}).get("results", []) + for part in result.get("content") or [] if isinstance(part, dict) and part.get("type") == "image"] + + +def _duplicate_call_id(document: dict[str, Any]) -> None: + step = document["steps"][2] + step["tool_calls"][0]["tool_call_id"] = "toolu_01" + step["observation"]["results"][0]["source_call_id"] = "toolu_01" + + +def _dangling_reference(document: dict[str, Any]) -> None: + for step in document["steps"]: + for result in (step.get("observation") or {}).get("results", []): + for ref in result.get("subagent_trajectory_ref") or []: + ref["trajectory_id"] = "nobody" + + +def _rename_media(document: dict[str, Any]) -> None: + _parts(document)[0]["source"]["path"] = "screens/1.png" + + +def _one_more(field: str) -> Callable[[dict[str, Any]], None]: + def mutate(document: dict[str, Any]) -> None: + document["final_metrics"][field] += 1 + return mutate + + +def _over_cached(document: dict[str, Any]) -> None: + metrics = document["steps"][1]["metrics"] + metrics["cached_tokens"] = metrics["prompt_tokens"] + 1 + + +def _text_parts(document: dict[str, Any]) -> None: + step = document["steps"][-1] + step["message"] = [{"type": "text", "text": step["message"]}] + + +#: A fixture and the change that breaks one rule. +NEGATIVE: dict[str, tuple[str, Callable[[dict[str, Any]], None]]] = { + "json": ("coding", lambda d: d["steps"][1]["tool_calls"][0]["arguments"].update(x=float("nan"))), + "atif": ("coding", lambda d: d["steps"][1].update(step_id=9)), + "profile": ("coding", lambda d: d["steps"][0]["extra"]["agentenv"].update(turn="zero")), + "profile-key": ("coding", lambda d: d["steps"][0]["extra"]["agentenv"].update(later=1)), + "profile-version": ("coding", lambda d: d["extra"]["agentenv"].update(version="2.0")), + "vocabulary": ("coding", lambda d: d["steps"][-1]["extra"]["agentenv"].update(state="paused")), + "outside-key": ("coding", lambda d: d["steps"][0]["extra"].update(note="x")), + "call-id": ("coding", _duplicate_call_id), + "reference": ("subagent", _dangling_reference), + "totals": ("coding", _one_more("total_prompt_tokens")), + "total-steps": ("coding", _one_more("total_steps")), + "usage": ("coding", _over_cached), + "media-path": ("computer_use", _rename_media), + "timestamp": ("coding", lambda d: d["steps"][0].update(timestamp="2026-11-04T17:02:11.204+00:00")), + "harbor-save": ("coding", lambda d: d["steps"][1].update(model_name=None)), + "text-message": ("coding", _text_parts), +} + + +def _negative(rule: str) -> dict[str, Any]: + name, mutate = NEGATIVE[rule] + document = _fixture(name) + mutate(document) + return document + + +@pytest.mark.parametrize("path", sorted(FIXTURES.glob("*.json")), ids=lambda p: p.stem) +def test_every_fixture_is_clean_for_a_producer_and_for_a_reader(path): + document = json.loads(path.read_text(encoding="utf-8")) + assert validate(document, media=FIXTURES) == [] + assert validate(document, reader=True, media=FIXTURES) == [] + + +@pytest.mark.parametrize("name", sorted(CASES)) +def test_every_page_a_case_served_passes_the_live_rule(name): + document, pages, media = build(name) + assert validate_live(pages, document, media=media) == [] + + +def test_every_rule_has_a_negative_fixture(): + assert set(NEGATIVE) | {"media-bytes", "live-prefix", "live-unanswered"} == set(RULES) + + +@pytest.mark.parametrize("rule", sorted(NEGATIVE)) +def test_a_negative_fixture_fails_with_its_rules_id_and_the_cli_reports_it(rule, tmp_path, capsys): + document = _negative(rule) + assert rule in _rules(errors_in(validate(document))) + path = tmp_path / "trajectory.json" + path.write_text(json.dumps(document), encoding="utf-8") + assert main(["validate", str(path)]) == 1 + assert f"error {rule} " in capsys.readouterr().out + + +@pytest.mark.parametrize("rule", sorted(NEGATIVE)) +def test_a_reader_sees_each_rule_at_its_reader_severity(rule): + found = {f.severity for f in validate(_negative(rule), reader=True) if f.rule == rule} + assert found == ({RULES[rule].reader} if RULES[rule].reader else set()) + + +def test_an_unknown_profile_key_fails_a_producer_and_only_warns_a_reader(tmp_path, capsys): + document = _negative("profile-key") + assert [(f.rule, f.severity, f.where) for f in validate(document)] == [ + ("profile-key", "error", "steps[0].extra.agentenv.later")] + assert [(f.rule, f.severity) for f in validate(document, reader=True)] == [("profile-key", "warning")] + path = tmp_path / "trajectory.json" + path.write_text(json.dumps(document), encoding="utf-8") + assert main(["validate", "--reader", str(path)]) == 0 + assert "warning profile-key steps[0].extra.agentenv.later" in capsys.readouterr().out + + +def test_media_bytes_are_checked_against_their_name_and_profile(): + document = _fixture("computer_use") + path = _parts(document)[0]["source"]["path"] + assert validate(document, media=FIXTURES) == [] + assert "media-bytes" in _rules(errors_in(validate(document, media={}))) + assert "media-bytes" in _rules(errors_in(validate(document, media={path: b"not the screenshot"}))) + + +def test_totals_sum_the_sub_agents_steps_and_skip_a_count_a_step_lacks(): + document = _fixture("subagent") + root = sum(s["metrics"]["prompt_tokens"] for s in document["steps"] if s.get("metrics")) + assert root < document["final_metrics"]["total_prompt_tokens"] + assert validate(document) == [] + sub_step = next(s for s in document["subagent_trajectories"][0]["steps"] if s.get("metrics")) + del sub_step["metrics"]["cached_tokens"] + assert validate(document) == [] + document["final_metrics"]["total_prompt_tokens"] += 1 + assert [f.where for f in validate(document)] == ["final_metrics.total_prompt_tokens"] + sub_step["metrics"]["prompt_tokens"] = None + assert "totals" not in _rules(validate(document)) + + +def test_a_count_given_as_a_numeric_string_is_read_as_the_models_read_it(): + document = _fixture("coding") + metrics = document["steps"][1]["metrics"] + metrics["prompt_tokens"] = str(metrics["prompt_tokens"]) + assert _rules(validate(document)) == {"harbor-save"} + metrics["cached_tokens"] = str(int(metrics["prompt_tokens"]) + 1) + assert "usage" in _rules(errors_in(validate(document))) + + +def test_a_sub_agent_embedded_in_a_sub_agent_is_checked_at_its_full_path(): + document = _fixture("subagent") + sub = document["subagent_trajectories"][0] + nested = copy.deepcopy(sub) + nested["trajectory_id"] = "deeper" + for step in nested["steps"]: + for call in step.get("tool_calls") or []: + call["tool_call_id"] += "-deeper" + for result in (step.get("observation") or {}).get("results", []): + result["source_call_id"] += "-deeper" + nested["steps"][0].setdefault("extra", {}).setdefault("agentenv", {})["turn"] = "zero" + sub["subagent_trajectories"] = [nested] + assert ("profile", "subagent_trajectories[0].subagent_trajectories[0].steps[0].extra.agentenv.turn") in { + (f.rule, f.where) for f in errors_in(validate(document))} + + +def test_a_reference_resolves_to_a_sub_agent_embedded_around_it(): + document = _fixture("subagent") + sub = document["subagent_trajectories"][0] + step = next(s for s in sub["steps"] if s.get("tool_calls")) + result = next(r for r in step["observation"]["results"]) + result["subagent_trajectory_ref"] = [{"trajectory_id": sub["trajectory_id"]}] + assert "reference" not in _rules(validate(document)) + result["subagent_trajectory_ref"] = [{"trajectory_id": "nobody"}] + assert "reference" in _rules(errors_in(validate(document))) + + +def _open_page(name: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, bytes]]: + """A case's final document and a page served while its task ran, with two closed steps.""" + document, pages, media = build(name) + page = next(p for p in pages[:-1] if p is not None and len(p["steps"]) >= 3) + return document, copy.deepcopy(page), media + + +def test_the_live_check_flags_a_page_that_differs_from_the_final_read_by_one_step(): + document, page, media = _open_page("coding") + assert validate_live([page], document, media=media) == [] + page["steps"][1]["message"] = "a step served, then changed" + assert [(f.rule, f.where) for f in errors_in(validate_live([page], document, media=media))] == [ + ("live-prefix", "final: ")] + + +def test_the_live_check_flags_a_page_serving_a_call_with_no_result_before_its_task_ends(): + document, page, media = _open_page("coding") + del page["steps"][1]["observation"] + assert "live-unanswered" in _rules(errors_in(validate_live([page], document, media=media))) + repeated = [f.where for f in validate_live([page, copy.deepcopy(page)], document, media=media) + if f.rule == "live-unanswered"] + assert repeated == ["page 0: steps[1].tool_calls[0]"] + + +def _agent(step_id: int, call: str, *, result: bool = True, spawns: str | None = None, + profile: dict[str, Any] | None = None) -> dict[str, Any]: + step: dict[str, Any] = {"step_id": step_id, "source": "agent", "message": "", + "tool_calls": [{"tool_call_id": call, "function_name": "run", "arguments": {}}]} + if result: + answer: dict[str, Any] = {"source_call_id": call, "content": "done"} + if spawns: + answer["subagent_trajectory_ref"] = [{"trajectory_id": spawns}] + step["observation"] = {"results": [answer]} + if profile: + step["extra"] = {"agentenv": profile} + return step + + +def _sub(trajectory_id: str, steps: list[dict[str, Any]], **more: Any) -> dict[str, Any]: + return {"schema_version": "ATIF-v1.8", "trajectory_id": trajectory_id, "agent": {"name": "a", "version": "1"}, + "steps": steps, **more} + + +def test_children_that_share_an_id_under_different_parents_keep_their_own_tasks(): + user = {"step_id": 1, "source": "user", "message": "go", "extra": {"agentenv": {"task_id": "t0", "turn": 0}}} + later = {"step_id": 3, "source": "user", "message": "again", "extra": {"agentenv": {"task_id": "t1", "turn": 1}}} + page = {"schema_version": "ATIF-v1.8", "session_id": "ctx", "agent": {"name": "a", "version": "1"}, + "steps": [user, _agent(2, "c0", spawns="A", profile={"state": "completed"}), later, + _agent(4, "c1", spawns="B")], + "subagent_trajectories": [ + _sub("A", [_agent(1, "a0", spawns="worker")], + subagent_trajectories=[_sub("worker", [_agent(1, "w0", result=False)])]), + _sub("B", [_agent(1, "b0", spawns="worker")], + subagent_trajectories=[_sub("worker", [_agent(1, "w1", result=False)])]), + ]} + unanswered = [f.where for f in validate_live([page], page) if f.rule == "live-unanswered"] + assert unanswered == ["page 0: subagent_trajectories[1].subagent_trajectories[0].steps[0].tool_calls[0]"] + assert "reference" not in _rules(validate(page)) + + +def test_a_page_that_serves_nothing_after_serving_steps_breaks_the_live_rule(): + document, page, media = _open_page("coding") + assert "live-prefix" in _rules(errors_in(validate_live([page, None], document, media=media))) + + +def test_python_m_runs_the_cli(tmp_path): + clean = subprocess.run([sys.executable, "-m", "agentenv_protocol.atif", "validate", "--media", str(FIXTURES), + str(FIXTURES / "computer_use.json")], capture_output=True, text=True) + assert (clean.returncode, clean.stdout.splitlines()[-1].split(": ")[-1]) == (0, "valid, 0 errors, 0 warnings") + path = tmp_path / "trajectory.json" + path.write_text(json.dumps(_negative("call-id")), encoding="utf-8") + broken = subprocess.run([sys.executable, "-m", "agentenv_protocol.atif", "validate", str(path)], + capture_output=True, text=True) + assert broken.returncode == 1 + assert "error call-id steps[2].tool_calls[0]: 'toolu_01' is also the id at steps[1].tool_calls[0]" in broken.stdout + + +def test_the_cli_exits_2_on_a_file_it_cant_read(tmp_path, capsys): + assert main(["validate", str(tmp_path / "missing.json")]) == 2 + assert "can't read" in capsys.readouterr().err + + +def test_profile_md_lists_every_rule_with_its_severities(): + for rule in RULES.values(): + assert f"| `{rule.id}` | {rule.holds} | {rule.producer} | {rule.reader or 'not checked'} |" in SPEC + for rule in KIT_RULES.values(): + assert f"| `{rule.id}` | {rule.holds} | {rule.producer} |" in SPEC