Repository navigation
feat(protocol): the ATIF validator, its CLI and the adapter kit (Y1b, FD-6075) #793
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
earakely-scale
merged 5 commits into
main
from
edgararakelyan/fd-6075-y1b-atif-validator
Nov 19, 2026
Merged
Changes from all commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
4a89710
feat(protocol): the ATIF validator, its CLI and the adapter kit (Y1b,…
earakely-scale 21a4da1
fix(protocol): the validator sums every embedded sub-agent, places a …
earakely-scale 919884d
fix(protocol): the validator walks sub-agents at any depth, counts as…
earakely-scale 0179042
fix(protocol): the live check places each sub-agent by its scoped pat…
earakely-scale 5ebd6ee
perf(protocol): the live check validates a repeated page once, and er…
earakely-scale File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
packages/agentenv-protocol/src/agentenv_protocol/atif/__main__.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| """``python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]``: check an ATIF document, printing each | ||
| finding; exit 1 when any is an error, 2 when the file can't be read.""" | ||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import sys | ||
| from pathlib import Path | ||
| from typing import Sequence | ||
|
|
||
| from .validator import errors_in, parse, validate | ||
|
|
||
|
|
||
| def main(argv: Sequence[str] | None = None) -> int: | ||
| parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.atif", | ||
| description="Check ATIF documents against ATIF and the agentenv profile.") | ||
| commands = parser.add_subparsers(dest="command", required=True) | ||
| check = commands.add_parser("validate", help="check one document; exit 1 when it has an error") | ||
| check.add_argument("file", type=Path, help="the trajectory.json to check") | ||
| check.add_argument("--media", type=Path, metavar="DIR", | ||
| help="the folder the document's media paths are relative to, which holds media/; " | ||
| "given, each file is checked against its name and profile") | ||
| check.add_argument("--reader", action="store_true", | ||
| help="check someone else's document for reading: what a producer must not write warns") | ||
| args = parser.parse_args(argv) | ||
| try: | ||
| document, findings = parse(args.file) | ||
| except OSError as exc: | ||
| print(f"can't read {args.file}: {exc.strerror or exc}", file=sys.stderr) | ||
| return 2 | ||
| if not findings: | ||
| findings = validate(document, reader=args.reader, media=args.media) | ||
| for finding in findings: | ||
| print(finding) | ||
| failed = errors_in(findings) | ||
| print(f"{args.file}: {'invalid' if failed else 'valid'}, {len(failed)} errors, " | ||
| f"{len(findings) - len(failed)} warnings") | ||
| return 1 if failed else 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| sys.exit(main()) |
106 changes: 106 additions & 0 deletions
106
packages/agentenv-protocol/src/agentenv_protocol/atif/kit.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| """The adapter kit, for an agent's CI: run an adapter over recorded tasks, as the SDK runs it live, and check what it | ||
| writes. The live rule holds after every record, the final document passes the validator as a producer's, it equals | ||
| the checked-in document when one is given, and ``from_atif`` turns ATIF without the profile back into a native log.""" | ||
| from __future__ import annotations | ||
|
|
||
| import json | ||
| from dataclasses import dataclass, field | ||
| from pathlib import Path | ||
| from typing import Any, Mapping, Sequence | ||
|
|
||
| from .adapter import Adapter, Native | ||
| from .profile import profile_of | ||
| from .reader import strip_profile | ||
| from .runner import AdapterRun, RecordedTask | ||
| from .validator import ERROR, WARNING, Finding, Rule, _difference, errors_in, validate_live | ||
|
|
||
| KIT_RULES: dict[str, Rule] = {rule.id: rule for rule in ( | ||
| Rule("kit-run", ERROR, None, "The adapter takes every recorded record without raising"), | ||
| Rule("kit-expected", ERROR, None, "The document equals the checked-in expected document"), | ||
| Rule("kit-unmatched", WARNING, None, "Every native record has a handler; one that none takes is only counted"), | ||
| Rule("kit-from-atif", ERROR, None, "`from_atif` turns a document without the profile into a native log"), | ||
| )} | ||
|
|
||
|
|
||
| @dataclass | ||
| class KitResult: | ||
| """What the kit found, with the document and every live page the adapter's run served.""" | ||
|
|
||
| document: dict[str, Any] | None | ||
| pages: list[dict[str, Any] | None] = field(default_factory=list) | ||
| media: dict[str, bytes] = field(default_factory=dict) | ||
| findings: list[Finding] = field(default_factory=list) | ||
|
|
||
| @property | ||
| def errors(self) -> list[Finding]: | ||
| return errors_in(self.findings) | ||
|
|
||
| def raise_for_errors(self) -> None: | ||
| """Raise AssertionError listing every finding when any is an error.""" | ||
| if self.errors: | ||
| raise AssertionError("the adapter kit found:\n" + "\n".join(str(f) for f in self.findings)) | ||
|
|
||
|
|
||
| def check_adapter(adapter: type[Adapter], recorded: Sequence[RecordedTask], *, | ||
| expected: Mapping[str, Any] | Path | None = None, | ||
| from_atif: Sequence[Mapping[str, Any]] | None = None, | ||
| media: Mapping[str, bytes] | None = None) -> KitResult: | ||
| """Run ``adapter`` over ``recorded``, one A2A context's tasks in order, and check it. ``expected`` is the document | ||
| the run should write, or the file holding it. ``from_atif`` are the documents its ``from_atif`` is tried on; by | ||
| default, the run's own document with the profile stripped. ``media`` holds, by path, the files of any Media in the | ||
| recorded tasks' inbound or final content; the adapter's own media come from the run.""" | ||
| run = AdapterRun(adapter) | ||
| result = KitResult(document=None) | ||
| where = "" | ||
| try: | ||
| for t, task in enumerate(recorded): | ||
| where = f"task {t}, its start" | ||
| run.start_task(task.context) | ||
| result.pages.append(run.page()) | ||
| for r, fed in enumerate(task.records): | ||
| where = f"task {t}, record {r}" + (f" of aux {fed.aux!r}" if fed.aux is not None else "") | ||
| if fed.aux is None: | ||
| run.native(fed.record) | ||
| else: | ||
| run.aux(fed.aux, fed.record) | ||
| result.pages.append(run.page()) | ||
| where = f"task {t}, its end" | ||
| run.end_task(state=task.state, final=task.final, ended_at=task.ended_at) | ||
| result.pages.append(run.page()) | ||
| except Exception as exc: | ||
| result.findings.append(Finding("kit-run", ERROR, where, f"{type(exc).__name__}: {exc}")) | ||
| return result | ||
| result.document = run.document() | ||
| result.media = {**(media or {}), **run.media_files} | ||
| if result.document is None: | ||
| result.findings.append(Finding("kit-run", ERROR, "", "the run wrote no step")) | ||
| return result | ||
| result.findings.extend(validate_live(result.pages, result.document, media=result.media)) | ||
| unmatched = profile_of(result.document).get("unmatched") | ||
| if unmatched: | ||
| counts = ", ".join(f"{kind} ({count})" for kind, count in sorted(unmatched.items())) | ||
| result.findings.append(Finding("kit-unmatched", WARNING, "extra.agentenv.unmatched", | ||
| f"no handler takes these native records: {counts}")) | ||
| if expected is not None: | ||
| want = json.loads(expected.read_text(encoding="utf-8")) if isinstance(expected, Path) else expected | ||
| if result.document != want: | ||
| result.findings.append(Finding("kit-expected", ERROR, _difference(want, result.document), | ||
| "the document differs from the expected one here")) | ||
| for index, document in enumerate(from_atif if from_atif is not None else [strip_profile(result.document)]): | ||
| _from_atif(adapter, document, index, result) | ||
| return result | ||
|
|
||
|
|
||
| def _from_atif(adapter: type[Adapter], document: Mapping[str, Any], index: int, result: KitResult) -> None: | ||
| where = f"from_atif document {index}" | ||
| try: | ||
| native = adapter.from_atif(document) | ||
| except NotImplementedError as exc: | ||
| result.findings.append(Finding("kit-from-atif", WARNING, where, str(exc))) | ||
| return | ||
| except Exception as exc: | ||
| result.findings.append(Finding("kit-from-atif", ERROR, where, f"raised {type(exc).__name__}: {exc}")) | ||
| return | ||
| if not isinstance(native, Native) or not isinstance(native.log, list): | ||
| result.findings.append(Finding("kit-from-atif", ERROR, where, | ||
| f"returned {type(native).__name__}, not a Native holding a list log")) | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.