From 7b2bb1f0ccdd9effbeafb696ac31f04bc9a6f2a9 Mon Sep 17 00:00:00 2001 From: Hannu Varjoranta Date: Thu, 22 Oct 2026 22:58:37 +0300 Subject: [PATCH 1/7] cli: add read-only test intelligence commands --- avrea_cli/commands/tests.py | 297 +++++++++++++ avrea_cli/helpers.py | 8 +- avrea_cli/main.py | 7 + docs/REFERENCE.md | 110 +++++ docs/reference.json | 680 ++++++++++++++++++++++++++++++ docs/starlight/_cli-sidebar.mjs | 1 + docs/starlight/reference/index.md | 1 + docs/starlight/reference/tests.md | 113 +++++ man/man1/avr-audit-events.1 | 1 + man/man1/avr-auth.1 | 1 + man/man1/avr-billing.1 | 1 + man/man1/avr-cache.1 | 1 + man/man1/avr-config.1 | 1 + man/man1/avr-firewall.1 | 1 + man/man1/avr-health.1 | 1 + man/man1/avr-job.1 | 1 + man/man1/avr-log.1 | 1 + man/man1/avr-org.1 | 1 + man/man1/avr-pr.1 | 1 + man/man1/avr-repo.1 | 1 + man/man1/avr-run.1 | 1 + man/man1/avr-sbom.1 | 1 + man/man1/avr-settings.1 | 1 + man/man1/avr-status.1 | 1 + man/man1/avr-tests-captures.1 | 35 ++ man/man1/avr-tests-history.1 | 42 ++ man/man1/avr-tests-list.1 | 36 ++ man/man1/avr-tests-status.1 | 30 ++ man/man1/avr-tests-view.1 | 40 ++ man/man1/avr-tests.1 | 51 +++ man/man1/avr-token.1 | 1 + man/man1/avr-vm.1 | 1 + man/man1/avr-workflow.1 | 1 + man/man1/avr.1 | 4 + tests/test_tests.py | 132 ++++++ 35 files changed, 1602 insertions(+), 4 deletions(-) create mode 100644 avrea_cli/commands/tests.py create mode 100644 docs/starlight/reference/tests.md create mode 100644 man/man1/avr-tests-captures.1 create mode 100644 man/man1/avr-tests-history.1 create mode 100644 man/man1/avr-tests-list.1 create mode 100644 man/man1/avr-tests-status.1 create mode 100644 man/man1/avr-tests-view.1 create mode 100644 man/man1/avr-tests.1 create mode 100644 tests/test_tests.py diff --git a/avrea_cli/commands/tests.py b/avrea_cli/commands/tests.py new file mode 100644 index 00000000..6b4158ce --- /dev/null +++ b/avrea_cli/commands/tests.py @@ -0,0 +1,297 @@ +"""Read retained test evidence through the customer API.""" + +from avrea_cli.api_client import ApiClient +from avrea_cli.click_ext import GhGroup +from avrea_cli.config import CliConfig +from avrea_cli.display import escape_control_characters +from avrea_cli.helpers import ensure_authenticated +from avrea_cli.helpers import ensure_ctx +from avrea_cli.helpers import get_org_id +from avrea_cli.helpers import handle_http_error +from avrea_cli.helpers import validate_cursor +from avrea_cli.json_output import emit_json_record +from avrea_cli.json_output import handle_json_meta +from avrea_cli.json_output import json_options +from avrea_cli.json_output import make_schema +from avrea_cli.json_output import split_fields +from avrea_cli.output import output_list +from avrea_cli.repo_context import resolve_repo_or_detect +from typing import Any +import click +import httpx +import re + +_FIELDS = make_schema( + "organization_id", + "repository_id", + "test_id", + "configuration_digest", + "capture_id", + "status", + "reason", + "pending", + "as_of_revision", + "current_revision", + "data", + "pagination", + "truncated", +) +_CONTROL_FIELDS = make_schema("organization_id", "repository_id", "data") +_repo = click.option("--repo", "repo_id", help="Repository (owner/name or rep-ID). Defaults to the git remote.") +_org = click.option("--org", "org_id", help="Organization ID or slug. Defaults to the configured organization.") +_limit = click.option( + "-L", "--limit", type=click.IntRange(1, 100), default=50, show_default=True, help="Rows per page (1-100)." +) +_cursor = click.option("--cursor", help="Opaque next_cursor from the previous page. One page is read per invocation.") + + +def _digest(_ctx: click.Context, param: click.Parameter, value: str) -> str: + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise click.BadParameter("must be a 64-character lowercase hex identifier", param=param) + return value + + +def _capture(_ctx: click.Context, param: click.Parameter, value: str) -> str: + if not re.fullmatch(r"tec-[0-9a-f]{32}", value): + raise click.BadParameter("must be a tec- identifier followed by 32 lowercase hex characters", param=param) + return value + + +def _page_params(limit: int, cursor: str | None) -> dict[str, Any]: + cursor = validate_cursor(cursor, max_length=8192) + return {"limit": limit, **({"cursor": cursor} if cursor is not None else {})} + + +def _read( + ctx: click.Context, + org_id: str | None, + repo_id: str | None, + suffix: str, + params: dict[str, Any] | None = None, +) -> dict[str, Any]: + client: ApiClient = ctx.obj["client"] + config: CliConfig = ctx.obj["config"] + ensure_authenticated(config) + organization = get_org_id(config, org_id, client=client) + repository = resolve_repo_or_detect(client, config, organization, repo_id, required=True) + try: + response = client.public_get(f"/orgs/{organization}/repos/{repository}/tests/{suffix}", params=params) + except httpx.HTTPStatusError as exc: + handle_http_error(exc, "read test evidence") + return {**response, "organization_id": organization, "repository_id": repository} + + +def _show(response: dict[str, Any], json_fields: str | None, jq_expr: str | None, *, detail: bool = False) -> None: + if json_fields is not None: + emit_json_record(response, split_fields(json_fields, _FIELDS), _FIELDS, jq_expr) + return + click.echo(f"Organization: {response['organization_id']} Repository: {response['repository_id']}") + for field in ("test_id", "configuration_digest", "capture_id", "status", "reason"): + if response.get(field) is not None: + click.echo(f"{field}: {escape_control_characters(str(response[field]))}") + if response.get("pending"): + click.echo("New results are still being indexed.") + if response.get("truncated"): + click.echo( + "This result is partial. The detail read returns at most 500 occurrences; " + "--limit controls the requested count." + ) + rows = response.get("data") or [] + if detail: + for row in rows: + click.echo() + for key in ("suite", "class_name", "name", "outcome", "duration_ms", "failure_summary"): + if row.get(key) is not None: + click.echo(f"{key}: {escape_control_characters(str(row[key]))}") + elif rows: + if "test_id" in rows[0]: + display_rows = [{**row, "name": (row.get("identity") or {}).get("name") or row["test_id"]} for row in rows] + display_rows = [ + { + key: escape_control_characters(str(value)) if isinstance(value, str) else value + for key, value in row.items() + } + for row in display_rows + ] + output_list( + display_rows, + columns=["name", "test_id", "configuration_digest", "failed_count", "errored_count", "passed_count"], + column_labels=["Test", "Test ID", "Configuration", "Failed", "Errors", "Passed"], + ) + else: + columns = [ + key + for key in ( + "capture_id", + "captured_at", + "armed_at", + "state", + "detail_status", + "commit_sha", + "reason", + "passed_count", + "failed_count", + "errored_count", + "duration_total_ms", + ) + if key in rows[0] + ] + display_rows = [ + {key: "-" if value is None else escape_control_characters(str(value)) for key, value in row.items()} + for row in rows + ] + output_list(display_rows, columns=columns, column_labels=columns) + else: + click.echo("No recorded results in this page. This does not establish that no tests ran.") + next_cursor = (response.get("pagination") or {}).get("next_cursor") + if next_cursor: + click.echo(f"Next page: --cursor {escape_control_characters(str(next_cursor))}", err=True) + + +@click.group(cls=GhGroup) +@click.pass_context +def tests(ctx: click.Context) -> None: + """Inspect recorded tests, failures, and reporting status.""" + ensure_ctx(ctx) + + +@tests.command("list") +@_repo +@_org +@_limit +@_cursor +@json_options +@click.pass_context +def tests_list( + ctx: click.Context, + repo_id: str | None, + org_id: str | None, + limit: int, + cursor: str | None, + json_fields: str | None, + jq_expr: str | None, +) -> None: + """List one page of test histories, grouped by test and configuration. + + Counts describe each returned history window, not repository totals. + Names appear when retained private detail is available; otherwise IDs are shown. + Use --json '*' to retain status, pagination and repository context for agents. + """ + if handle_json_meta(json_fields, jq_expr, _FIELDS): + return + _show(_read(ctx, org_id, repo_id, "history", _page_params(limit, cursor)), json_fields, jq_expr) + + +@tests.command("history") +@click.argument("test_id", callback=_digest) +@click.option( + "--configuration", + "configuration_digest", + required=True, + callback=_digest, + help="History grouping identifier returned by tests list; not a platform label.", +) +@_repo +@_org +@_limit +@_cursor +@json_options +@click.pass_context +def tests_history( + ctx: click.Context, + test_id: str, + configuration_digest: str, + repo_id: str | None, + org_id: str | None, + limit: int, + cursor: str | None, + json_fields: str | None, + jq_expr: str | None, +) -> None: + """Read recorded outcomes for one test and configuration, newest first.""" + if handle_json_meta(json_fields, jq_expr, _FIELDS): + return + params = {**_page_params(limit, cursor), "configuration": configuration_digest} + response = _read(ctx, org_id, repo_id, f"history/{test_id}", params) + response.update(test_id=test_id, configuration_digest=configuration_digest) + _show(response, json_fields, jq_expr) + + +@tests.command("view") +@click.argument("test_id", callback=_digest) +@click.option("--capture", "capture_id", required=True, callback=_capture, help="Exact capture from tests history.") +@_repo +@_org +@click.option( + "-L", "--limit", type=click.IntRange(1, 500), default=50, show_default=True, help="Occurrences to read (1-500)." +) +@json_options +@click.pass_context +def tests_view( + ctx: click.Context, + test_id: str, + capture_id: str, + repo_id: str | None, + org_id: str | None, + limit: int, + json_fields: str | None, + jq_expr: str | None, +) -> None: + """Read the exact retained result, including available failure messages. + + An expired or unrecorded result is never replaced with a different capture. + A recorded test failure is data, not a CLI error. No tests are rerun. + """ + if handle_json_meta(json_fields, jq_expr, _FIELDS): + return + response = _read(ctx, org_id, repo_id, f"history/{test_id}/detail/{capture_id}", {"limit": limit}) + response.update(test_id=test_id, capture_id=capture_id) + _show(response, json_fields, jq_expr, detail=True) + + +@tests.command("captures") +@_repo +@_org +@_limit +@_cursor +@json_options +@click.pass_context +def tests_captures( + ctx: click.Context, + repo_id: str | None, + org_id: str | None, + limit: int, + cursor: str | None, + json_fields: str | None, + jq_expr: str | None, +) -> None: + """List report deliveries, including missing, partial and failed captures.""" + if handle_json_meta(json_fields, jq_expr, _FIELDS): + return + _show(_read(ctx, org_id, repo_id, "captures", _page_params(limit, cursor)), json_fields, jq_expr) + + +@tests.command("status") +@_repo +@_org +@json_options +@click.pass_context +def tests_status( + ctx: click.Context, repo_id: str | None, org_id: str | None, json_fields: str | None, jq_expr: str | None +) -> None: + """Read collection settings. Use --json for retained-history lifecycle details.""" + if handle_json_meta(json_fields, jq_expr, _CONTROL_FIELDS): + return + result = _read(ctx, org_id, repo_id, "control") + response = { + "organization_id": result.pop("organization_id"), + "repository_id": result.pop("repository_id"), + "data": result, + } + if json_fields is not None: + emit_json_record(response, split_fields(json_fields, _CONTROL_FIELDS), _CONTROL_FIELDS, jq_expr) + return + click.echo(f"Organization: {response['organization_id']} Repository: {response['repository_id']}") + click.echo(f"Collection: {'on' if result['capture_enabled'] else 'off'}") + click.echo(f"Can manage collection: {'yes' if result['can_manage'] else 'no'}") + click.echo("Use 'avr tests captures' to inspect delivery and missing-report evidence.") diff --git a/avrea_cli/helpers.py b/avrea_cli/helpers.py index b5452312..3cc6821a 100644 --- a/avrea_cli/helpers.py +++ b/avrea_cli/helpers.py @@ -267,13 +267,13 @@ def get_verified_org_slug(client: ApiClient, org_id: str) -> str: raise click.ClickException("Could not verify the selected organization for this Avrea run URL.") -# Opaque base64 cursors are short — Avrea's cursors are well under 200 chars. +# Most list cursors are short; endpoints with snapshot cursors can set a larger bound. # 512 leaves headroom for protocol changes while still rejecting paste-mistakes # (e.g. a whole file pasted into --cursor) before they hit the server. _MAX_CURSOR_LEN = 512 -def validate_cursor(cursor: str | None) -> str | None: +def validate_cursor(cursor: str | None, *, max_length: int = _MAX_CURSOR_LEN) -> str | None: """Bound a user-supplied pagination cursor at the CLI boundary. Cursors are opaque base64 from a previous response, so the only useful @@ -284,9 +284,9 @@ def validate_cursor(cursor: str | None) -> str | None: through.""" if cursor is None: return None - if not cursor or cursor.strip() != cursor or len(cursor) > _MAX_CURSOR_LEN: + if not cursor or cursor.strip() != cursor or len(cursor) > max_length: raise click.BadParameter( - f"cursor must be 1..{_MAX_CURSOR_LEN} chars and not surrounded by whitespace (got {len(cursor)} chars)", + f"cursor must be 1..{max_length} chars and not surrounded by whitespace (got {len(cursor)} chars)", param_hint="--cursor", ) return cursor diff --git a/avrea_cli/main.py b/avrea_cli/main.py index 2fc6e85d..b70129c0 100644 --- a/avrea_cli/main.py +++ b/avrea_cli/main.py @@ -38,6 +38,13 @@ ("vm", "avrea_cli.vm", "vm", "Manage long-running VMs (SSH/RDP/VNC).", "Core Commands"), ("workflow", "avrea_cli.commands.workflow", "workflow", "List and view workflow definitions.", "Core Commands"), ("cache", "avrea_cli.commands.cache", "cache", "Inspect and manage the Avrea build cache.", "Core Commands"), + ( + "tests", + "avrea_cli.commands.tests", + "tests", + "Inspect recorded tests, failures, and reporting status.", + "Core Commands", + ), ("sbom", "avrea_cli.commands.sbom", "sbom", "View, generate, and download repository SBOMs.", "Core Commands"), ("log", "avrea_cli.commands.log", "log", "Search across runner execution logs.", "Core Commands"), ("auth", "avrea_cli.commands.auth_cmd", "auth_group", "Authenticate and manage credentials.", "Setup & Config"), diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 7cee30e2..d1bf291b 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -42,6 +42,7 @@ avr [GLOBAL OPTIONS] COMMAND [ARGS]... - [`avr vm`](#avr-vm) — Manage long-running VMs (SSH/RDP/VNC). - [`avr workflow`](#avr-workflow) — List and view workflow definitions. - [`avr cache`](#avr-cache) — Inspect and manage the Avrea build cache. +- [`avr tests`](#avr-tests) — Inspect recorded tests, failures, and reporting status. - [`avr sbom`](#avr-sbom) — View, generate, and download repository SBOMs. - [`avr log`](#avr-log) — Search across runner execution logs. @@ -1303,6 +1304,115 @@ JSON FIELDS - `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. - `-q, --jq ` — Filter --json output through a jq expression. +### `avr tests` + +Inspect recorded tests, failures, and reporting status. + +```sh +avr tests [OPTIONS] COMMAND [ARGS]... +``` + +#### `avr tests captures` + +List report deliveries, including missing, partial and failed captures. + +```sh +avr tests captures [OPTIONS] +``` + +**Options** + +- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. +- `--org ` — Organization ID or slug. Defaults to the configured organization. +- `-L, --limit ` — Rows per page (1-100). _(default: `50`)_ +- `--cursor ` — Opaque next_cursor from the previous page. One page is read per invocation. +- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- `-q, --jq ` — Filter --json output through a jq expression. + +#### `avr tests history` + +Read recorded outcomes for one test and configuration, newest first. + +```sh +avr tests history [OPTIONS] TEST_ID +``` + +**Arguments** + +- `TEST_ID` + +**Options** + +- `--configuration ` — History grouping identifier returned by tests list; not a platform label. _(required)_ +- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. +- `--org ` — Organization ID or slug. Defaults to the configured organization. +- `-L, --limit ` — Rows per page (1-100). _(default: `50`)_ +- `--cursor ` — Opaque next_cursor from the previous page. One page is read per invocation. +- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- `-q, --jq ` — Filter --json output through a jq expression. + +#### `avr tests list` + +List one page of test histories, grouped by test and configuration. + +```sh +avr tests list [OPTIONS] +``` + +Counts describe each returned history window, not repository totals. +Names appear when retained private detail is available; otherwise IDs are shown. +Use --json '*' to retain status, pagination and repository context for agents. + +**Options** + +- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. +- `--org ` — Organization ID or slug. Defaults to the configured organization. +- `-L, --limit ` — Rows per page (1-100). _(default: `50`)_ +- `--cursor ` — Opaque next_cursor from the previous page. One page is read per invocation. +- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- `-q, --jq ` — Filter --json output through a jq expression. + +#### `avr tests status` + +Read collection settings. + +```sh +avr tests status [OPTIONS] +``` + +Read collection settings. Use --json for retained-history lifecycle details. + +**Options** + +- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. +- `--org ` — Organization ID or slug. Defaults to the configured organization. +- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- `-q, --jq ` — Filter --json output through a jq expression. + +#### `avr tests view` + +Read the exact retained result, including available failure messages. + +```sh +avr tests view [OPTIONS] TEST_ID +``` + +An expired or unrecorded result is never replaced with a different capture. +A recorded test failure is data, not a CLI error. No tests are rerun. + +**Arguments** + +- `TEST_ID` + +**Options** + +- `--capture ` — Exact capture from tests history. _(required)_ +- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. +- `--org ` — Organization ID or slug. Defaults to the configured organization. +- `-L, --limit ` — Occurrences to read (1-500). _(default: `50`)_ +- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- `-q, --jq ` — Filter --json output through a jq expression. + ### `avr sbom` View, generate, and download repository SBOMs. diff --git a/docs/reference.json b/docs/reference.json index 2f2e1c16..aaf82f99 100644 --- a/docs/reference.json +++ b/docs/reference.json @@ -6248,6 +6248,686 @@ } ] }, + { + "name": "tests", + "path": [ + "avr", + "tests" + ], + "full": "avr tests", + "is_group": true, + "short_help": "Inspect recorded tests, failures, and reporting status.", + "help": "Inspect recorded tests, failures, and reporting status.", + "epilog": null, + "deprecated": false, + "hidden": false, + "usage": "avr tests [OPTIONS] COMMAND [ARGS]...", + "params": [], + "subcommands": [ + { + "name": "captures", + "path": [ + "avr", + "tests", + "captures" + ], + "full": "avr tests captures", + "is_group": false, + "short_help": "List report deliveries, including missing, partial and failed captures.", + "help": "List report deliveries, including missing, partial and failed captures.", + "epilog": null, + "deprecated": false, + "hidden": false, + "usage": "avr tests captures [OPTIONS]", + "params": [ + { + "name": "repo_id", + "kind": "option", + "opts": [ + "--repo" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "org_id", + "kind": "option", + "opts": [ + "--org" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Organization ID or slug. Defaults to the configured organization.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "limit", + "kind": "option", + "opts": [ + "-L", + "--limit" + ], + "secondary_opts": [], + "type": "integer range", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Rows per page (1-100).", + "hidden": false, + "show_default": true, + "default": 50 + }, + { + "name": "cursor", + "kind": "option", + "opts": [ + "--cursor" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Opaque next_cursor from the previous page. One page is read per invocation.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "json_fields", + "kind": "option", + "opts": [ + "--json" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "jq_expr", + "kind": "option", + "opts": [ + "-q", + "--jq" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Filter --json output through a jq expression.", + "hidden": false, + "show_default": false, + "default": null + } + ], + "subcommands": [] + }, + { + "name": "history", + "path": [ + "avr", + "tests", + "history" + ], + "full": "avr tests history", + "is_group": false, + "short_help": "Read recorded outcomes for one test and configuration, newest first.", + "help": "Read recorded outcomes for one test and configuration, newest first.", + "epilog": null, + "deprecated": false, + "hidden": false, + "usage": "avr tests history [OPTIONS] TEST_ID", + "params": [ + { + "name": "test_id", + "kind": "argument", + "opts": [ + "test_id" + ], + "secondary_opts": [], + "type": "text", + "required": true, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": null, + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "configuration_digest", + "kind": "option", + "opts": [ + "--configuration" + ], + "secondary_opts": [], + "type": "text", + "required": true, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "History grouping identifier returned by tests list; not a platform label.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "repo_id", + "kind": "option", + "opts": [ + "--repo" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "org_id", + "kind": "option", + "opts": [ + "--org" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Organization ID or slug. Defaults to the configured organization.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "limit", + "kind": "option", + "opts": [ + "-L", + "--limit" + ], + "secondary_opts": [], + "type": "integer range", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Rows per page (1-100).", + "hidden": false, + "show_default": true, + "default": 50 + }, + { + "name": "cursor", + "kind": "option", + "opts": [ + "--cursor" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Opaque next_cursor from the previous page. One page is read per invocation.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "json_fields", + "kind": "option", + "opts": [ + "--json" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "jq_expr", + "kind": "option", + "opts": [ + "-q", + "--jq" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Filter --json output through a jq expression.", + "hidden": false, + "show_default": false, + "default": null + } + ], + "subcommands": [] + }, + { + "name": "list", + "path": [ + "avr", + "tests", + "list" + ], + "full": "avr tests list", + "is_group": false, + "short_help": "List one page of test histories, grouped by test and configuration.", + "help": "List one page of test histories, grouped by test and configuration.\n\nCounts describe each returned history window, not repository totals.\nNames appear when retained private detail is available; otherwise IDs are shown.\nUse --json '*' to retain status, pagination and repository context for agents.\n", + "epilog": null, + "deprecated": false, + "hidden": false, + "usage": "avr tests list [OPTIONS]", + "params": [ + { + "name": "repo_id", + "kind": "option", + "opts": [ + "--repo" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "org_id", + "kind": "option", + "opts": [ + "--org" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Organization ID or slug. Defaults to the configured organization.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "limit", + "kind": "option", + "opts": [ + "-L", + "--limit" + ], + "secondary_opts": [], + "type": "integer range", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Rows per page (1-100).", + "hidden": false, + "show_default": true, + "default": 50 + }, + { + "name": "cursor", + "kind": "option", + "opts": [ + "--cursor" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Opaque next_cursor from the previous page. One page is read per invocation.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "json_fields", + "kind": "option", + "opts": [ + "--json" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "jq_expr", + "kind": "option", + "opts": [ + "-q", + "--jq" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Filter --json output through a jq expression.", + "hidden": false, + "show_default": false, + "default": null + } + ], + "subcommands": [] + }, + { + "name": "status", + "path": [ + "avr", + "tests", + "status" + ], + "full": "avr tests status", + "is_group": false, + "short_help": "Read collection settings.", + "help": "Read collection settings. Use --json for retained-history lifecycle details.", + "epilog": null, + "deprecated": false, + "hidden": false, + "usage": "avr tests status [OPTIONS]", + "params": [ + { + "name": "repo_id", + "kind": "option", + "opts": [ + "--repo" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "org_id", + "kind": "option", + "opts": [ + "--org" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Organization ID or slug. Defaults to the configured organization.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "json_fields", + "kind": "option", + "opts": [ + "--json" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "jq_expr", + "kind": "option", + "opts": [ + "-q", + "--jq" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Filter --json output through a jq expression.", + "hidden": false, + "show_default": false, + "default": null + } + ], + "subcommands": [] + }, + { + "name": "view", + "path": [ + "avr", + "tests", + "view" + ], + "full": "avr tests view", + "is_group": false, + "short_help": "Read the exact retained result, including available failure messages.", + "help": "Read the exact retained result, including available failure messages.\n\nAn expired or unrecorded result is never replaced with a different capture.\nA recorded test failure is data, not a CLI error. No tests are rerun.\n", + "epilog": null, + "deprecated": false, + "hidden": false, + "usage": "avr tests view [OPTIONS] TEST_ID", + "params": [ + { + "name": "test_id", + "kind": "argument", + "opts": [ + "test_id" + ], + "secondary_opts": [], + "type": "text", + "required": true, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": null, + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "capture_id", + "kind": "option", + "opts": [ + "--capture" + ], + "secondary_opts": [], + "type": "text", + "required": true, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Exact capture from tests history.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "repo_id", + "kind": "option", + "opts": [ + "--repo" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "org_id", + "kind": "option", + "opts": [ + "--org" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Organization ID or slug. Defaults to the configured organization.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "limit", + "kind": "option", + "opts": [ + "-L", + "--limit" + ], + "secondary_opts": [], + "type": "integer range", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Occurrences to read (1-500).", + "hidden": false, + "show_default": true, + "default": 50 + }, + { + "name": "json_fields", + "kind": "option", + "opts": [ + "--json" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", + "hidden": false, + "show_default": false, + "default": null + }, + { + "name": "jq_expr", + "kind": "option", + "opts": [ + "-q", + "--jq" + ], + "secondary_opts": [], + "type": "text", + "required": false, + "multiple": false, + "nargs": 1, + "is_flag": false, + "envvar": null, + "help": "Filter --json output through a jq expression.", + "hidden": false, + "show_default": false, + "default": null + } + ], + "subcommands": [] + } + ] + }, { "name": "sbom", "path": [ diff --git a/docs/starlight/_cli-sidebar.mjs b/docs/starlight/_cli-sidebar.mjs index 185c7e2d..c7cf89f7 100644 --- a/docs/starlight/_cli-sidebar.mjs +++ b/docs/starlight/_cli-sidebar.mjs @@ -18,6 +18,7 @@ export default { { label: "avr vm", slug: "cli/reference/vm" }, { label: "avr workflow", slug: "cli/reference/workflow" }, { label: "avr cache", slug: "cli/reference/cache" }, + { label: "avr tests", slug: "cli/reference/tests" }, { label: "avr sbom", slug: "cli/reference/sbom" }, { label: "avr log", slug: "cli/reference/log" }, ], diff --git a/docs/starlight/reference/index.md b/docs/starlight/reference/index.md index 23fa6a8e..c0b5ca5c 100644 --- a/docs/starlight/reference/index.md +++ b/docs/starlight/reference/index.md @@ -30,6 +30,7 @@ avr [GLOBAL OPTIONS] COMMAND [ARGS]... - [`avr vm`](./vm/) — Manage long-running VMs (SSH/RDP/VNC). - [`avr workflow`](./workflow/) — List and view workflow definitions. - [`avr cache`](./cache/) — Inspect and manage the Avrea build cache. +- [`avr tests`](./tests/) — Inspect recorded tests, failures, and reporting status. - [`avr sbom`](./sbom/) — View, generate, and download repository SBOMs. - [`avr log`](./log/) — Search across runner execution logs. diff --git a/docs/starlight/reference/tests.md b/docs/starlight/reference/tests.md new file mode 100644 index 00000000..a7564d18 --- /dev/null +++ b/docs/starlight/reference/tests.md @@ -0,0 +1,113 @@ +--- +title: avr tests +description: "Inspect recorded tests, failures, and reporting status." +--- + +Inspect recorded tests, failures, and reporting status. + +```sh +avr tests [OPTIONS] COMMAND [ARGS]... +``` + +## Subcommands + +### `avr tests captures` + +List report deliveries, including missing, partial and failed captures. + +```sh +avr tests captures [OPTIONS] +``` + +**Options** + +- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. +- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. +- -L, --limit <INTEGER RANGE> — Rows per page (1-100). _(default: `50`)_ +- --cursor <TEXT> — Opaque next_cursor from the previous page. One page is read per invocation. +- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- -q, --jq <TEXT> — Filter --json output through a jq expression. + +### `avr tests history` + +Read recorded outcomes for one test and configuration, newest first. + +```sh +avr tests history [OPTIONS] TEST_ID +``` + +**Arguments** + +- TEST_ID + +**Options** + +- --configuration <TEXT> — History grouping identifier returned by tests list; not a platform label. _(required)_ +- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. +- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. +- -L, --limit <INTEGER RANGE> — Rows per page (1-100). _(default: `50`)_ +- --cursor <TEXT> — Opaque next_cursor from the previous page. One page is read per invocation. +- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- -q, --jq <TEXT> — Filter --json output through a jq expression. + +### `avr tests list` + +List one page of test histories, grouped by test and configuration. + +```sh +avr tests list [OPTIONS] +``` + +Counts describe each returned history window, not repository totals. +Names appear when retained private detail is available; otherwise IDs are shown. +Use --json '*' to retain status, pagination and repository context for agents. + +**Options** + +- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. +- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. +- -L, --limit <INTEGER RANGE> — Rows per page (1-100). _(default: `50`)_ +- --cursor <TEXT> — Opaque next_cursor from the previous page. One page is read per invocation. +- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- -q, --jq <TEXT> — Filter --json output through a jq expression. + +### `avr tests status` + +Read collection settings. + +```sh +avr tests status [OPTIONS] +``` + +Read collection settings. Use --json for retained-history lifecycle details. + +**Options** + +- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. +- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. +- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- -q, --jq <TEXT> — Filter --json output through a jq expression. + +### `avr tests view` + +Read the exact retained result, including available failure messages. + +```sh +avr tests view [OPTIONS] TEST_ID +``` + +An expired or unrecorded result is never replaced with a different capture. +A recorded test failure is data, not a CLI error. No tests are rerun. + +**Arguments** + +- TEST_ID + +**Options** + +- --capture <TEXT> — Exact capture from tests history. _(required)_ +- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. +- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. +- -L, --limit <INTEGER RANGE> — Occurrences to read (1-500). _(default: `50`)_ +- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +- -q, --jq <TEXT> — Filter --json output through a jq expression. diff --git a/man/man1/avr-audit-events.1 b/man/man1/avr-audit-events.1 index 58939f2b..27a732a7 100644 --- a/man/man1/avr-audit-events.1 +++ b/man/man1/avr-audit-events.1 @@ -21,6 +21,7 @@ List audit events for the organization. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-auth.1 b/man/man1/avr-auth.1 index 203caefc..d1c758d7 100644 --- a/man/man1/avr-auth.1 +++ b/man/man1/avr-auth.1 @@ -30,6 +30,7 @@ Switch the default host used when AVR_HOST isn't set. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-token (1), diff --git a/man/man1/avr-billing.1 b/man/man1/avr-billing.1 index 9762073a..be771b67 100644 --- a/man/man1/avr-billing.1 +++ b/man/man1/avr-billing.1 @@ -33,6 +33,7 @@ Update billing settings. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-cache.1 b/man/man1/avr-cache.1 index cd3623f8..35994e4d 100644 --- a/man/man1/avr-cache.1 +++ b/man/man1/avr-cache.1 @@ -26,6 +26,7 @@ Show cache usage summary for a repository. .BR avr-job (1), .BR avr-vm (1), .BR avr-workflow (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-config.1 b/man/man1/avr-config.1 index fdcfacaf..122b42a8 100644 --- a/man/man1/avr-config.1 +++ b/man/man1/avr-config.1 @@ -30,6 +30,7 @@ Remove a configuration override. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-firewall.1 b/man/man1/avr-firewall.1 index 7dbdbd71..690c60a8 100644 --- a/man/man1/avr-firewall.1 +++ b/man/man1/avr-firewall.1 @@ -39,6 +39,7 @@ Show the resolved (merged) firewall rule list for a repository. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-health.1 b/man/man1/avr-health.1 index 981ba5b0..ed6cf837 100644 --- a/man/man1/avr-health.1 +++ b/man/man1/avr-health.1 @@ -32,6 +32,7 @@ Filter --json output through a jq expression. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-job.1 b/man/man1/avr-job.1 index d1e2fe05..abc782a9 100644 --- a/man/man1/avr-job.1 +++ b/man/man1/avr-job.1 @@ -35,6 +35,7 @@ Watch active jobs with auto-refresh (Ctrl+C to stop). .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-log.1 b/man/man1/avr-log.1 index 162c2bb7..349bdb4c 100644 --- a/man/man1/avr-log.1 +++ b/man/man1/avr-log.1 @@ -21,6 +21,7 @@ Search logs for a repository. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-auth (1), .BR avr-token (1), diff --git a/man/man1/avr-org.1 b/man/man1/avr-org.1 index b081c3d3..5610a106 100644 --- a/man/man1/avr-org.1 +++ b/man/man1/avr-org.1 @@ -36,6 +36,7 @@ Configure SAML single sign-on for an organization. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-pr.1 b/man/man1/avr-pr.1 index edf214d4..ed4f29e1 100644 --- a/man/man1/avr-pr.1 +++ b/man/man1/avr-pr.1 @@ -20,6 +20,7 @@ List pull requests across repositories. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-repo.1 b/man/man1/avr-repo.1 index 11afc8bd..7943a7fb 100644 --- a/man/man1/avr-repo.1 +++ b/man/man1/avr-repo.1 @@ -30,6 +30,7 @@ Request and browse mirrors of public GitHub repositories. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-run.1 b/man/man1/avr-run.1 index 494d20ba..726bcfd2 100644 --- a/man/man1/avr-run.1 +++ b/man/man1/avr-run.1 @@ -38,6 +38,7 @@ Watch a workflow run until it completes. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-sbom.1 b/man/man1/avr-sbom.1 index 81c2e65c..08046f15 100644 --- a/man/man1/avr-sbom.1 +++ b/man/man1/avr-sbom.1 @@ -30,6 +30,7 @@ Show an SBOM's summary: dependency counts, licenses, and artifacts. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-log (1), .BR avr-auth (1), .BR avr-token (1), diff --git a/man/man1/avr-settings.1 b/man/man1/avr-settings.1 index a57adcab..2e3eb049 100644 --- a/man/man1/avr-settings.1 +++ b/man/man1/avr-settings.1 @@ -30,6 +30,7 @@ Set a setting value. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-status.1 b/man/man1/avr-status.1 index e7948ec7..88a4f3e4 100644 --- a/man/man1/avr-status.1 +++ b/man/man1/avr-status.1 @@ -29,6 +29,7 @@ Output raw JSON. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-tests-captures.1 b/man/man1/avr-tests-captures.1 new file mode 100644 index 00000000..b2d50c6d --- /dev/null +++ b/man/man1/avr-tests-captures.1 @@ -0,0 +1,35 @@ +.TH AVR\-TESTS\-CAPTURES 1 "" "avr 0.4.1" "Avrea CLI" +.SH NAME +avr-tests-captures \- List report deliveries, including missing, partial and failed captures. +.SH SYNOPSIS +.B avr tests captures +[OPTIONS] +.SH DESCRIPTION +List report deliveries, including missing, partial and failed captures. +.PP +.SH OPTIONS +.TP +\fB\-\-repo\fR \fITEXT\fR +Repository (owner/name or rep-ID). Defaults to the git remote. +.TP +\fB\-\-org\fR \fITEXT\fR +Organization ID or slug. Defaults to the configured organization. +.TP +\fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR +Rows per page (1-100). (default: 50) +.TP +\fB\-\-cursor\fR \fITEXT\fR +Opaque next_cursor from the previous page. One page is read per invocation. +.TP +\fB\-\-json\fR \fITEXT\fR +Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +.TP +\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR +Filter --json output through a jq expression. +.SH SEE ALSO +.BR avr (1), +.BR avr-tests (1), +.BR avr-tests-history (1), +.BR avr-tests-list (1), +.BR avr-tests-status (1), +.BR avr-tests-view (1) diff --git a/man/man1/avr-tests-history.1 b/man/man1/avr-tests-history.1 new file mode 100644 index 00000000..dadd6ee3 --- /dev/null +++ b/man/man1/avr-tests-history.1 @@ -0,0 +1,42 @@ +.TH AVR\-TESTS\-HISTORY 1 "" "avr 0.4.1" "Avrea CLI" +.SH NAME +avr-tests-history \- Read recorded outcomes for one test and configuration, newest first. +.SH SYNOPSIS +.B avr tests history +[OPTIONS] TEST_ID +.SH DESCRIPTION +Read recorded outcomes for one test and configuration, newest first. +.PP +.SH ARGUMENTS +.TP +\fITEST_ID\fR +\& +.SH OPTIONS +.TP +\fB\-\-configuration\fR \fITEXT\fR +History grouping identifier returned by tests list; not a platform label. (required) +.TP +\fB\-\-repo\fR \fITEXT\fR +Repository (owner/name or rep-ID). Defaults to the git remote. +.TP +\fB\-\-org\fR \fITEXT\fR +Organization ID or slug. Defaults to the configured organization. +.TP +\fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR +Rows per page (1-100). (default: 50) +.TP +\fB\-\-cursor\fR \fITEXT\fR +Opaque next_cursor from the previous page. One page is read per invocation. +.TP +\fB\-\-json\fR \fITEXT\fR +Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +.TP +\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR +Filter --json output through a jq expression. +.SH SEE ALSO +.BR avr (1), +.BR avr-tests (1), +.BR avr-tests-captures (1), +.BR avr-tests-list (1), +.BR avr-tests-status (1), +.BR avr-tests-view (1) diff --git a/man/man1/avr-tests-list.1 b/man/man1/avr-tests-list.1 new file mode 100644 index 00000000..3d1c7cd6 --- /dev/null +++ b/man/man1/avr-tests-list.1 @@ -0,0 +1,36 @@ +.TH AVR\-TESTS\-LIST 1 "" "avr 0.4.1" "Avrea CLI" +.SH NAME +avr-tests-list \- List one page of test histories, grouped by test and configuration. +.SH SYNOPSIS +.B avr tests list +[OPTIONS] +.SH DESCRIPTION +List one page of test histories, grouped by test and configuration. +.PP +Counts describe each returned history window, not repository totals. Names appear when retained private detail is available; otherwise IDs are shown. Use --json '*' to retain status, pagination and repository context for agents. +.SH OPTIONS +.TP +\fB\-\-repo\fR \fITEXT\fR +Repository (owner/name or rep-ID). Defaults to the git remote. +.TP +\fB\-\-org\fR \fITEXT\fR +Organization ID or slug. Defaults to the configured organization. +.TP +\fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR +Rows per page (1-100). (default: 50) +.TP +\fB\-\-cursor\fR \fITEXT\fR +Opaque next_cursor from the previous page. One page is read per invocation. +.TP +\fB\-\-json\fR \fITEXT\fR +Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +.TP +\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR +Filter --json output through a jq expression. +.SH SEE ALSO +.BR avr (1), +.BR avr-tests (1), +.BR avr-tests-captures (1), +.BR avr-tests-history (1), +.BR avr-tests-status (1), +.BR avr-tests-view (1) diff --git a/man/man1/avr-tests-status.1 b/man/man1/avr-tests-status.1 new file mode 100644 index 00000000..43c52306 --- /dev/null +++ b/man/man1/avr-tests-status.1 @@ -0,0 +1,30 @@ +.TH AVR\-TESTS\-STATUS 1 "" "avr 0.4.1" "Avrea CLI" +.SH NAME +avr-tests-status \- Read collection settings. +.SH SYNOPSIS +.B avr tests status +[OPTIONS] +.SH DESCRIPTION +Read collection settings. +.PP +Read collection settings. Use --json for retained-history lifecycle details. +.SH OPTIONS +.TP +\fB\-\-repo\fR \fITEXT\fR +Repository (owner/name or rep-ID). Defaults to the git remote. +.TP +\fB\-\-org\fR \fITEXT\fR +Organization ID or slug. Defaults to the configured organization. +.TP +\fB\-\-json\fR \fITEXT\fR +Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +.TP +\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR +Filter --json output through a jq expression. +.SH SEE ALSO +.BR avr (1), +.BR avr-tests (1), +.BR avr-tests-captures (1), +.BR avr-tests-history (1), +.BR avr-tests-list (1), +.BR avr-tests-view (1) diff --git a/man/man1/avr-tests-view.1 b/man/man1/avr-tests-view.1 new file mode 100644 index 00000000..38ee6e15 --- /dev/null +++ b/man/man1/avr-tests-view.1 @@ -0,0 +1,40 @@ +.TH AVR\-TESTS\-VIEW 1 "" "avr 0.4.1" "Avrea CLI" +.SH NAME +avr-tests-view \- Read the exact retained result, including available failure messages. +.SH SYNOPSIS +.B avr tests view +[OPTIONS] TEST_ID +.SH DESCRIPTION +Read the exact retained result, including available failure messages. +.PP +An expired or unrecorded result is never replaced with a different capture. A recorded test failure is data, not a CLI error. No tests are rerun. +.SH ARGUMENTS +.TP +\fITEST_ID\fR +\& +.SH OPTIONS +.TP +\fB\-\-capture\fR \fITEXT\fR +Exact capture from tests history. (required) +.TP +\fB\-\-repo\fR \fITEXT\fR +Repository (owner/name or rep-ID). Defaults to the git remote. +.TP +\fB\-\-org\fR \fITEXT\fR +Organization ID or slug. Defaults to the configured organization. +.TP +\fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR +Occurrences to read (1-500). (default: 50) +.TP +\fB\-\-json\fR \fITEXT\fR +Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. +.TP +\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR +Filter --json output through a jq expression. +.SH SEE ALSO +.BR avr (1), +.BR avr-tests (1), +.BR avr-tests-captures (1), +.BR avr-tests-history (1), +.BR avr-tests-list (1), +.BR avr-tests-status (1) diff --git a/man/man1/avr-tests.1 b/man/man1/avr-tests.1 new file mode 100644 index 00000000..a1e92f74 --- /dev/null +++ b/man/man1/avr-tests.1 @@ -0,0 +1,51 @@ +.TH AVR\-TESTS 1 "" "avr 0.4.1" "Avrea CLI" +.SH NAME +avr-tests \- Inspect recorded tests, failures, and reporting status. +.SH SYNOPSIS +.B avr tests +[OPTIONS] COMMAND [ARGS]... +.SH DESCRIPTION +Inspect recorded tests, failures, and reporting status. +.PP +.SH COMMANDS +.TP +captures +List report deliveries, including missing, partial and failed captures. +.TP +history +Read recorded outcomes for one test and configuration, newest first. +.TP +list +List one page of test histories, grouped by test and configuration. +.TP +status +Read collection settings. +.TP +view +Read the exact retained result, including available failure messages. +.SH SEE ALSO +.BR avr (1), +.BR avr-status (1), +.BR avr-run (1), +.BR avr-pr (1), +.BR avr-job (1), +.BR avr-vm (1), +.BR avr-workflow (1), +.BR avr-cache (1), +.BR avr-sbom (1), +.BR avr-log (1), +.BR avr-auth (1), +.BR avr-token (1), +.BR avr-config (1), +.BR avr-settings (1), +.BR avr-firewall (1), +.BR avr-billing (1), +.BR avr-audit-events (1), +.BR avr-repo (1), +.BR avr-org (1), +.BR avr-health (1), +.BR avr-tests-captures (1), +.BR avr-tests-history (1), +.BR avr-tests-list (1), +.BR avr-tests-status (1), +.BR avr-tests-view (1) diff --git a/man/man1/avr-token.1 b/man/man1/avr-token.1 index d3aa186b..e8d92781 100644 --- a/man/man1/avr-token.1 +++ b/man/man1/avr-token.1 @@ -31,6 +31,7 @@ Show a token's details and grants, including a revoked or expired one. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-vm.1 b/man/man1/avr-vm.1 index 018e6410..fab7ad48 100644 --- a/man/man1/avr-vm.1 +++ b/man/man1/avr-vm.1 @@ -65,6 +65,7 @@ Open a VNC desktop on a macOS VM (Screen Sharing) over an SSH tunnel. .BR avr-job (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-workflow.1 b/man/man1/avr-workflow.1 index f8e51354..f5ade8c8 100644 --- a/man/man1/avr-workflow.1 +++ b/man/man1/avr-workflow.1 @@ -26,6 +26,7 @@ View a workflow with aggregate stats and per-job breakdown. .BR avr-job (1), .BR avr-vm (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr.1 b/man/man1/avr.1 index 128f2ef0..87509991 100644 --- a/man/man1/avr.1 +++ b/man/man1/avr.1 @@ -46,6 +46,9 @@ List and view workflow definitions. cache Inspect and manage the Avrea build cache. .TP +tests +Inspect recorded tests, failures, and reporting status. +.TP sbom View, generate, and download repository SBOMs. .TP @@ -90,6 +93,7 @@ Check Avrea platform status. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/tests/test_tests.py b/tests/test_tests.py new file mode 100644 index 00000000..9b2aed82 --- /dev/null +++ b/tests/test_tests.py @@ -0,0 +1,132 @@ +"""Customer test-evidence commands preserve scope and bounded API responses.""" + +from avrea_cli.main import cli +from click.testing import CliRunner +from tests.conftest import FakeApi +from typing import Any +import json +import pytest + +ORG = "org-019a0000000070008000000000000001" +REPO = "rep-019a0000000070008000000000000002" +TEST = "a" * 64 +CONFIG = "b" * 64 +CAPTURE = "tec-019a0000000070008000000000000003" +BASE = f"/orgs/{ORG}/repos/{REPO}/tests" +SCOPE = ["--org", ORG, "--repo", REPO] +PAGE: dict[str, Any] = { + "status": "available", + "pending": True, + "reason": None, + "as_of_revision": 1, + "current_revision": 2, + "data": [{"test_id": TEST, "configuration_digest": CONFIG, "failed_count": 1, "passed_count": 2}], + "pagination": {"next_cursor": "c" * 900}, +} + + +def test_list_preserves_page_status_and_context(runner: CliRunner, api: FakeApi) -> None: + api.reply("GET", f"{BASE}/history", json=PAGE) + result = runner.invoke(cli, ["tests", "list", *SCOPE, "--limit", "7", "--cursor", "c" * 900, "--json", "*"]) + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["organization_id"] == ORG + assert response["repository_id"] == REPO + assert response["data"] == PAGE["data"] + assert response["pending"] is True + assert response["pagination"] == PAGE["pagination"] + assert len(api.requests) == 1 + assert dict(api.requests[0].url.params) == {"limit": "7", "cursor": "c" * 900} + + +def test_history_passes_exact_configuration(runner: CliRunner, api: FakeApi) -> None: + api.reply("GET", f"{BASE}/history/{TEST}", json={**PAGE, "data": [{"capture_id": CAPTURE}]}) + result = runner.invoke(cli, ["tests", "history", TEST, "--configuration", CONFIG, *SCOPE, "--json", "*"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["configuration_digest"] == CONFIG + assert api.requests[0].url.params["configuration"] == CONFIG + + +@pytest.mark.parametrize("state", ["available", "expired", "not_recorded"]) +def test_view_keeps_exact_capture_without_fallback(runner: CliRunner, api: FakeApi, state: str) -> None: + rows = ( + [{"name": "test_checkout", "outcome": "failed", "failure_summary": "expected 200, got 500"}] + if state == "available" + else [] + ) + api.reply( + "GET", + f"{BASE}/history/{TEST}/detail/{CAPTURE}", + json={"status": state, "data": rows, "truncated": state == "available"}, + ) + result = runner.invoke(cli, ["tests", "view", TEST, "--capture", CAPTURE, *SCOPE, "--limit", "500", "--json", "*"]) + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert (response["test_id"], response["capture_id"], response["status"]) == (TEST, CAPTURE, state) + assert response["data"] == rows + assert response["truncated"] == (state == "available") + assert len(api.requests) == 1 + + +@pytest.mark.parametrize("status", [403, 404, 409, 503]) +def test_denied_or_failed_reads_are_not_empty_success(runner: CliRunner, api: FakeApi, status: int) -> None: + api.reply("GET", f"{BASE}/history", status=status, json={"detail": "Evidence is unavailable"}) + result = runner.invoke(cli, ["tests", "list", *SCOPE, "--json", "*"]) + assert result.exit_code != 0 + assert len(api.requests) == 1 + assert "Evidence is unavailable" in result.output + + +def test_status_reads_control_without_mutation(runner: CliRunner, api: FakeApi) -> None: + state = {"capture_enabled": False, "can_manage": True, "repository_generation": 2, "purges": [], "events": []} + api.reply("GET", f"{BASE}/control", json=state) + result = runner.invoke(cli, ["tests", "status", *SCOPE, "--json", "*"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data"] == state + assert len(api.requests) == 1 + + +def test_captures_preserves_missing_report_reason(runner: CliRunner, api: FakeApi) -> None: + rows = [{"capture_id": CAPTURE, "state": "no_report", "reason": "no_supported_report"}] + api.reply("GET", f"{BASE}/captures", json={"data": rows, "pagination": {"next_cursor": None}}) + result = runner.invoke(cli, ["tests", "captures", *SCOPE, "--json", "*"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data"] == rows + + +def test_name_terminal_controls_are_escaped(runner: CliRunner, api: FakeApi) -> None: + row = {**PAGE["data"][0], "identity": {"name": "test_\x1b[31mcheckout"}} + api.reply("GET", f"{BASE}/history", json={**PAGE, "data": [row]}) + result = runner.invoke(cli, ["tests", "list", *SCOPE]) + assert result.exit_code == 0, result.output + assert "\x1b" not in result.output + assert "checkout" in result.output + + +@pytest.mark.parametrize( + "args", + [ + ["list", "--cursor", " "], + ["history", "bad", "--configuration", CONFIG], + ["view", TEST, "--capture", "bad"], + ["list", "--json", "no_such_field"], + ], +) +def test_bad_arguments_make_no_requests(runner: CliRunner, api: FakeApi, args: list[str]) -> None: + result = runner.invoke(cli, ["tests", *args, *SCOPE]) + assert result.exit_code != 0 + assert api.requests == [] + + +def test_empty_capture_page_and_partial_detail_are_explicit(runner: CliRunner, api: FakeApi) -> None: + api.reply("GET", f"{BASE}/captures", json={"data": [], "pagination": {"next_cursor": None}}) + result = runner.invoke(cli, ["tests", "captures", *SCOPE]) + assert result.exit_code == 0, result.output + assert "No recorded results in this page" in result.output + api.reply( + "GET", f"{BASE}/history/{TEST}/detail/{CAPTURE}", json={"status": "available", "data": [], "truncated": True} + ) + result = runner.invoke(cli, ["tests", "view", TEST, "--capture", CAPTURE, *SCOPE, "--limit", "500"]) + assert result.exit_code == 0, result.output + assert "This result is partial" in result.output + assert "Increase --limit" not in result.output From aa0527cbe900678d941195ff486290e84d85a3e5 Mon Sep 17 00:00:00 2001 From: Hannu Varjoranta Date: Thu, 22 Oct 2026 23:14:22 +0300 Subject: [PATCH 2/7] cli: explain unavailable test durations --- avrea_cli/commands/tests.py | 34 ++++++++++++++++------------------ tests/test_tests.py | 9 +++++++++ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/avrea_cli/commands/tests.py b/avrea_cli/commands/tests.py index 6b4158ce..b89487d3 100644 --- a/avrea_cli/commands/tests.py +++ b/avrea_cli/commands/tests.py @@ -119,28 +119,26 @@ def _show(response: dict[str, Any], json_fields: str | None, jq_expr: str | None column_labels=["Test", "Test ID", "Configuration", "Failed", "Errors", "Passed"], ) else: - columns = [ - key - for key in ( - "capture_id", - "captured_at", - "armed_at", - "state", - "detail_status", - "commit_sha", - "reason", - "passed_count", - "failed_count", - "errored_count", - "duration_total_ms", - ) - if key in rows[0] - ] + labels = { + "capture_id": "Capture", + "captured_at": "Captured at", + "armed_at": "Armed at", + "state": "Delivery", + "detail_status": "Detail", + "commit_sha": "Commit", + "reason": "Reason", + "passed_count": "Passed", + "failed_count": "Failed", + "errored_count": "Errors", + "duration_total_ms": "Test time (ms)", + "duration_status": "Duration status", + } + columns = [key for key in labels if key in rows[0]] display_rows = [ {key: "-" if value is None else escape_control_characters(str(value)) for key, value in row.items()} for row in rows ] - output_list(display_rows, columns=columns, column_labels=columns) + output_list(display_rows, columns=columns, column_labels=[labels[key] for key in columns]) else: click.echo("No recorded results in this page. This does not establish that no tests ran.") next_cursor = (response.get("pagination") or {}).get("next_cursor") diff --git a/tests/test_tests.py b/tests/test_tests.py index 9b2aed82..4b86775d 100644 --- a/tests/test_tests.py +++ b/tests/test_tests.py @@ -130,3 +130,12 @@ def test_empty_capture_page_and_partial_detail_are_explicit(runner: CliRunner, a assert result.exit_code == 0, result.output assert "This result is partial" in result.output assert "Increase --limit" not in result.output + + +@pytest.mark.parametrize("duration_status", ["expired_detail", "unavailable"]) +def test_history_text_explains_missing_duration(runner: CliRunner, api: FakeApi, duration_status: str) -> None: + row = {"capture_id": CAPTURE, "duration_total_ms": None, "duration_status": duration_status} + api.reply("GET", f"{BASE}/history/{TEST}", json={**PAGE, "data": [row]}) + result = runner.invoke(cli, ["tests", "history", TEST, "--configuration", CONFIG, *SCOPE]) + assert result.exit_code == 0, result.output + assert duration_status in result.output From fc046028cdcfb038208c3e55cd6e309a9dbc3bd5 Mon Sep 17 00:00:00 2001 From: Hannu Varjoranta Date: Wed, 28 Oct 2026 16:06:55 +0200 Subject: [PATCH 3/7] cli: replace evidence navigation with job test results --- README.md | 21 ++ avrea_cli/commands/tests.py | 389 +++++++++------------ avrea_cli/main.py | 2 +- docs/REFERENCE.md | 103 +----- docs/reference.json | 539 ++---------------------------- docs/starlight/index.md | 21 ++ docs/starlight/reference/index.md | 2 +- docs/starlight/reference/tests.md | 105 ++---- man/man1/avr-tests-captures.1 | 35 -- man/man1/avr-tests-history.1 | 42 --- man/man1/avr-tests-list.1 | 33 +- man/man1/avr-tests-status.1 | 30 -- man/man1/avr-tests-view.1 | 40 --- man/man1/avr-tests.1 | 25 +- man/man1/avr.1 | 2 +- tests/test_tests.py | 284 ++++++++++------ 16 files changed, 476 insertions(+), 1197 deletions(-) delete mode 100644 man/man1/avr-tests-captures.1 delete mode 100644 man/man1/avr-tests-history.1 delete mode 100644 man/man1/avr-tests-status.1 delete mode 100644 man/man1/avr-tests-view.1 diff --git a/README.md b/README.md index 2f92c91b..c35ed07a 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,27 @@ avr run logs # full log dump, paginated avr job logs --follow # tail one job in real time ``` +**Read a job's recorded tests** + +```sh +avr tests # names, outcomes, timings and failure messages +avr tests --output json # results with reporting state and pagination +avr tests --limit 500 # follow short pages up to the requested count +avr tests --cursor '' # continue with pagination.next_cursor +``` + +Use the Avrea job ID from `avr job list`; the repository is resolved from that +job. `avr tests list ` is equivalent. `--json FIELDS` and `--jq` also +work. Each invocation reads at most 1,000 occurrences and 20 result pages, +preserving repeated tests and returning a cursor when more results remain. +Missing or expired reports are distinct from a retained report containing zero +tests. A report marked complete does not establish that every expected test ran. + +This requires an API deployment supporting job test results and access to the +repository's test evidence. Only Avrea job IDs are supported in this first +slice. Provider job IDs/URLs, immutable result links and anchored test history +follow separately; this command does not rerun tests or change collection. + **Trigger a workflow_dispatch** ```sh diff --git a/avrea_cli/commands/tests.py b/avrea_cli/commands/tests.py index b89487d3..df4048d3 100644 --- a/avrea_cli/commands/tests.py +++ b/avrea_cli/commands/tests.py @@ -1,4 +1,4 @@ -"""Read retained test evidence through the customer API.""" +"""Read the retained test results of a job.""" from avrea_cli.api_client import ApiClient from avrea_cli.click_ext import GhGroup @@ -15,281 +15,204 @@ from avrea_cli.json_output import make_schema from avrea_cli.json_output import split_fields from avrea_cli.output import output_list -from avrea_cli.repo_context import resolve_repo_or_detect +from avrea_cli.repo_context import resolve_repo from typing import Any import click import httpx import re +import shlex -_FIELDS = make_schema( - "organization_id", - "repository_id", - "test_id", - "configuration_digest", - "capture_id", +_SNAPSHOT_FIELDS = ( + "job_id", + "execution_id", "status", + "reporting_state", "reason", - "pending", - "as_of_revision", - "current_revision", - "data", - "pagination", - "truncated", + "commit_sha", + "observed_at", + "total_count", ) -_CONTROL_FIELDS = make_schema("organization_id", "repository_id", "data") -_repo = click.option("--repo", "repo_id", help="Repository (owner/name or rep-ID). Defaults to the git remote.") -_org = click.option("--org", "org_id", help="Organization ID or slug. Defaults to the configured organization.") -_limit = click.option( - "-L", "--limit", type=click.IntRange(1, 100), default=50, show_default=True, help="Rows per page (1-100)." -) -_cursor = click.option("--cursor", help="Opaque next_cursor from the previous page. One page is read per invocation.") - +_FIELDS = make_schema("organization_id", "repository_id", *_SNAPSHOT_FIELDS, "data", "pagination") +_MAX_PAGES = 20 -def _digest(_ctx: click.Context, param: click.Parameter, value: str) -> str: - if not re.fullmatch(r"[0-9a-f]{64}", value): - raise click.BadParameter("must be a 64-character lowercase hex identifier", param=param) - return value +class JobTestsGroup(GhGroup): + """Accept the documented job shorthand without swallowing unknown verbs.""" -def _capture(_ctx: click.Context, param: click.Parameter, value: str) -> str: - if not re.fullmatch(r"tec-[0-9a-f]{32}", value): - raise click.BadParameter("must be a tec- identifier followed by 32 lowercase hex characters", param=param) - return value + def resolve_command( + self, ctx: click.Context, args: list[str] + ) -> tuple[str | None, click.Command | None, list[str]]: + if args and args[0].startswith("job-"): + args = ["list", *args] + return super().resolve_command(ctx, args) -def _page_params(limit: int, cursor: str | None) -> dict[str, Any]: - cursor = validate_cursor(cursor, max_length=8192) - return {"limit": limit, **({"cursor": cursor} if cursor is not None else {})} +def _job_id(_ctx: click.Context, param: click.Parameter, value: str) -> str: + if not re.fullmatch(r"job-[0-9a-f]{32}", value): + raise click.BadParameter( + "use the Avrea job ID from 'avr job list' (job- followed by 32 lowercase hex characters)", param=param + ) + return value -def _read( - ctx: click.Context, - org_id: str | None, - repo_id: str | None, - suffix: str, - params: dict[str, Any] | None = None, -) -> dict[str, Any]: - client: ApiClient = ctx.obj["client"] - config: CliConfig = ctx.obj["config"] - ensure_authenticated(config) - organization = get_org_id(config, org_id, client=client) - repository = resolve_repo_or_detect(client, config, organization, repo_id, required=True) +def _get(client: ApiClient, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: try: - response = client.public_get(f"/orgs/{organization}/repos/{repository}/tests/{suffix}", params=params) + return client.public_get(path, params=params) except httpx.HTTPStatusError as exc: - handle_http_error(exc, "read test evidence") - return {**response, "organization_id": organization, "repository_id": repository} - - -def _show(response: dict[str, Any], json_fields: str | None, jq_expr: str | None, *, detail: bool = False) -> None: - if json_fields is not None: - emit_json_record(response, split_fields(json_fields, _FIELDS), _FIELDS, jq_expr) - return - click.echo(f"Organization: {response['organization_id']} Repository: {response['repository_id']}") - for field in ("test_id", "configuration_digest", "capture_id", "status", "reason"): - if response.get(field) is not None: - click.echo(f"{field}: {escape_control_characters(str(response[field]))}") - if response.get("pending"): - click.echo("New results are still being indexed.") - if response.get("truncated"): - click.echo( - "This result is partial. The detail read returns at most 500 occurrences; " - "--limit controls the requested count." + handle_http_error(exc, "read job test results") + + +def _job_repository(client: ApiClient, organization: str, job: str) -> str: + response = _get(client, f"/orgs/{organization}/jobs/{job}") + data = response.get("data", response) + repository = data.get("repository_id") if isinstance(data, dict) else None + if not isinstance(repository, str) or not re.fullmatch(r"rep-[0-9a-f]{32}", repository): + raise click.ClickException("The job has no available repository. Check the job and selected organization.") + if data.get("job_id") != job: + raise click.ClickException("The API returned a different job. Retry with the exact Avrea job ID.") + return repository + + +def _results(client: ApiClient, path: str, job: str, limit: int, cursor: str | None) -> dict[str, Any]: + """Follow short pages within a row and request budget, retaining the resume cursor.""" + rows: list[dict[str, Any]] = [] + snapshot: dict[str, Any] | None = None + seen = {cursor} if cursor else set() + for _ in range(_MAX_PAGES): + page_limit = min(100, limit - len(rows)) + params: dict[str, Any] = {"limit": page_limit} + if cursor is not None: + params["cursor"] = cursor + page = _get(client, path, params) + data = page.get("data") + pagination = page.get("pagination") + if page.get("job_id") != job or not isinstance(data, list) or not isinstance(pagination, dict): + raise click.ClickException("The API returned invalid job test results.") + if len(data) > page_limit or not all(isinstance(row, dict) for row in data): + raise click.ClickException("The API returned an invalid test result page.") + if snapshot is None: + snapshot = page + elif any(page.get(key) != snapshot.get(key) for key in _SNAPSHOT_FIELDS): + raise click.ClickException("Job results changed while reading pages. Retry without --cursor.") + cursor = pagination.get("next_cursor") + if cursor is not None: + if not isinstance(cursor, str): + raise click.ClickException("The API returned an invalid next cursor.") + validate_cursor(cursor, max_length=1024) + if cursor in seen: + raise click.ClickException("The API repeated a test-results cursor; stopping pagination.") + seen.add(cursor) + rows.extend(data) + if cursor is None or len(rows) == limit: + break + assert snapshot is not None + return {**snapshot, "data": rows, "pagination": {"next_cursor": cursor}} + + +def _text(value: Any) -> str: + return "unknown" if value is None else escape_control_characters(str(value)) + + +def _show(response: dict[str, Any]) -> None: + click.echo(f"Job: {_text(response['job_id'])}") + click.echo(f"Organization: {_text(response['organization_id'])} Repository: {_text(response['repository_id'])}") + click.echo(f"Execution: {_text(response.get('execution_id'))} Commit: {_text(response.get('commit_sha'))}") + click.echo(f"Reporting: {_text(response.get('reporting_state'))} Results: {_text(response.get('status'))}") + if response.get("reason"): + click.echo(f"Reason: {_text(response['reason'])}") + click.echo(f"Retained occurrences: {_text(response.get('total_count'))}; shown: {len(response['data'])}") + rows = response["data"] + if rows: + columns = ("suite", "class_name", "name", "outcome", "duration_ms") + output_list( + [{key: _text(row.get(key)) for key in columns} for row in rows], + columns=columns, + column_labels=["Suite", "Class", "Test", "Outcome", "Time (ms)"], ) - rows = response.get("data") or [] - if detail: for row in rows: - click.echo() - for key in ("suite", "class_name", "name", "outcome", "duration_ms", "failure_summary"): - if row.get(key) is not None: - click.echo(f"{key}: {escape_control_characters(str(row[key]))}") - elif rows: - if "test_id" in rows[0]: - display_rows = [{**row, "name": (row.get("identity") or {}).get("name") or row["test_id"]} for row in rows] - display_rows = [ - { - key: escape_control_characters(str(value)) if isinstance(value, str) else value - for key, value in row.items() - } - for row in display_rows - ] - output_list( - display_rows, - columns=["name", "test_id", "configuration_digest", "failed_count", "errored_count", "passed_count"], - column_labels=["Test", "Test ID", "Configuration", "Failed", "Errors", "Passed"], - ) - else: - labels = { - "capture_id": "Capture", - "captured_at": "Captured at", - "armed_at": "Armed at", - "state": "Delivery", - "detail_status": "Detail", - "commit_sha": "Commit", - "reason": "Reason", - "passed_count": "Passed", - "failed_count": "Failed", - "errored_count": "Errors", - "duration_total_ms": "Test time (ms)", - "duration_status": "Duration status", - } - columns = [key for key in labels if key in rows[0]] - display_rows = [ - {key: "-" if value is None else escape_control_characters(str(value)) for key, value in row.items()} - for row in rows - ] - output_list(display_rows, columns=columns, column_labels=[labels[key] for key in columns]) + if row.get("failure_summary"): + click.echo(f"\n{_text(row.get('name'))}: {_text(row['failure_summary'])}") + elif response.get("status") == "available" and response.get("total_count") == 0: + click.echo("The retained report contains zero test occurrences. This does not establish that no tests ran.") else: - click.echo("No recorded results in this page. This does not establish that no tests ran.") - next_cursor = (response.get("pagination") or {}).get("next_cursor") - if next_cursor: - click.echo(f"Next page: --cursor {escape_control_characters(str(next_cursor))}", err=True) + click.echo("No test results returned. Missing or expired reports do not mean that zero tests ran.") + if response.get("reporting_state") == "complete": + click.echo("Complete describes the collected reports, not whether every expected test ran.") -@click.group(cls=GhGroup) +@click.group(cls=JobTestsGroup) @click.pass_context def tests(ctx: click.Context) -> None: - """Inspect recorded tests, failures, and reporting status.""" - ensure_ctx(ctx) - + """Read a job's tests, timings and recorded failure messages. -@tests.command("list") -@_repo -@_org -@_limit -@_cursor -@json_options -@click.pass_context -def tests_list( - ctx: click.Context, - repo_id: str | None, - org_id: str | None, - limit: int, - cursor: str | None, - json_fields: str | None, - jq_expr: str | None, -) -> None: - """List one page of test histories, grouped by test and configuration. - - Counts describe each returned history window, not repository totals. - Names appear when retained private detail is available; otherwise IDs are shown. - Use --json '*' to retain status, pagination and repository context for agents. + Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from + avr job list. No report capture ID or configuration digest is required. """ - if handle_json_meta(json_fields, jq_expr, _FIELDS): - return - _show(_read(ctx, org_id, repo_id, "history", _page_params(limit, cursor)), json_fields, jq_expr) + ensure_ctx(ctx) -@tests.command("history") -@click.argument("test_id", callback=_digest) +@tests.command("list") +@click.argument("job", metavar="JOB", callback=_job_id) +@click.option("--org", "org_id", help="Organization ID or slug. Defaults to the configured organization.") +@click.option("--repo", "repo_id", help="Repository (owner/name or rep-ID). Defaults to the job's repository.") @click.option( - "--configuration", - "configuration_digest", - required=True, - callback=_digest, - help="History grouping identifier returned by tests list; not a platform label.", + "-L", + "--limit", + type=click.IntRange(1, 1000), + default=100, + show_default=True, + help=f"Maximum occurrences to return; short pages are followed, up to {_MAX_PAGES} result pages.", ) -@_repo -@_org -@_limit -@_cursor -@json_options -@click.pass_context -def tests_history( - ctx: click.Context, - test_id: str, - configuration_digest: str, - repo_id: str | None, - org_id: str | None, - limit: int, - cursor: str | None, - json_fields: str | None, - jq_expr: str | None, -) -> None: - """Read recorded outcomes for one test and configuration, newest first.""" - if handle_json_meta(json_fields, jq_expr, _FIELDS): - return - params = {**_page_params(limit, cursor), "configuration": configuration_digest} - response = _read(ctx, org_id, repo_id, f"history/{test_id}", params) - response.update(test_id=test_id, configuration_digest=configuration_digest) - _show(response, json_fields, jq_expr) - - -@tests.command("view") -@click.argument("test_id", callback=_digest) -@click.option("--capture", "capture_id", required=True, callback=_capture, help="Exact capture from tests history.") -@_repo -@_org +@click.option("--cursor", help="Continue from a previous response's next_cursor.") @click.option( - "-L", "--limit", type=click.IntRange(1, 500), default=50, show_default=True, help="Occurrences to read (1-500)." + "--output", + type=click.Choice(["text", "json"]), + help="Output format. JSON includes status, context and the next cursor.", ) @json_options @click.pass_context -def tests_view( +def tests_list( ctx: click.Context, - test_id: str, - capture_id: str, - repo_id: str | None, + job: str, org_id: str | None, - limit: int, - json_fields: str | None, - jq_expr: str | None, -) -> None: - """Read the exact retained result, including available failure messages. - - An expired or unrecorded result is never replaced with a different capture. - A recorded test failure is data, not a CLI error. No tests are rerun. - """ - if handle_json_meta(json_fields, jq_expr, _FIELDS): - return - response = _read(ctx, org_id, repo_id, f"history/{test_id}/detail/{capture_id}", {"limit": limit}) - response.update(test_id=test_id, capture_id=capture_id) - _show(response, json_fields, jq_expr, detail=True) - - -@tests.command("captures") -@_repo -@_org -@_limit -@_cursor -@json_options -@click.pass_context -def tests_captures( - ctx: click.Context, repo_id: str | None, - org_id: str | None, limit: int, cursor: str | None, + output: str | None, json_fields: str | None, jq_expr: str | None, ) -> None: - """List report deliveries, including missing, partial and failed captures.""" - if handle_json_meta(json_fields, jq_expr, _FIELDS): - return - _show(_read(ctx, org_id, repo_id, "captures", _page_params(limit, cursor)), json_fields, jq_expr) + """Read the latest confirmed execution of JOB, including repeated tests. + Names, outcomes, recorded durations and available failure messages are + shown together. A failed test is data, not a command error. Missing, + pending, partial and expired evidence are reported explicitly; an older + execution is never substituted. Complete describes collected reports. -@tests.command("status") -@_repo -@_org -@json_options -@click.pass_context -def tests_status( - ctx: click.Context, repo_id: str | None, org_id: str | None, json_fields: str | None, jq_expr: str | None -) -> None: - """Read collection settings. Use --json for retained-history lifecycle details.""" - if handle_json_meta(json_fields, jq_expr, _CONTROL_FIELDS): + Use --output json for the complete response or --json FIELDS to select + fields. Continue with --cursor when pagination.next_cursor is present. + API refusal or a changed report fails the command without partial output. + """ + if output == "text" and json_fields is not None: + raise click.UsageError("--output text cannot be combined with --json") + if output == "json" and json_fields is None: + json_fields = "*" + if handle_json_meta(json_fields, jq_expr, _FIELDS): return - result = _read(ctx, org_id, repo_id, "control") - response = { - "organization_id": result.pop("organization_id"), - "repository_id": result.pop("repository_id"), - "data": result, - } + cursor = validate_cursor(cursor, max_length=1024) + client: ApiClient = ctx.obj["client"] + config: CliConfig = ctx.obj["config"] + ensure_authenticated(config) + organization = get_org_id(config, org_id, client=client) + repository = ( + resolve_repo(client, config, organization, repo_id) if repo_id else _job_repository(client, organization, job) + ) + response = _results(client, f"/orgs/{organization}/repos/{repository}/jobs/{job}/tests", job, limit, cursor) + response.update(organization_id=organization, repository_id=repository) if json_fields is not None: - emit_json_record(response, split_fields(json_fields, _CONTROL_FIELDS), _CONTROL_FIELDS, jq_expr) - return - click.echo(f"Organization: {response['organization_id']} Repository: {response['repository_id']}") - click.echo(f"Collection: {'on' if result['capture_enabled'] else 'off'}") - click.echo(f"Can manage collection: {'yes' if result['can_manage'] else 'no'}") - click.echo("Use 'avr tests captures' to inspect delivery and missing-report evidence.") + emit_json_record(response, split_fields(json_fields, _FIELDS), _FIELDS, jq_expr) + else: + _show(response) + next_cursor = response["pagination"]["next_cursor"] + if next_cursor: + click.echo(f"More results available. Continue with --cursor {shlex.quote(_text(next_cursor))}", err=True) diff --git a/avrea_cli/main.py b/avrea_cli/main.py index b70129c0..93c954b6 100644 --- a/avrea_cli/main.py +++ b/avrea_cli/main.py @@ -42,7 +42,7 @@ "tests", "avrea_cli.commands.tests", "tests", - "Inspect recorded tests, failures, and reporting status.", + "Read a job's tests, timings and recorded failure messages.", "Core Commands", ), ("sbom", "avrea_cli.commands.sbom", "sbom", "View, generate, and download repository SBOMs.", "Core Commands"), diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index d1bf291b..c96088aa 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -42,7 +42,7 @@ avr [GLOBAL OPTIONS] COMMAND [ARGS]... - [`avr vm`](#avr-vm) — Manage long-running VMs (SSH/RDP/VNC). - [`avr workflow`](#avr-workflow) — List and view workflow definitions. - [`avr cache`](#avr-cache) — Inspect and manage the Avrea build cache. -- [`avr tests`](#avr-tests) — Inspect recorded tests, failures, and reporting status. +- [`avr tests`](#avr-tests) — Read a job's tests, timings and recorded failure messages. - [`avr sbom`](#avr-sbom) — View, generate, and download repository SBOMs. - [`avr log`](#avr-log) — Search across runner execution logs. @@ -1306,110 +1306,43 @@ JSON FIELDS ### `avr tests` -Inspect recorded tests, failures, and reporting status. +Read a job's tests, timings and recorded failure messages. ```sh avr tests [OPTIONS] COMMAND [ARGS]... ``` -#### `avr tests captures` - -List report deliveries, including missing, partial and failed captures. - -```sh -avr tests captures [OPTIONS] -``` - -**Options** - -- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. -- `--org ` — Organization ID or slug. Defaults to the configured organization. -- `-L, --limit ` — Rows per page (1-100). _(default: `50`)_ -- `--cursor ` — Opaque next_cursor from the previous page. One page is read per invocation. -- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- `-q, --jq ` — Filter --json output through a jq expression. - -#### `avr tests history` - -Read recorded outcomes for one test and configuration, newest first. - -```sh -avr tests history [OPTIONS] TEST_ID -``` - -**Arguments** - -- `TEST_ID` - -**Options** - -- `--configuration ` — History grouping identifier returned by tests list; not a platform label. _(required)_ -- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. -- `--org ` — Organization ID or slug. Defaults to the configured organization. -- `-L, --limit ` — Rows per page (1-100). _(default: `50`)_ -- `--cursor ` — Opaque next_cursor from the previous page. One page is read per invocation. -- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- `-q, --jq ` — Filter --json output through a jq expression. +Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from +avr job list. No report capture ID or configuration digest is required. #### `avr tests list` -List one page of test histories, grouped by test and configuration. +Read the latest confirmed execution of JOB, including repeated tests. ```sh -avr tests list [OPTIONS] +avr tests list [OPTIONS] JOB ``` -Counts describe each returned history window, not repository totals. -Names appear when retained private detail is available; otherwise IDs are shown. -Use --json '*' to retain status, pagination and repository context for agents. - -**Options** - -- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. -- `--org ` — Organization ID or slug. Defaults to the configured organization. -- `-L, --limit ` — Rows per page (1-100). _(default: `50`)_ -- `--cursor ` — Opaque next_cursor from the previous page. One page is read per invocation. -- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- `-q, --jq ` — Filter --json output through a jq expression. - -#### `avr tests status` - -Read collection settings. - -```sh -avr tests status [OPTIONS] -``` - -Read collection settings. Use --json for retained-history lifecycle details. - -**Options** - -- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. -- `--org ` — Organization ID or slug. Defaults to the configured organization. -- `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- `-q, --jq ` — Filter --json output through a jq expression. - -#### `avr tests view` - -Read the exact retained result, including available failure messages. - -```sh -avr tests view [OPTIONS] TEST_ID -``` +Names, outcomes, recorded durations and available failure messages are +shown together. A failed test is data, not a command error. Missing, +pending, partial and expired evidence are reported explicitly; an older +execution is never substituted. Complete describes collected reports. -An expired or unrecorded result is never replaced with a different capture. -A recorded test failure is data, not a CLI error. No tests are rerun. +Use --output json for the complete response or --json FIELDS to select +fields. Continue with --cursor when pagination.next_cursor is present. +API refusal or a changed report fails the command without partial output. **Arguments** -- `TEST_ID` +- `JOB` **Options** -- `--capture ` — Exact capture from tests history. _(required)_ -- `--repo ` — Repository (owner/name or rep-ID). Defaults to the git remote. - `--org ` — Organization ID or slug. Defaults to the configured organization. -- `-L, --limit ` — Occurrences to read (1-500). _(default: `50`)_ +- `--repo ` — Repository (owner/name or rep-ID). Defaults to the job's repository. +- `-L, --limit ` — Maximum occurrences to return; short pages are followed, up to 20 result pages. _(default: `100`)_ +- `--cursor ` — Continue from a previous response's next_cursor. +- `--output ` — Output format. JSON includes status, context and the next cursor. _(choices: `text`, `json`)_ - `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. - `-q, --jq ` — Filter --json output through a jq expression. diff --git a/docs/reference.json b/docs/reference.json index aaf82f99..64bd1ac6 100644 --- a/docs/reference.json +++ b/docs/reference.json @@ -6256,8 +6256,8 @@ ], "full": "avr tests", "is_group": true, - "short_help": "Inspect recorded tests, failures, and reporting status.", - "help": "Inspect recorded tests, failures, and reporting status.", + "short_help": "Read a job's tests, timings and recorded failure messages.", + "help": "Read a job's tests, timings and recorded failure messages.\n\nUse avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from\navr job list. No report capture ID or configuration digest is required.\n", "epilog": null, "deprecated": false, "hidden": false, @@ -6265,155 +6265,26 @@ "params": [], "subcommands": [ { - "name": "captures", - "path": [ - "avr", - "tests", - "captures" - ], - "full": "avr tests captures", - "is_group": false, - "short_help": "List report deliveries, including missing, partial and failed captures.", - "help": "List report deliveries, including missing, partial and failed captures.", - "epilog": null, - "deprecated": false, - "hidden": false, - "usage": "avr tests captures [OPTIONS]", - "params": [ - { - "name": "repo_id", - "kind": "option", - "opts": [ - "--repo" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "org_id", - "kind": "option", - "opts": [ - "--org" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Organization ID or slug. Defaults to the configured organization.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "limit", - "kind": "option", - "opts": [ - "-L", - "--limit" - ], - "secondary_opts": [], - "type": "integer range", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Rows per page (1-100).", - "hidden": false, - "show_default": true, - "default": 50 - }, - { - "name": "cursor", - "kind": "option", - "opts": [ - "--cursor" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Opaque next_cursor from the previous page. One page is read per invocation.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "json_fields", - "kind": "option", - "opts": [ - "--json" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "jq_expr", - "kind": "option", - "opts": [ - "-q", - "--jq" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Filter --json output through a jq expression.", - "hidden": false, - "show_default": false, - "default": null - } - ], - "subcommands": [] - }, - { - "name": "history", + "name": "list", "path": [ "avr", "tests", - "history" + "list" ], - "full": "avr tests history", + "full": "avr tests list", "is_group": false, - "short_help": "Read recorded outcomes for one test and configuration, newest first.", - "help": "Read recorded outcomes for one test and configuration, newest first.", + "short_help": "Read the latest confirmed execution of JOB, including repeated tests.", + "help": "Read the latest confirmed execution of JOB, including repeated tests.\n\nNames, outcomes, recorded durations and available failure messages are\nshown together. A failed test is data, not a command error. Missing,\npending, partial and expired evidence are reported explicitly; an older\nexecution is never substituted. Complete describes collected reports.\n\nUse --output json for the complete response or --json FIELDS to select\nfields. Continue with --cursor when pagination.next_cursor is present.\nAPI refusal or a changed report fails the command without partial output.\n", "epilog": null, "deprecated": false, "hidden": false, - "usage": "avr tests history [OPTIONS] TEST_ID", + "usage": "avr tests list [OPTIONS] JOB", "params": [ { - "name": "test_id", + "name": "job", "kind": "argument", "opts": [ - "test_id" + "job" ], "secondary_opts": [], "type": "text", @@ -6427,42 +6298,6 @@ "show_default": false, "default": null }, - { - "name": "configuration_digest", - "kind": "option", - "opts": [ - "--configuration" - ], - "secondary_opts": [], - "type": "text", - "required": true, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "History grouping identifier returned by tests list; not a platform label.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "repo_id", - "kind": "option", - "opts": [ - "--repo" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", - "hidden": false, - "show_default": false, - "default": null - }, { "name": "org_id", "kind": "option", @@ -6481,99 +6316,6 @@ "show_default": false, "default": null }, - { - "name": "limit", - "kind": "option", - "opts": [ - "-L", - "--limit" - ], - "secondary_opts": [], - "type": "integer range", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Rows per page (1-100).", - "hidden": false, - "show_default": true, - "default": 50 - }, - { - "name": "cursor", - "kind": "option", - "opts": [ - "--cursor" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Opaque next_cursor from the previous page. One page is read per invocation.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "json_fields", - "kind": "option", - "opts": [ - "--json" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "jq_expr", - "kind": "option", - "opts": [ - "-q", - "--jq" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Filter --json output through a jq expression.", - "hidden": false, - "show_default": false, - "default": null - } - ], - "subcommands": [] - }, - { - "name": "list", - "path": [ - "avr", - "tests", - "list" - ], - "full": "avr tests list", - "is_group": false, - "short_help": "List one page of test histories, grouped by test and configuration.", - "help": "List one page of test histories, grouped by test and configuration.\n\nCounts describe each returned history window, not repository totals.\nNames appear when retained private detail is available; otherwise IDs are shown.\nUse --json '*' to retain status, pagination and repository context for agents.\n", - "epilog": null, - "deprecated": false, - "hidden": false, - "usage": "avr tests list [OPTIONS]", - "params": [ { "name": "repo_id", "kind": "option", @@ -6587,25 +6329,7 @@ "nargs": 1, "is_flag": false, "envvar": null, - "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "org_id", - "kind": "option", - "opts": [ - "--org" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Organization ID or slug. Defaults to the configured organization.", + "help": "Repository (owner/name or rep-ID). Defaults to the job's repository.", "hidden": false, "show_default": false, "default": null @@ -6624,10 +6348,10 @@ "nargs": 1, "is_flag": false, "envvar": null, - "help": "Rows per page (1-100).", + "help": "Maximum occurrences to return; short pages are followed, up to 20 result pages.", "hidden": false, "show_default": true, - "default": 50 + "default": 100 }, { "name": "cursor", @@ -6642,249 +6366,32 @@ "nargs": 1, "is_flag": false, "envvar": null, - "help": "Opaque next_cursor from the previous page. One page is read per invocation.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "json_fields", - "kind": "option", - "opts": [ - "--json" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "jq_expr", - "kind": "option", - "opts": [ - "-q", - "--jq" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Filter --json output through a jq expression.", - "hidden": false, - "show_default": false, - "default": null - } - ], - "subcommands": [] - }, - { - "name": "status", - "path": [ - "avr", - "tests", - "status" - ], - "full": "avr tests status", - "is_group": false, - "short_help": "Read collection settings.", - "help": "Read collection settings. Use --json for retained-history lifecycle details.", - "epilog": null, - "deprecated": false, - "hidden": false, - "usage": "avr tests status [OPTIONS]", - "params": [ - { - "name": "repo_id", - "kind": "option", - "opts": [ - "--repo" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "org_id", - "kind": "option", - "opts": [ - "--org" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Organization ID or slug. Defaults to the configured organization.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "json_fields", - "kind": "option", - "opts": [ - "--json" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Output JSON. Pass comma-separated field names, \"*\" for all fields, or \"?\" to list available fields.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "jq_expr", - "kind": "option", - "opts": [ - "-q", - "--jq" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Filter --json output through a jq expression.", - "hidden": false, - "show_default": false, - "default": null - } - ], - "subcommands": [] - }, - { - "name": "view", - "path": [ - "avr", - "tests", - "view" - ], - "full": "avr tests view", - "is_group": false, - "short_help": "Read the exact retained result, including available failure messages.", - "help": "Read the exact retained result, including available failure messages.\n\nAn expired or unrecorded result is never replaced with a different capture.\nA recorded test failure is data, not a CLI error. No tests are rerun.\n", - "epilog": null, - "deprecated": false, - "hidden": false, - "usage": "avr tests view [OPTIONS] TEST_ID", - "params": [ - { - "name": "test_id", - "kind": "argument", - "opts": [ - "test_id" - ], - "secondary_opts": [], - "type": "text", - "required": true, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": null, + "help": "Continue from a previous response's next_cursor.", "hidden": false, "show_default": false, "default": null }, { - "name": "capture_id", + "name": "output", "kind": "option", "opts": [ - "--capture" + "--output" ], "secondary_opts": [], - "type": "text", - "required": true, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Exact capture from tests history.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "repo_id", - "kind": "option", - "opts": [ - "--repo" - ], - "secondary_opts": [], - "type": "text", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Repository (owner/name or rep-ID). Defaults to the git remote.", - "hidden": false, - "show_default": false, - "default": null - }, - { - "name": "org_id", - "kind": "option", - "opts": [ - "--org" - ], - "secondary_opts": [], - "type": "text", + "type": "choice", "required": false, "multiple": false, "nargs": 1, "is_flag": false, "envvar": null, - "help": "Organization ID or slug. Defaults to the configured organization.", + "help": "Output format. JSON includes status, context and the next cursor.", "hidden": false, "show_default": false, - "default": null - }, - { - "name": "limit", - "kind": "option", - "opts": [ - "-L", - "--limit" - ], - "secondary_opts": [], - "type": "integer range", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Occurrences to read (1-500).", - "hidden": false, - "show_default": true, - "default": 50 + "default": null, + "choices": [ + "text", + "json" + ] }, { "name": "json_fields", diff --git a/docs/starlight/index.md b/docs/starlight/index.md index d1a88b87..31ed5924 100644 --- a/docs/starlight/index.md +++ b/docs/starlight/index.md @@ -162,6 +162,27 @@ avr run logs # full log dump, paginated avr job logs --follow # tail one job in real time ``` +**Read a job's recorded tests** + +```sh +avr tests # names, outcomes, timings and failure messages +avr tests --output json # results with reporting state and pagination +avr tests --limit 500 # follow short pages up to the requested count +avr tests --cursor '' # continue with pagination.next_cursor +``` + +Use the Avrea job ID from `avr job list`; the repository is resolved from that +job. `avr tests list ` is equivalent. `--json FIELDS` and `--jq` also +work. Each invocation reads at most 1,000 occurrences and 20 result pages, +preserving repeated tests and returning a cursor when more results remain. +Missing or expired reports are distinct from a retained report containing zero +tests. A report marked complete does not establish that every expected test ran. + +This requires an API deployment supporting job test results and access to the +repository's test evidence. Only Avrea job IDs are supported in this first +slice. Provider job IDs/URLs, immutable result links and anchored test history +follow separately; this command does not rerun tests or change collection. + **Trigger a workflow_dispatch** ```sh diff --git a/docs/starlight/reference/index.md b/docs/starlight/reference/index.md index c0b5ca5c..6aa1eb34 100644 --- a/docs/starlight/reference/index.md +++ b/docs/starlight/reference/index.md @@ -30,7 +30,7 @@ avr [GLOBAL OPTIONS] COMMAND [ARGS]... - [`avr vm`](./vm/) — Manage long-running VMs (SSH/RDP/VNC). - [`avr workflow`](./workflow/) — List and view workflow definitions. - [`avr cache`](./cache/) — Inspect and manage the Avrea build cache. -- [`avr tests`](./tests/) — Inspect recorded tests, failures, and reporting status. +- [`avr tests`](./tests/) — Read a job's tests, timings and recorded failure messages. - [`avr sbom`](./sbom/) — View, generate, and download repository SBOMs. - [`avr log`](./log/) — Search across runner execution logs. diff --git a/docs/starlight/reference/tests.md b/docs/starlight/reference/tests.md index a7564d18..59b9bb50 100644 --- a/docs/starlight/reference/tests.md +++ b/docs/starlight/reference/tests.md @@ -1,113 +1,46 @@ --- title: avr tests -description: "Inspect recorded tests, failures, and reporting status." +description: "Read a job's tests, timings and recorded failure messages." --- -Inspect recorded tests, failures, and reporting status. +Read a job's tests, timings and recorded failure messages. ```sh avr tests [OPTIONS] COMMAND [ARGS]... ``` -## Subcommands - -### `avr tests captures` - -List report deliveries, including missing, partial and failed captures. - -```sh -avr tests captures [OPTIONS] -``` - -**Options** - -- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. -- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. -- -L, --limit <INTEGER RANGE> — Rows per page (1-100). _(default: `50`)_ -- --cursor <TEXT> — Opaque next_cursor from the previous page. One page is read per invocation. -- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- -q, --jq <TEXT> — Filter --json output through a jq expression. - -### `avr tests history` +Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from +avr job list. No report capture ID or configuration digest is required. -Read recorded outcomes for one test and configuration, newest first. - -```sh -avr tests history [OPTIONS] TEST_ID -``` - -**Arguments** - -- TEST_ID - -**Options** - -- --configuration <TEXT> — History grouping identifier returned by tests list; not a platform label. _(required)_ -- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. -- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. -- -L, --limit <INTEGER RANGE> — Rows per page (1-100). _(default: `50`)_ -- --cursor <TEXT> — Opaque next_cursor from the previous page. One page is read per invocation. -- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- -q, --jq <TEXT> — Filter --json output through a jq expression. +## Subcommands ### `avr tests list` -List one page of test histories, grouped by test and configuration. - -```sh -avr tests list [OPTIONS] -``` - -Counts describe each returned history window, not repository totals. -Names appear when retained private detail is available; otherwise IDs are shown. -Use --json '*' to retain status, pagination and repository context for agents. - -**Options** - -- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. -- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. -- -L, --limit <INTEGER RANGE> — Rows per page (1-100). _(default: `50`)_ -- --cursor <TEXT> — Opaque next_cursor from the previous page. One page is read per invocation. -- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- -q, --jq <TEXT> — Filter --json output through a jq expression. - -### `avr tests status` - -Read collection settings. +Read the latest confirmed execution of JOB, including repeated tests. ```sh -avr tests status [OPTIONS] +avr tests list [OPTIONS] JOB ``` -Read collection settings. Use --json for retained-history lifecycle details. - -**Options** - -- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. -- --org <TEXT> — Organization ID or slug. Defaults to the configured organization. -- --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -- -q, --jq <TEXT> — Filter --json output through a jq expression. - -### `avr tests view` - -Read the exact retained result, including available failure messages. - -```sh -avr tests view [OPTIONS] TEST_ID -``` +Names, outcomes, recorded durations and available failure messages are +shown together. A failed test is data, not a command error. Missing, +pending, partial and expired evidence are reported explicitly; an older +execution is never substituted. Complete describes collected reports. -An expired or unrecorded result is never replaced with a different capture. -A recorded test failure is data, not a CLI error. No tests are rerun. +Use --output json for the complete response or --json FIELDS to select +fields. Continue with --cursor when pagination.next_cursor is present. +API refusal or a changed report fails the command without partial output. **Arguments** -- TEST_ID +- JOB **Options** -- --capture <TEXT> — Exact capture from tests history. _(required)_ -- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the git remote. - --org <TEXT> — Organization ID or slug. Defaults to the configured organization. -- -L, --limit <INTEGER RANGE> — Occurrences to read (1-500). _(default: `50`)_ +- --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the job's repository. +- -L, --limit <INTEGER RANGE> — Maximum occurrences to return; short pages are followed, up to 20 result pages. _(default: `100`)_ +- --cursor <TEXT> — Continue from a previous response's next_cursor. +- --output <CHOICE> — Output format. JSON includes status, context and the next cursor. _(choices: `text`, `json`)_ - --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. - -q, --jq <TEXT> — Filter --json output through a jq expression. diff --git a/man/man1/avr-tests-captures.1 b/man/man1/avr-tests-captures.1 deleted file mode 100644 index b2d50c6d..00000000 --- a/man/man1/avr-tests-captures.1 +++ /dev/null @@ -1,35 +0,0 @@ -.TH AVR\-TESTS\-CAPTURES 1 "" "avr 0.4.1" "Avrea CLI" -.SH NAME -avr-tests-captures \- List report deliveries, including missing, partial and failed captures. -.SH SYNOPSIS -.B avr tests captures -[OPTIONS] -.SH DESCRIPTION -List report deliveries, including missing, partial and failed captures. -.PP -.SH OPTIONS -.TP -\fB\-\-repo\fR \fITEXT\fR -Repository (owner/name or rep-ID). Defaults to the git remote. -.TP -\fB\-\-org\fR \fITEXT\fR -Organization ID or slug. Defaults to the configured organization. -.TP -\fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR -Rows per page (1-100). (default: 50) -.TP -\fB\-\-cursor\fR \fITEXT\fR -Opaque next_cursor from the previous page. One page is read per invocation. -.TP -\fB\-\-json\fR \fITEXT\fR -Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -.TP -\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR -Filter --json output through a jq expression. -.SH SEE ALSO -.BR avr (1), -.BR avr-tests (1), -.BR avr-tests-history (1), -.BR avr-tests-list (1), -.BR avr-tests-status (1), -.BR avr-tests-view (1) diff --git a/man/man1/avr-tests-history.1 b/man/man1/avr-tests-history.1 deleted file mode 100644 index dadd6ee3..00000000 --- a/man/man1/avr-tests-history.1 +++ /dev/null @@ -1,42 +0,0 @@ -.TH AVR\-TESTS\-HISTORY 1 "" "avr 0.4.1" "Avrea CLI" -.SH NAME -avr-tests-history \- Read recorded outcomes for one test and configuration, newest first. -.SH SYNOPSIS -.B avr tests history -[OPTIONS] TEST_ID -.SH DESCRIPTION -Read recorded outcomes for one test and configuration, newest first. -.PP -.SH ARGUMENTS -.TP -\fITEST_ID\fR -\& -.SH OPTIONS -.TP -\fB\-\-configuration\fR \fITEXT\fR -History grouping identifier returned by tests list; not a platform label. (required) -.TP -\fB\-\-repo\fR \fITEXT\fR -Repository (owner/name or rep-ID). Defaults to the git remote. -.TP -\fB\-\-org\fR \fITEXT\fR -Organization ID or slug. Defaults to the configured organization. -.TP -\fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR -Rows per page (1-100). (default: 50) -.TP -\fB\-\-cursor\fR \fITEXT\fR -Opaque next_cursor from the previous page. One page is read per invocation. -.TP -\fB\-\-json\fR \fITEXT\fR -Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -.TP -\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR -Filter --json output through a jq expression. -.SH SEE ALSO -.BR avr (1), -.BR avr-tests (1), -.BR avr-tests-captures (1), -.BR avr-tests-list (1), -.BR avr-tests-status (1), -.BR avr-tests-view (1) diff --git a/man/man1/avr-tests-list.1 b/man/man1/avr-tests-list.1 index 3d1c7cd6..420f934b 100644 --- a/man/man1/avr-tests-list.1 +++ b/man/man1/avr-tests-list.1 @@ -1,26 +1,35 @@ .TH AVR\-TESTS\-LIST 1 "" "avr 0.4.1" "Avrea CLI" .SH NAME -avr-tests-list \- List one page of test histories, grouped by test and configuration. +avr-tests-list \- Read the latest confirmed execution of JOB, including repeated tests. .SH SYNOPSIS .B avr tests list -[OPTIONS] +[OPTIONS] JOB .SH DESCRIPTION -List one page of test histories, grouped by test and configuration. +Read the latest confirmed execution of JOB, including repeated tests. .PP -Counts describe each returned history window, not repository totals. Names appear when retained private detail is available; otherwise IDs are shown. Use --json '*' to retain status, pagination and repository context for agents. -.SH OPTIONS +Names, outcomes, recorded durations and available failure messages are shown together. A failed test is data, not a command error. Missing, pending, partial and expired evidence are reported explicitly; an older execution is never substituted. Complete describes collected reports. +.PP +Use --output json for the complete response or --json FIELDS to select fields. Continue with --cursor when pagination.next_cursor is present. API refusal or a changed report fails the command without partial output. +.SH ARGUMENTS .TP -\fB\-\-repo\fR \fITEXT\fR -Repository (owner/name or rep-ID). Defaults to the git remote. +\fIJOB\fR +\& +.SH OPTIONS .TP \fB\-\-org\fR \fITEXT\fR Organization ID or slug. Defaults to the configured organization. .TP +\fB\-\-repo\fR \fITEXT\fR +Repository (owner/name or rep-ID). Defaults to the job's repository. +.TP \fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR -Rows per page (1-100). (default: 50) +Maximum occurrences to return; short pages are followed, up to 20 result pages. (default: 100) .TP \fB\-\-cursor\fR \fITEXT\fR -Opaque next_cursor from the previous page. One page is read per invocation. +Continue from a previous response's next_cursor. +.TP +\fB\-\-output\fR \fICHOICE\fR +Output format. JSON includes status, context and the next cursor. (choices: text, json) .TP \fB\-\-json\fR \fITEXT\fR Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. @@ -29,8 +38,4 @@ Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to lis Filter --json output through a jq expression. .SH SEE ALSO .BR avr (1), -.BR avr-tests (1), -.BR avr-tests-captures (1), -.BR avr-tests-history (1), -.BR avr-tests-status (1), -.BR avr-tests-view (1) +.BR avr-tests (1) diff --git a/man/man1/avr-tests-status.1 b/man/man1/avr-tests-status.1 deleted file mode 100644 index 43c52306..00000000 --- a/man/man1/avr-tests-status.1 +++ /dev/null @@ -1,30 +0,0 @@ -.TH AVR\-TESTS\-STATUS 1 "" "avr 0.4.1" "Avrea CLI" -.SH NAME -avr-tests-status \- Read collection settings. -.SH SYNOPSIS -.B avr tests status -[OPTIONS] -.SH DESCRIPTION -Read collection settings. -.PP -Read collection settings. Use --json for retained-history lifecycle details. -.SH OPTIONS -.TP -\fB\-\-repo\fR \fITEXT\fR -Repository (owner/name or rep-ID). Defaults to the git remote. -.TP -\fB\-\-org\fR \fITEXT\fR -Organization ID or slug. Defaults to the configured organization. -.TP -\fB\-\-json\fR \fITEXT\fR -Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -.TP -\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR -Filter --json output through a jq expression. -.SH SEE ALSO -.BR avr (1), -.BR avr-tests (1), -.BR avr-tests-captures (1), -.BR avr-tests-history (1), -.BR avr-tests-list (1), -.BR avr-tests-view (1) diff --git a/man/man1/avr-tests-view.1 b/man/man1/avr-tests-view.1 deleted file mode 100644 index 38ee6e15..00000000 --- a/man/man1/avr-tests-view.1 +++ /dev/null @@ -1,40 +0,0 @@ -.TH AVR\-TESTS\-VIEW 1 "" "avr 0.4.1" "Avrea CLI" -.SH NAME -avr-tests-view \- Read the exact retained result, including available failure messages. -.SH SYNOPSIS -.B avr tests view -[OPTIONS] TEST_ID -.SH DESCRIPTION -Read the exact retained result, including available failure messages. -.PP -An expired or unrecorded result is never replaced with a different capture. A recorded test failure is data, not a CLI error. No tests are rerun. -.SH ARGUMENTS -.TP -\fITEST_ID\fR -\& -.SH OPTIONS -.TP -\fB\-\-capture\fR \fITEXT\fR -Exact capture from tests history. (required) -.TP -\fB\-\-repo\fR \fITEXT\fR -Repository (owner/name or rep-ID). Defaults to the git remote. -.TP -\fB\-\-org\fR \fITEXT\fR -Organization ID or slug. Defaults to the configured organization. -.TP -\fB\-L\fR, \fB\-\-limit\fR \fIINTEGER RANGE\fR -Occurrences to read (1-500). (default: 50) -.TP -\fB\-\-json\fR \fITEXT\fR -Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. -.TP -\fB\-q\fR, \fB\-\-jq\fR \fITEXT\fR -Filter --json output through a jq expression. -.SH SEE ALSO -.BR avr (1), -.BR avr-tests (1), -.BR avr-tests-captures (1), -.BR avr-tests-history (1), -.BR avr-tests-list (1), -.BR avr-tests-status (1) diff --git a/man/man1/avr-tests.1 b/man/man1/avr-tests.1 index a1e92f74..b26b8e8e 100644 --- a/man/man1/avr-tests.1 +++ b/man/man1/avr-tests.1 @@ -1,28 +1,17 @@ .TH AVR\-TESTS 1 "" "avr 0.4.1" "Avrea CLI" .SH NAME -avr-tests \- Inspect recorded tests, failures, and reporting status. +avr-tests \- Read a job's tests, timings and recorded failure messages. .SH SYNOPSIS .B avr tests [OPTIONS] COMMAND [ARGS]... .SH DESCRIPTION -Inspect recorded tests, failures, and reporting status. +Read a job's tests, timings and recorded failure messages. .PP +Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from avr job list. No report capture ID or configuration digest is required. .SH COMMANDS .TP -captures -List report deliveries, including missing, partial and failed captures. -.TP -history -Read recorded outcomes for one test and configuration, newest first. -.TP list -List one page of test histories, grouped by test and configuration. -.TP -status -Read collection settings. -.TP -view -Read the exact retained result, including available failure messages. +Read the latest confirmed execution of JOB, including repeated tests. .SH SEE ALSO .BR avr (1), .BR avr-status (1), @@ -44,8 +33,4 @@ Read the exact retained result, including available failure messages. .BR avr-repo (1), .BR avr-org (1), .BR avr-health (1), -.BR avr-tests-captures (1), -.BR avr-tests-history (1), -.BR avr-tests-list (1), -.BR avr-tests-status (1), -.BR avr-tests-view (1) +.BR avr-tests-list (1) diff --git a/man/man1/avr.1 b/man/man1/avr.1 index 87509991..bce0dfa2 100644 --- a/man/man1/avr.1 +++ b/man/man1/avr.1 @@ -47,7 +47,7 @@ cache Inspect and manage the Avrea build cache. .TP tests -Inspect recorded tests, failures, and reporting status. +Read a job's tests, timings and recorded failure messages. .TP sbom View, generate, and download repository SBOMs. diff --git a/tests/test_tests.py b/tests/test_tests.py index 4b86775d..e855d1cd 100644 --- a/tests/test_tests.py +++ b/tests/test_tests.py @@ -1,4 +1,4 @@ -"""Customer test-evidence commands preserve scope and bounded API responses.""" +"""Job results are read through the authorized repository and exact API cursor.""" from avrea_cli.main import cli from click.testing import CliRunner @@ -9,133 +9,231 @@ ORG = "org-019a0000000070008000000000000001" REPO = "rep-019a0000000070008000000000000002" -TEST = "a" * 64 -CONFIG = "b" * 64 -CAPTURE = "tec-019a0000000070008000000000000003" -BASE = f"/orgs/{ORG}/repos/{REPO}/tests" -SCOPE = ["--org", ORG, "--repo", REPO] -PAGE: dict[str, Any] = { - "status": "available", - "pending": True, - "reason": None, - "as_of_revision": 1, - "current_revision": 2, - "data": [{"test_id": TEST, "configuration_digest": CONFIG, "failed_count": 1, "passed_count": 2}], - "pagination": {"next_cursor": "c" * 900}, +JOB = "job-019a0000000070008000000000000003" +EXECUTION = "exe-019a0000000070008000000000000004" +JOB_PATH = f"/orgs/{ORG}/jobs/{JOB}" +RESULTS_PATH = f"/orgs/{ORG}/repos/{REPO}/jobs/{JOB}/tests" +SCOPE = ["--org", ORG] +ROW = { + "test_id": "a" * 64, + "suite": "checkout", + "class_name": "TestCheckout", + "name": "test_payment", + "outcome": "failed", + "duration_ms": 21, + "failure_summary": "expected 200, got 500", } -def test_list_preserves_page_status_and_context(runner: CliRunner, api: FakeApi) -> None: - api.reply("GET", f"{BASE}/history", json=PAGE) - result = runner.invoke(cli, ["tests", "list", *SCOPE, "--limit", "7", "--cursor", "c" * 900, "--json", "*"]) +def page(rows: list[dict[str, Any]], *, cursor: str | None = None, **changes: Any) -> dict[str, Any]: + return { + "job_id": JOB, + "execution_id": EXECUTION, + "status": "available", + "reporting_state": "complete", + "reason": None, + "commit_sha": "b" * 40, + "observed_at": "2026-10-28T12:00:00Z", + "total_count": 3, + "data": rows, + "pagination": {"next_cursor": cursor}, + **changes, + } + + +@pytest.fixture +def job_api(api: FakeApi) -> FakeApi: + api.reply("GET", JOB_PATH, json={"data": {"job_id": JOB, "repository_id": REPO, "job_name": "Unit tests"}}) + return api + + +def test_job_short_and_empty_pages_continue_without_dropping_repeats(runner: CliRunner, job_api: FakeApi) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="first")) + job_api.reply("GET", RESULTS_PATH, json=page([], cursor="second")) + job_api.reply("GET", RESULTS_PATH, json=page([ROW, {**ROW, "outcome": "passed"}])) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--output", "json"]) assert result.exit_code == 0, result.output - response = json.loads(result.output) - assert response["organization_id"] == ORG - assert response["repository_id"] == REPO - assert response["data"] == PAGE["data"] - assert response["pending"] is True - assert response["pagination"] == PAGE["pagination"] - assert len(api.requests) == 1 - assert dict(api.requests[0].url.params) == {"limit": "7", "cursor": "c" * 900} + data = json.loads(result.output) + assert data["organization_id"] == ORG + assert data["repository_id"] == REPO + assert data["job_id"] == JOB + assert data["execution_id"] == EXECUTION + assert data["data"] == [ROW, ROW, {**ROW, "outcome": "passed"}] + assert data["pagination"]["next_cursor"] is None + assert [r.url.params.get("cursor") for r in job_api.sent("GET", RESULTS_PATH)] == [None, "first", "second"] -def test_history_passes_exact_configuration(runner: CliRunner, api: FakeApi) -> None: - api.reply("GET", f"{BASE}/history/{TEST}", json={**PAGE, "data": [{"capture_id": CAPTURE}]}) - result = runner.invoke(cli, ["tests", "history", TEST, "--configuration", CONFIG, *SCOPE, "--json", "*"]) +@pytest.mark.parametrize( + "state", ["pending", "missing", "skipped", "no_report", "partial", "complete", "failed", "not_recorded"] +) +def test_job_reporting_is_separate_from_availability(runner: CliRunner, job_api: FakeApi, state: str) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([], reporting_state=state, status="not_recorded", total_count=None)) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) assert result.exit_code == 0, result.output - assert json.loads(result.output)["configuration_digest"] == CONFIG - assert api.requests[0].url.params["configuration"] == CONFIG - - -@pytest.mark.parametrize("state", ["available", "expired", "not_recorded"]) -def test_view_keeps_exact_capture_without_fallback(runner: CliRunner, api: FakeApi, state: str) -> None: - rows = ( - [{"name": "test_checkout", "outcome": "failed", "failure_summary": "expected 200, got 500"}] - if state == "available" - else [] - ) - api.reply( - "GET", - f"{BASE}/history/{TEST}/detail/{CAPTURE}", - json={"status": state, "data": rows, "truncated": state == "available"}, - ) - result = runner.invoke(cli, ["tests", "view", TEST, "--capture", CAPTURE, *SCOPE, "--limit", "500", "--json", "*"]) + data = json.loads(result.output) + assert data["reporting_state"] == state + assert data["status"] == "not_recorded" + assert data["total_count"] is None + + +def test_job_text_shows_failure_and_escapes_terminal_controls(runner: CliRunner, job_api: FakeApi) -> None: + row = {**ROW, "name": "test_\x1b[31mpayment", "failure_summary": "bad\r\nline\x1b[2J"} + job_api.reply("GET", RESULTS_PATH, json=page([row])) + result = runner.invoke(cli, ["tests", JOB, *SCOPE]) assert result.exit_code == 0, result.output - response = json.loads(result.output) - assert (response["test_id"], response["capture_id"], response["status"]) == (TEST, CAPTURE, state) - assert response["data"] == rows - assert response["truncated"] == (state == "available") - assert len(api.requests) == 1 + assert "payment" in result.output + assert "failed" in result.output + assert "21" in result.output + assert "bad\\r\\nline" in result.output + assert "\x1b" not in result.output + assert "Reporting" in result.output -@pytest.mark.parametrize("status", [403, 404, 409, 503]) -def test_denied_or_failed_reads_are_not_empty_success(runner: CliRunner, api: FakeApi, status: int) -> None: - api.reply("GET", f"{BASE}/history", status=status, json={"detail": "Evidence is unavailable"}) - result = runner.invoke(cli, ["tests", "list", *SCOPE, "--json", "*"]) +@pytest.mark.parametrize("code", [403, 404, 409, 429, 503]) +def test_job_page_failure_never_emits_partial_success(runner: CliRunner, job_api: FakeApi, code: int) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="next")) + job_api.reply("GET", RESULTS_PATH, status=code, json={"detail": "Report changed or unavailable"}) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--output", "json"]) assert result.exit_code != 0 - assert len(api.requests) == 1 - assert "Evidence is unavailable" in result.output + assert not result.stdout -def test_status_reads_control_without_mutation(runner: CliRunner, api: FakeApi) -> None: - state = {"capture_enabled": False, "can_manage": True, "repository_generation": 2, "purges": [], "events": []} - api.reply("GET", f"{BASE}/control", json=state) - result = runner.invoke(cli, ["tests", "status", *SCOPE, "--json", "*"]) +def test_explicit_repository_does_not_require_job_metadata(runner: CliRunner, api: FakeApi) -> None: + api.reply("GET", RESULTS_PATH, json=page([ROW])) + result = runner.invoke(cli, ["tests", "list", JOB, *SCOPE, "--repo", REPO, "--json", "data,status"]) assert result.exit_code == 0, result.output - assert json.loads(result.output)["data"] == state + assert json.loads(result.stdout) == {"data": [ROW], "status": "available"} assert len(api.requests) == 1 -def test_captures_preserves_missing_report_reason(runner: CliRunner, api: FakeApi) -> None: - rows = [{"capture_id": CAPTURE, "state": "no_report", "reason": "no_supported_report"}] - api.reply("GET", f"{BASE}/captures", json={"data": rows, "pagination": {"next_cursor": None}}) - result = runner.invoke(cli, ["tests", "captures", *SCOPE, "--json", "*"]) +def test_limit_preserves_next_cursor_and_request_sizes(runner: CliRunner, job_api: FakeApi) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="one")) + job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="two")) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--limit", "2", "--json", "*"]) + assert result.exit_code == 0, result.output + response = json.loads(result.stdout) + assert len(response["data"]) == 2 + assert response["pagination"] == {"next_cursor": "two"} + assert "--cursor two" in result.stderr + assert [dict(r.url.params) for r in job_api.sent("GET", RESULTS_PATH)] == [ + {"limit": "2"}, + {"limit": "1", "cursor": "one"}, + ] + + +def test_page_budget_returns_a_resume_cursor(runner: CliRunner, job_api: FakeApi) -> None: + for i in range(20): + job_api.reply("GET", RESULTS_PATH, json=page([], cursor=f"page-{i}")) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) assert result.exit_code == 0, result.output - assert json.loads(result.output)["data"] == rows + assert len(job_api.sent("GET", RESULTS_PATH)) == 20 + assert json.loads(result.stdout)["pagination"]["next_cursor"] == "page-19" + assert "page-19" in result.stderr -def test_name_terminal_controls_are_escaped(runner: CliRunner, api: FakeApi) -> None: - row = {**PAGE["data"][0], "identity": {"name": "test_\x1b[31mcheckout"}} - api.reply("GET", f"{BASE}/history", json={**PAGE, "data": [row]}) - result = runner.invoke(cli, ["tests", "list", *SCOPE]) +@pytest.mark.parametrize( + "changes", + [ + {"execution_id": "exe-019a0000000070008000000000000005"}, + {"status": "expired"}, + {"total_count": 4}, + {"job_id": "job-019a0000000070008000000000000006"}, + ], +) +def test_changed_job_pages_are_not_combined(runner: CliRunner, job_api: FakeApi, changes: dict[str, Any]) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="next")) + job_api.reply("GET", RESULTS_PATH, json=page([ROW], **changes)) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) + assert result.exit_code != 0 + assert not result.stdout + + +@pytest.mark.parametrize( + "bad_page", + [ + page([ROW], cursor="next"), + page([ROW], cursor=" "), + page([ROW], pagination={"next_cursor": 17}), + page([ROW], data="invalid"), + ], +) +def test_invalid_pagination_stops_without_partial_output( + runner: CliRunner, job_api: FakeApi, bad_page: dict[str, Any] +) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="next")) + job_api.reply("GET", RESULTS_PATH, json=bad_page) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) + assert result.exit_code != 0 + assert not result.stdout + assert len(job_api.sent("GET", RESULTS_PATH)) == 2 + + +def test_cursor_resume_uses_the_exact_value(runner: CliRunner, job_api: FakeApi) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW])) + cursor = "x" * 900 + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--cursor", cursor, "--output", "json"]) assert result.exit_code == 0, result.output - assert "\x1b" not in result.output - assert "checkout" in result.output + assert job_api.sent("GET", RESULTS_PATH)[0].url.params["cursor"] == cursor @pytest.mark.parametrize( "args", [ - ["list", "--cursor", " "], - ["history", "bad", "--configuration", CONFIG], - ["view", TEST, "--capture", "bad"], - ["list", "--json", "no_such_field"], + ["list", "not-a-job"], + [JOB, "--cursor", " "], + [JOB, "--cursor", "x" * 1025], + [JOB, "--limit", "0"], + [JOB, "--limit", "1001"], + [JOB, "--json", "capture_id"], + [JOB, "--output", "text", "--json", "*"], + [JOB, "--jq", ".data"], ], ) -def test_bad_arguments_make_no_requests(runner: CliRunner, api: FakeApi, args: list[str]) -> None: +def test_invalid_options_make_no_requests(runner: CliRunner, api: FakeApi, args: list[str]) -> None: result = runner.invoke(cli, ["tests", *args, *SCOPE]) assert result.exit_code != 0 assert api.requests == [] -def test_empty_capture_page_and_partial_detail_are_explicit(runner: CliRunner, api: FakeApi) -> None: - api.reply("GET", f"{BASE}/captures", json={"data": [], "pagination": {"next_cursor": None}}) - result = runner.invoke(cli, ["tests", "captures", *SCOPE]) +@pytest.mark.parametrize( + "state,total,message", + [ + ("expired", None, "Missing or expired reports"), + ("not_recorded", None, "Missing or expired reports"), + ("available", 0, "retained report contains zero"), + ], +) +def test_empty_results_explain_availability( + runner: CliRunner, job_api: FakeApi, state: str, total: int | None, message: str +) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([], status=state, total_count=total)) + result = runner.invoke(cli, ["tests", JOB, *SCOPE]) assert result.exit_code == 0, result.output - assert "No recorded results in this page" in result.output - api.reply( - "GET", f"{BASE}/history/{TEST}/detail/{CAPTURE}", json={"status": "available", "data": [], "truncated": True} - ) - result = runner.invoke(cli, ["tests", "view", TEST, "--capture", CAPTURE, *SCOPE, "--limit", "500"]) + assert message in result.stdout + assert state in result.stdout + assert len(job_api.sent("GET", RESULTS_PATH)) == 1 + + +def test_wrong_organization_does_not_fall_back_to_repository_scan(runner: CliRunner, api: FakeApi) -> None: + api.reply("GET", JOB_PATH, status=404, json={"detail": "Job not found"}) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) + assert result.exit_code != 0 + assert not result.stdout + assert len(api.requests) == 1 + + +def test_json_preserves_raw_test_names_and_failure_text(runner: CliRunner, job_api: FakeApi) -> None: + row = {**ROW, "name": "line\nbreak", "failure_summary": "\x1b[31merror"} + job_api.reply("GET", RESULTS_PATH, json=page([row])) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--output", "json"]) assert result.exit_code == 0, result.output - assert "This result is partial" in result.output - assert "Increase --limit" not in result.output + assert json.loads(result.stdout)["data"] == [row] -@pytest.mark.parametrize("duration_status", ["expired_detail", "unavailable"]) -def test_history_text_explains_missing_duration(runner: CliRunner, api: FakeApi, duration_status: str) -> None: - row = {"capture_id": CAPTURE, "duration_total_ms": None, "duration_status": duration_status} - api.reply("GET", f"{BASE}/history/{TEST}", json={**PAGE, "data": [row]}) - result = runner.invoke(cli, ["tests", "history", TEST, "--configuration", CONFIG, *SCOPE]) +def test_help_only_advertises_the_shipped_job_journey(runner: CliRunner, api: FakeApi) -> None: + result = runner.invoke(cli, ["tests", "--help"]) assert result.exit_code == 0, result.output - assert duration_status in result.output + assert "avr tests JOB" in result.stdout + assert "history" not in result.stdout + assert "view" not in result.stdout + assert api.requests == [] From 14ab63f551c6984caba271ffb9b01ecb72a14ac3 Mon Sep 17 00:00:00 2001 From: Hannu Varjoranta Date: Wed, 28 Oct 2026 16:09:31 +0200 Subject: [PATCH 4/7] cli: regenerate test docs against current command tree --- man/man1/avr-artifact.1 | 1 + man/man1/avr-tests-list.1 | 2 +- man/man1/avr-tests.1 | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/man/man1/avr-artifact.1 b/man/man1/avr-artifact.1 index 26e957e3..c0eea362 100644 --- a/man/man1/avr-artifact.1 +++ b/man/man1/avr-artifact.1 @@ -29,6 +29,7 @@ List artifacts in REPO, optionally restricted to one run. .BR avr-vm (1), .BR avr-workflow (1), .BR avr-cache (1), +.BR avr-tests (1), .BR avr-sbom (1), .BR avr-log (1), .BR avr-auth (1), diff --git a/man/man1/avr-tests-list.1 b/man/man1/avr-tests-list.1 index 420f934b..82978d0b 100644 --- a/man/man1/avr-tests-list.1 +++ b/man/man1/avr-tests-list.1 @@ -1,4 +1,4 @@ -.TH AVR\-TESTS\-LIST 1 "" "avr 0.4.1" "Avrea CLI" +.TH AVR\-TESTS\-LIST 1 "" "avr 0.5.0" "Avrea CLI" .SH NAME avr-tests-list \- Read the latest confirmed execution of JOB, including repeated tests. .SH SYNOPSIS diff --git a/man/man1/avr-tests.1 b/man/man1/avr-tests.1 index b26b8e8e..1e738390 100644 --- a/man/man1/avr-tests.1 +++ b/man/man1/avr-tests.1 @@ -1,4 +1,4 @@ -.TH AVR\-TESTS 1 "" "avr 0.4.1" "Avrea CLI" +.TH AVR\-TESTS 1 "" "avr 0.5.0" "Avrea CLI" .SH NAME avr-tests \- Read a job's tests, timings and recorded failure messages. .SH SYNOPSIS @@ -14,6 +14,7 @@ list Read the latest confirmed execution of JOB, including repeated tests. .SH SEE ALSO .BR avr (1), +.BR avr-artifact (1), .BR avr-status (1), .BR avr-run (1), .BR avr-pr (1), From 765ec56fa237c41037594587ad1bb13d74ee6466 Mon Sep 17 00:00:00 2001 From: Hannu Varjoranta Date: Wed, 28 Oct 2026 19:42:00 +0200 Subject: [PATCH 5/7] cli: make test result return path explicit --- avrea_cli/commands/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/avrea_cli/commands/tests.py b/avrea_cli/commands/tests.py index df4048d3..7bc3cede 100644 --- a/avrea_cli/commands/tests.py +++ b/avrea_cli/commands/tests.py @@ -57,9 +57,10 @@ def _job_id(_ctx: click.Context, param: click.Parameter, value: str) -> str: def _get(client: ApiClient, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: try: - return client.public_get(path, params=params) + response = client.public_get(path, params=params) except httpx.HTTPStatusError as exc: handle_http_error(exc, "read job test results") + return response def _job_repository(client: ApiClient, organization: str, job: str) -> str: From 74bcd76e48b3981dcc52fc20bee3e17580431687 Mon Sep 17 00:00:00 2001 From: Hannu Varjoranta Date: Fri, 30 Oct 2026 18:07:02 +0200 Subject: [PATCH 6/7] cli: read test pages at the requested size and label failures by class - Request the remaining limit, up to the API's 1,000, instead of 100 per page; the server reads one block per request whatever the limit - Label each failure message with class and test name, so same-named tests in different classes stay distinguishable - Describe only what the command does now in its help and the README --- README.md | 5 ++--- avrea_cli/commands/tests.py | 8 +++++--- docs/REFERENCE.md | 2 +- docs/reference.json | 2 +- docs/starlight/index.md | 5 ++--- docs/starlight/reference/tests.md | 2 +- man/man1/avr-tests.1 | 2 +- tests/test_tests.py | 20 ++++++++++++++++++++ 8 files changed, 33 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c35ed07a..0a339eb4 100644 --- a/README.md +++ b/README.md @@ -174,9 +174,8 @@ Missing or expired reports are distinct from a retained report containing zero tests. A report marked complete does not establish that every expected test ran. This requires an API deployment supporting job test results and access to the -repository's test evidence. Only Avrea job IDs are supported in this first -slice. Provider job IDs/URLs, immutable result links and anchored test history -follow separately; this command does not rerun tests or change collection. +repository's test evidence. Only Avrea job IDs are supported. The command does +not rerun tests or change collection. **Trigger a workflow_dispatch** diff --git a/avrea_cli/commands/tests.py b/avrea_cli/commands/tests.py index 7bc3cede..7006d0a6 100644 --- a/avrea_cli/commands/tests.py +++ b/avrea_cli/commands/tests.py @@ -80,7 +80,7 @@ def _results(client: ApiClient, path: str, job: str, limit: int, cursor: str | N snapshot: dict[str, Any] | None = None seen = {cursor} if cursor else set() for _ in range(_MAX_PAGES): - page_limit = min(100, limit - len(rows)) + page_limit = limit - len(rows) params: dict[str, Any] = {"limit": page_limit} if cursor is not None: params["cursor"] = cursor @@ -132,7 +132,9 @@ def _show(response: dict[str, Any]) -> None: ) for row in rows: if row.get("failure_summary"): - click.echo(f"\n{_text(row.get('name'))}: {_text(row['failure_summary'])}") + # The class keeps same-named tests in different classes apart. + label = ".".join(_text(row.get(key)) for key in ("class_name", "name") if row.get(key)) + click.echo(f"\n{label or 'unknown'}: {_text(row['failure_summary'])}") elif response.get("status") == "available" and response.get("total_count") == 0: click.echo("The retained report contains zero test occurrences. This does not establish that no tests ran.") else: @@ -147,7 +149,7 @@ def tests(ctx: click.Context) -> None: """Read a job's tests, timings and recorded failure messages. Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from - avr job list. No report capture ID or configuration digest is required. + avr job list. """ ensure_ctx(ctx) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index c96088aa..102224f0 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -1313,7 +1313,7 @@ avr tests [OPTIONS] COMMAND [ARGS]... ``` Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from -avr job list. No report capture ID or configuration digest is required. +avr job list. #### `avr tests list` diff --git a/docs/reference.json b/docs/reference.json index 64bd1ac6..05a99652 100644 --- a/docs/reference.json +++ b/docs/reference.json @@ -6257,7 +6257,7 @@ "full": "avr tests", "is_group": true, "short_help": "Read a job's tests, timings and recorded failure messages.", - "help": "Read a job's tests, timings and recorded failure messages.\n\nUse avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from\navr job list. No report capture ID or configuration digest is required.\n", + "help": "Read a job's tests, timings and recorded failure messages.\n\nUse avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from\navr job list.\n", "epilog": null, "deprecated": false, "hidden": false, diff --git a/docs/starlight/index.md b/docs/starlight/index.md index 31ed5924..68445f0b 100644 --- a/docs/starlight/index.md +++ b/docs/starlight/index.md @@ -179,9 +179,8 @@ Missing or expired reports are distinct from a retained report containing zero tests. A report marked complete does not establish that every expected test ran. This requires an API deployment supporting job test results and access to the -repository's test evidence. Only Avrea job IDs are supported in this first -slice. Provider job IDs/URLs, immutable result links and anchored test history -follow separately; this command does not rerun tests or change collection. +repository's test evidence. Only Avrea job IDs are supported. The command does +not rerun tests or change collection. **Trigger a workflow_dispatch** diff --git a/docs/starlight/reference/tests.md b/docs/starlight/reference/tests.md index 59b9bb50..0f596d1a 100644 --- a/docs/starlight/reference/tests.md +++ b/docs/starlight/reference/tests.md @@ -10,7 +10,7 @@ avr tests [OPTIONS] COMMAND [ARGS]... ``` Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from -avr job list. No report capture ID or configuration digest is required. +avr job list. ## Subcommands diff --git a/man/man1/avr-tests.1 b/man/man1/avr-tests.1 index 1e738390..fb6d240a 100644 --- a/man/man1/avr-tests.1 +++ b/man/man1/avr-tests.1 @@ -7,7 +7,7 @@ avr-tests \- Read a job's tests, timings and recorded failure messages. .SH DESCRIPTION Read a job's tests, timings and recorded failure messages. .PP -Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from avr job list. No report capture ID or configuration digest is required. +Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from avr job list. .SH COMMANDS .TP list diff --git a/tests/test_tests.py b/tests/test_tests.py index e855d1cd..6a83e44d 100644 --- a/tests/test_tests.py +++ b/tests/test_tests.py @@ -121,6 +121,26 @@ def test_limit_preserves_next_cursor_and_request_sizes(runner: CliRunner, job_ap ] +def test_large_limit_is_read_in_one_request_when_the_page_holds_it(runner: CliRunner, job_api: FakeApi) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW] * 1000, total_count=1000)) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--limit", "1000", "--json", "*"]) + assert result.exit_code == 0, result.output + assert len(json.loads(result.stdout)["data"]) == 1000 + assert [dict(r.url.params) for r in job_api.sent("GET", RESULTS_PATH)] == [{"limit": "1000"}] + + +def test_failure_messages_name_the_class_of_same_named_tests(runner: CliRunner, job_api: FakeApi) -> None: + rows = [ + {**ROW, "class_name": "TestCart", "name": "test_init", "failure_summary": "cart failed"}, + {**ROW, "class_name": "TestOrder", "name": "test_init", "failure_summary": "order failed"}, + ] + job_api.reply("GET", RESULTS_PATH, json=page(rows)) + result = runner.invoke(cli, ["tests", JOB, *SCOPE]) + assert result.exit_code == 0, result.output + assert "TestCart.test_init: cart failed" in result.stdout + assert "TestOrder.test_init: order failed" in result.stdout + + def test_page_budget_returns_a_resume_cursor(runner: CliRunner, job_api: FakeApi) -> None: for i in range(20): job_api.reply("GET", RESULTS_PATH, json=page([], cursor=f"page-{i}")) From f10fe3ab531f477d242f9f9d93a7e542f0f011ec Mon Sep 17 00:00:00 2001 From: Hannu Varjoranta Date: Sat, 31 Oct 2026 08:40:47 +0200 Subject: [PATCH 7/7] cli: use --json only and accept native job IDs in avr tests - Drop --output text|json; --json '*' already returns the full response and no other avr command has --output. - Accept awj- job IDs, which avr job list shows and the route accepts. - 404s now say what to check: a stale --cursor, Repository Tests off, or the job outside the repository; a missing job suggests --org. - An invalid next_cursor from the API is an API error (exit 1), not a usage error about an option the user did not pass. - An empty page after a cursor on available results says "No more results." instead of the missing-report caveat. - Failure-path tests assert the exit code and message, and cover every snapshot field, oversized pages, job mismatches, the complete caveat and the reason line. --- README.md | 6 +- avrea_cli/commands/tests.py | 59 ++++++++------ docs/REFERENCE.md | 7 +- docs/reference.json | 26 +------ docs/starlight/index.md | 6 +- docs/starlight/reference/tests.md | 7 +- man/man1/avr-tests-list.1 | 5 +- man/man1/avr-tests.1 | 2 +- tests/test_tests.py | 124 +++++++++++++++++++++++------- 9 files changed, 151 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index 0a339eb4..383aeb9c 100644 --- a/README.md +++ b/README.md @@ -161,13 +161,13 @@ avr job logs --follow # tail one job in real time ```sh avr tests # names, outcomes, timings and failure messages -avr tests --output json # results with reporting state and pagination +avr tests --json '*' # results with reporting state and pagination avr tests --limit 500 # follow short pages up to the requested count avr tests --cursor '' # continue with pagination.next_cursor ``` -Use the Avrea job ID from `avr job list`; the repository is resolved from that -job. `avr tests list ` is equivalent. `--json FIELDS` and `--jq` also +Use the Avrea job ID (`job-` or `awj-`) from `avr job list`; the repository is +resolved from that job. `avr tests list ` is equivalent. `--json FIELDS` and `--jq` also work. Each invocation reads at most 1,000 occurrences and 20 result pages, preserving repeated tests and returning a cursor when more results remain. Missing or expired reports are distinct from a retained report containing zero diff --git a/avrea_cli/commands/tests.py b/avrea_cli/commands/tests.py index 7006d0a6..9e5863cb 100644 --- a/avrea_cli/commands/tests.py +++ b/avrea_cli/commands/tests.py @@ -34,6 +34,8 @@ ) _FIELDS = make_schema("organization_id", "repository_id", *_SNAPSHOT_FIELDS, "data", "pagination") _MAX_PAGES = 20 +_JOB_ID = re.compile(r"(job|awj)-[0-9a-f]{32}") +_INVALID_NEXT_CURSOR = "The API returned an invalid next cursor." class JobTestsGroup(GhGroup): @@ -42,29 +44,37 @@ class JobTestsGroup(GhGroup): def resolve_command( self, ctx: click.Context, args: list[str] ) -> tuple[str | None, click.Command | None, list[str]]: - if args and args[0].startswith("job-"): + if args and args[0].startswith(("job-", "awj-")): args = ["list", *args] return super().resolve_command(ctx, args) def _job_id(_ctx: click.Context, param: click.Parameter, value: str) -> str: - if not re.fullmatch(r"job-[0-9a-f]{32}", value): + if not _JOB_ID.fullmatch(value): raise click.BadParameter( - "use the Avrea job ID from 'avr job list' (job- followed by 32 lowercase hex characters)", param=param + "use the Avrea job ID from 'avr job list' (job- or awj- followed by 32 lowercase hex characters)", + param=param, ) return value -def _get(client: ApiClient, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: +def _get( + client: ApiClient, path: str, action: str, not_found_hint: str, params: dict[str, Any] | None = None +) -> dict[str, Any]: try: response = client.public_get(path, params=params) except httpx.HTTPStatusError as exc: - handle_http_error(exc, "read job test results") + handle_http_error(exc, action, hints={404: not_found_hint}) return response def _job_repository(client: ApiClient, organization: str, job: str) -> str: - response = _get(client, f"/orgs/{organization}/jobs/{job}") + response = _get( + client, + f"/orgs/{organization}/jobs/{job}", + "read the job", + "Check the job ID, or pass --org with the job's organization.", + ) data = response.get("data", response) repository = data.get("repository_id") if isinstance(data, dict) else None if not isinstance(repository, str) or not re.fullmatch(r"rep-[0-9a-f]{32}", repository): @@ -84,7 +94,16 @@ def _results(client: ApiClient, path: str, job: str, limit: int, cursor: str | N params: dict[str, Any] = {"limit": page_limit} if cursor is not None: params["cursor"] = cursor - page = _get(client, path, params) + page = _get( + client, + path, + "read job test results", + # A new runtime invalidates old cursors, and the route also answers 404 + # when Repository Tests are off or the job is outside the repository. + "If you passed --cursor, restart without it. Otherwise check that Repository" + " Tests are enabled and that the job belongs to the repository.", + params, + ) data = page.get("data") pagination = page.get("pagination") if page.get("job_id") != job or not isinstance(data, list) or not isinstance(pagination, dict): @@ -98,8 +117,12 @@ def _results(client: ApiClient, path: str, job: str, limit: int, cursor: str | N cursor = pagination.get("next_cursor") if cursor is not None: if not isinstance(cursor, str): - raise click.ClickException("The API returned an invalid next cursor.") - validate_cursor(cursor, max_length=1024) + raise click.ClickException(_INVALID_NEXT_CURSOR) + try: + validate_cursor(cursor, max_length=1024) + except click.BadParameter: + # The server's cursor is not user input, so it gets no usage error. + raise click.ClickException(_INVALID_NEXT_CURSOR) from None if cursor in seen: raise click.ClickException("The API repeated a test-results cursor; stopping pagination.") seen.add(cursor) @@ -137,6 +160,8 @@ def _show(response: dict[str, Any]) -> None: click.echo(f"\n{label or 'unknown'}: {_text(row['failure_summary'])}") elif response.get("status") == "available" and response.get("total_count") == 0: click.echo("The retained report contains zero test occurrences. This does not establish that no tests ran.") + elif response.get("status") == "available": + click.echo("No more results.") else: click.echo("No test results returned. Missing or expired reports do not mean that zero tests ran.") if response.get("reporting_state") == "complete": @@ -148,8 +173,8 @@ def _show(response: dict[str, Any]) -> None: def tests(ctx: click.Context) -> None: """Read a job's tests, timings and recorded failure messages. - Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from - avr job list. + Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID + (job- or awj-) from avr job list. """ ensure_ctx(ctx) @@ -167,11 +192,6 @@ def tests(ctx: click.Context) -> None: help=f"Maximum occurrences to return; short pages are followed, up to {_MAX_PAGES} result pages.", ) @click.option("--cursor", help="Continue from a previous response's next_cursor.") -@click.option( - "--output", - type=click.Choice(["text", "json"]), - help="Output format. JSON includes status, context and the next cursor.", -) @json_options @click.pass_context def tests_list( @@ -181,7 +201,6 @@ def tests_list( repo_id: str | None, limit: int, cursor: str | None, - output: str | None, json_fields: str | None, jq_expr: str | None, ) -> None: @@ -192,14 +211,10 @@ def tests_list( pending, partial and expired evidence are reported explicitly; an older execution is never substituted. Complete describes collected reports. - Use --output json for the complete response or --json FIELDS to select + Use --json '*' for the complete response or --json FIELDS to select fields. Continue with --cursor when pagination.next_cursor is present. API refusal or a changed report fails the command without partial output. """ - if output == "text" and json_fields is not None: - raise click.UsageError("--output text cannot be combined with --json") - if output == "json" and json_fields is None: - json_fields = "*" if handle_json_meta(json_fields, jq_expr, _FIELDS): return cursor = validate_cursor(cursor, max_length=1024) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 102224f0..b444fe5b 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -1312,8 +1312,8 @@ Read a job's tests, timings and recorded failure messages. avr tests [OPTIONS] COMMAND [ARGS]... ``` -Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from -avr job list. +Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID +(job- or awj-) from avr job list. #### `avr tests list` @@ -1328,7 +1328,7 @@ shown together. A failed test is data, not a command error. Missing, pending, partial and expired evidence are reported explicitly; an older execution is never substituted. Complete describes collected reports. -Use --output json for the complete response or --json FIELDS to select +Use --json '*' for the complete response or --json FIELDS to select fields. Continue with --cursor when pagination.next_cursor is present. API refusal or a changed report fails the command without partial output. @@ -1342,7 +1342,6 @@ API refusal or a changed report fails the command without partial output. - `--repo ` — Repository (owner/name or rep-ID). Defaults to the job's repository. - `-L, --limit ` — Maximum occurrences to return; short pages are followed, up to 20 result pages. _(default: `100`)_ - `--cursor ` — Continue from a previous response's next_cursor. -- `--output ` — Output format. JSON includes status, context and the next cursor. _(choices: `text`, `json`)_ - `--json ` — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. - `-q, --jq ` — Filter --json output through a jq expression. diff --git a/docs/reference.json b/docs/reference.json index 05a99652..dcc512d6 100644 --- a/docs/reference.json +++ b/docs/reference.json @@ -6257,7 +6257,7 @@ "full": "avr tests", "is_group": true, "short_help": "Read a job's tests, timings and recorded failure messages.", - "help": "Read a job's tests, timings and recorded failure messages.\n\nUse avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from\navr job list.\n", + "help": "Read a job's tests, timings and recorded failure messages.\n\nUse avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID\n(job- or awj-) from avr job list.\n", "epilog": null, "deprecated": false, "hidden": false, @@ -6274,7 +6274,7 @@ "full": "avr tests list", "is_group": false, "short_help": "Read the latest confirmed execution of JOB, including repeated tests.", - "help": "Read the latest confirmed execution of JOB, including repeated tests.\n\nNames, outcomes, recorded durations and available failure messages are\nshown together. A failed test is data, not a command error. Missing,\npending, partial and expired evidence are reported explicitly; an older\nexecution is never substituted. Complete describes collected reports.\n\nUse --output json for the complete response or --json FIELDS to select\nfields. Continue with --cursor when pagination.next_cursor is present.\nAPI refusal or a changed report fails the command without partial output.\n", + "help": "Read the latest confirmed execution of JOB, including repeated tests.\n\nNames, outcomes, recorded durations and available failure messages are\nshown together. A failed test is data, not a command error. Missing,\npending, partial and expired evidence are reported explicitly; an older\nexecution is never substituted. Complete describes collected reports.\n\nUse --json '*' for the complete response or --json FIELDS to select\nfields. Continue with --cursor when pagination.next_cursor is present.\nAPI refusal or a changed report fails the command without partial output.\n", "epilog": null, "deprecated": false, "hidden": false, @@ -6371,28 +6371,6 @@ "show_default": false, "default": null }, - { - "name": "output", - "kind": "option", - "opts": [ - "--output" - ], - "secondary_opts": [], - "type": "choice", - "required": false, - "multiple": false, - "nargs": 1, - "is_flag": false, - "envvar": null, - "help": "Output format. JSON includes status, context and the next cursor.", - "hidden": false, - "show_default": false, - "default": null, - "choices": [ - "text", - "json" - ] - }, { "name": "json_fields", "kind": "option", diff --git a/docs/starlight/index.md b/docs/starlight/index.md index 68445f0b..06ba582a 100644 --- a/docs/starlight/index.md +++ b/docs/starlight/index.md @@ -166,13 +166,13 @@ avr job logs --follow # tail one job in real time ```sh avr tests # names, outcomes, timings and failure messages -avr tests --output json # results with reporting state and pagination +avr tests --json '*' # results with reporting state and pagination avr tests --limit 500 # follow short pages up to the requested count avr tests --cursor '' # continue with pagination.next_cursor ``` -Use the Avrea job ID from `avr job list`; the repository is resolved from that -job. `avr tests list ` is equivalent. `--json FIELDS` and `--jq` also +Use the Avrea job ID (`job-` or `awj-`) from `avr job list`; the repository is +resolved from that job. `avr tests list ` is equivalent. `--json FIELDS` and `--jq` also work. Each invocation reads at most 1,000 occurrences and 20 result pages, preserving repeated tests and returning a cursor when more results remain. Missing or expired reports are distinct from a retained report containing zero diff --git a/docs/starlight/reference/tests.md b/docs/starlight/reference/tests.md index 0f596d1a..a481f7d4 100644 --- a/docs/starlight/reference/tests.md +++ b/docs/starlight/reference/tests.md @@ -9,8 +9,8 @@ Read a job's tests, timings and recorded failure messages. avr tests [OPTIONS] COMMAND [ARGS]... ``` -Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from -avr job list. +Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID +(job- or awj-) from avr job list. ## Subcommands @@ -27,7 +27,7 @@ shown together. A failed test is data, not a command error. Missing, pending, partial and expired evidence are reported explicitly; an older execution is never substituted. Complete describes collected reports. -Use --output json for the complete response or --json FIELDS to select +Use --json '*' for the complete response or --json FIELDS to select fields. Continue with --cursor when pagination.next_cursor is present. API refusal or a changed report fails the command without partial output. @@ -41,6 +41,5 @@ API refusal or a changed report fails the command without partial output. - --repo <TEXT> — Repository (owner/name or rep-ID). Defaults to the job's repository. - -L, --limit <INTEGER RANGE> — Maximum occurrences to return; short pages are followed, up to 20 result pages. _(default: `100`)_ - --cursor <TEXT> — Continue from a previous response's next_cursor. -- --output <CHOICE> — Output format. JSON includes status, context and the next cursor. _(choices: `text`, `json`)_ - --json <TEXT> — Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. - -q, --jq <TEXT> — Filter --json output through a jq expression. diff --git a/man/man1/avr-tests-list.1 b/man/man1/avr-tests-list.1 index 82978d0b..f1016131 100644 --- a/man/man1/avr-tests-list.1 +++ b/man/man1/avr-tests-list.1 @@ -9,7 +9,7 @@ Read the latest confirmed execution of JOB, including repeated tests. .PP Names, outcomes, recorded durations and available failure messages are shown together. A failed test is data, not a command error. Missing, pending, partial and expired evidence are reported explicitly; an older execution is never substituted. Complete describes collected reports. .PP -Use --output json for the complete response or --json FIELDS to select fields. Continue with --cursor when pagination.next_cursor is present. API refusal or a changed report fails the command without partial output. +Use --json '*' for the complete response or --json FIELDS to select fields. Continue with --cursor when pagination.next_cursor is present. API refusal or a changed report fails the command without partial output. .SH ARGUMENTS .TP \fIJOB\fR @@ -28,9 +28,6 @@ Maximum occurrences to return; short pages are followed, up to 20 result pages. \fB\-\-cursor\fR \fITEXT\fR Continue from a previous response's next_cursor. .TP -\fB\-\-output\fR \fICHOICE\fR -Output format. JSON includes status, context and the next cursor. (choices: text, json) -.TP \fB\-\-json\fR \fITEXT\fR Output JSON. Pass comma-separated field names, "*" for all fields, or "?" to list available fields. .TP diff --git a/man/man1/avr-tests.1 b/man/man1/avr-tests.1 index fb6d240a..a8fca011 100644 --- a/man/man1/avr-tests.1 +++ b/man/man1/avr-tests.1 @@ -7,7 +7,7 @@ avr-tests \- Read a job's tests, timings and recorded failure messages. .SH DESCRIPTION Read a job's tests, timings and recorded failure messages. .PP -Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID from avr job list. +Use avr tests JOB (or avr tests list JOB). JOB is an Avrea job ID (job- or awj-) from avr job list. .SH COMMANDS .TP list diff --git a/tests/test_tests.py b/tests/test_tests.py index 6a83e44d..7d6de936 100644 --- a/tests/test_tests.py +++ b/tests/test_tests.py @@ -10,6 +10,7 @@ ORG = "org-019a0000000070008000000000000001" REPO = "rep-019a0000000070008000000000000002" JOB = "job-019a0000000070008000000000000003" +NATIVE_JOB = "awj-019a0000000070008000000000000007" EXECUTION = "exe-019a0000000070008000000000000004" JOB_PATH = f"/orgs/{ORG}/jobs/{JOB}" RESULTS_PATH = f"/orgs/{ORG}/repos/{REPO}/jobs/{JOB}/tests" @@ -51,7 +52,7 @@ def test_job_short_and_empty_pages_continue_without_dropping_repeats(runner: Cli job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="first")) job_api.reply("GET", RESULTS_PATH, json=page([], cursor="second")) job_api.reply("GET", RESULTS_PATH, json=page([ROW, {**ROW, "outcome": "passed"}])) - result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--output", "json"]) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) assert result.exit_code == 0, result.output data = json.loads(result.output) assert data["organization_id"] == ORG @@ -87,17 +88,45 @@ def test_job_text_shows_failure_and_escapes_terminal_controls(runner: CliRunner, assert "bad\\r\\nline" in result.output assert "\x1b" not in result.output assert "Reporting" in result.output + assert "Complete describes the collected reports" in result.output + + +def test_job_text_shows_the_reporting_reason(runner: CliRunner, job_api: FakeApi) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW], reporting_state="partial", reason="upload_failed")) + result = runner.invoke(cli, ["tests", JOB, *SCOPE]) + assert result.exit_code == 0, result.output + assert "Reason: upload_failed" in result.stdout + assert "Complete describes" not in result.stdout @pytest.mark.parametrize("code", [403, 404, 409, 429, 503]) def test_job_page_failure_never_emits_partial_success(runner: CliRunner, job_api: FakeApi, code: int) -> None: job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="next")) job_api.reply("GET", RESULTS_PATH, status=code, json={"detail": "Report changed or unavailable"}) - result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--output", "json"]) - assert result.exit_code != 0 + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) + assert result.exit_code == 1 + assert f"HTTP {code}" in result.stderr assert not result.stdout +def test_results_not_found_says_what_to_check(runner: CliRunner, job_api: FakeApi) -> None: + job_api.reply("GET", RESULTS_PATH, status=404, json={"detail": "Not Found"}) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--cursor", "stale"]) + assert result.exit_code == 1 + assert "restart without it" in result.stderr + assert "Repository Tests are enabled" in result.stderr + + +def test_native_job_ids_are_read_like_job_ids(runner: CliRunner, api: FakeApi) -> None: + native_results = f"/orgs/{ORG}/repos/{REPO}/jobs/{NATIVE_JOB}/tests" + api.reply("GET", f"/orgs/{ORG}/jobs/{NATIVE_JOB}", json={"data": {"job_id": NATIVE_JOB, "repository_id": REPO}}) + api.reply("GET", native_results, json={**page([ROW]), "job_id": NATIVE_JOB}) + result = runner.invoke(cli, ["tests", NATIVE_JOB, *SCOPE, "--json", "job_id,data"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == {"job_id": NATIVE_JOB, "data": [ROW]} + assert len(api.sent("GET", native_results)) == 1 + + def test_explicit_repository_does_not_require_job_metadata(runner: CliRunner, api: FakeApi) -> None: api.reply("GET", RESULTS_PATH, json=page([ROW])) result = runner.invoke(cli, ["tests", "list", JOB, *SCOPE, "--repo", REPO, "--json", "data,status"]) @@ -156,34 +185,72 @@ def test_page_budget_returns_a_resume_cursor(runner: CliRunner, job_api: FakeApi [ {"execution_id": "exe-019a0000000070008000000000000005"}, {"status": "expired"}, + {"reporting_state": "partial"}, + {"reason": "upload_failed"}, + {"commit_sha": "c" * 40}, + {"observed_at": "2026-10-28T12:05:00Z"}, {"total_count": 4}, - {"job_id": "job-019a0000000070008000000000000006"}, ], ) def test_changed_job_pages_are_not_combined(runner: CliRunner, job_api: FakeApi, changes: dict[str, Any]) -> None: job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="next")) job_api.reply("GET", RESULTS_PATH, json=page([ROW], **changes)) result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) - assert result.exit_code != 0 + assert result.exit_code == 1 + assert "Job results changed while reading pages" in result.stderr + assert not result.stdout + + +@pytest.mark.parametrize("page_number", [0, 1]) +def test_a_page_for_another_job_is_rejected(runner: CliRunner, job_api: FakeApi, page_number: int) -> None: + other = {"job_id": "job-019a0000000070008000000000000006"} + pages = [page([ROW], cursor="next"), page([ROW])] + pages[page_number] = {**pages[page_number], **other} + for body in pages[: page_number + 1]: + job_api.reply("GET", RESULTS_PATH, json=body) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) + assert result.exit_code == 1 + assert "invalid job test results" in result.stderr + assert not result.stdout + + +def test_a_page_larger_than_requested_is_rejected(runner: CliRunner, job_api: FakeApi) -> None: + job_api.reply("GET", RESULTS_PATH, json=page([ROW, ROW, ROW])) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--limit", "2", "--json", "*"]) + assert result.exit_code == 1 + assert "invalid test result page" in result.stderr assert not result.stdout +def test_job_lookup_for_another_job_is_rejected(runner: CliRunner, api: FakeApi) -> None: + other = "job-019a0000000070008000000000000006" + api.reply("GET", JOB_PATH, json={"data": {"job_id": other, "repository_id": REPO}}) + result = runner.invoke(cli, ["tests", JOB, *SCOPE]) + assert result.exit_code == 1 + assert "returned a different job" in result.stderr + assert len(api.requests) == 1 + + @pytest.mark.parametrize( - "bad_page", + "bad_page,message", [ - page([ROW], cursor="next"), - page([ROW], cursor=" "), - page([ROW], pagination={"next_cursor": 17}), - page([ROW], data="invalid"), + (page([ROW], cursor="next"), "repeated a test-results cursor"), + (page([ROW], cursor=" "), "invalid next cursor"), + (page([ROW], cursor=""), "invalid next cursor"), + (page([ROW], cursor="x" * 1025), "invalid next cursor"), + (page([ROW], pagination={"next_cursor": 17}), "invalid next cursor"), + (page([ROW], data="invalid"), "invalid job test results"), ], ) def test_invalid_pagination_stops_without_partial_output( - runner: CliRunner, job_api: FakeApi, bad_page: dict[str, Any] + runner: CliRunner, job_api: FakeApi, bad_page: dict[str, Any], message: str ) -> None: job_api.reply("GET", RESULTS_PATH, json=page([ROW], cursor="next")) job_api.reply("GET", RESULTS_PATH, json=bad_page) result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) - assert result.exit_code != 0 + assert result.exit_code == 1 + assert message in result.stderr + assert "--cursor" not in result.stderr assert not result.stdout assert len(job_api.sent("GET", RESULTS_PATH)) == 2 @@ -191,27 +258,28 @@ def test_invalid_pagination_stops_without_partial_output( def test_cursor_resume_uses_the_exact_value(runner: CliRunner, job_api: FakeApi) -> None: job_api.reply("GET", RESULTS_PATH, json=page([ROW])) cursor = "x" * 900 - result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--cursor", cursor, "--output", "json"]) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--cursor", cursor, "--json", "*"]) assert result.exit_code == 0, result.output assert job_api.sent("GET", RESULTS_PATH)[0].url.params["cursor"] == cursor @pytest.mark.parametrize( - "args", + "args,exit_code", [ - ["list", "not-a-job"], - [JOB, "--cursor", " "], - [JOB, "--cursor", "x" * 1025], - [JOB, "--limit", "0"], - [JOB, "--limit", "1001"], - [JOB, "--json", "capture_id"], - [JOB, "--output", "text", "--json", "*"], - [JOB, "--jq", ".data"], + (["list", "not-a-job"], 2), + (["list", "awj-019a"], 2), + ([JOB, "--cursor", " "], 2), + ([JOB, "--cursor", "x" * 1025], 2), + ([JOB, "--limit", "0"], 2), + ([JOB, "--limit", "1001"], 2), + ([JOB, "--output", "json"], 2), + ([JOB, "--jq", ".data"], 2), + ([JOB, "--json", "capture_id"], 1), ], ) -def test_invalid_options_make_no_requests(runner: CliRunner, api: FakeApi, args: list[str]) -> None: +def test_invalid_options_make_no_requests(runner: CliRunner, api: FakeApi, args: list[str], exit_code: int) -> None: result = runner.invoke(cli, ["tests", *args, *SCOPE]) - assert result.exit_code != 0 + assert result.exit_code == exit_code assert api.requests == [] @@ -221,6 +289,7 @@ def test_invalid_options_make_no_requests(runner: CliRunner, api: FakeApi, args: ("expired", None, "Missing or expired reports"), ("not_recorded", None, "Missing or expired reports"), ("available", 0, "retained report contains zero"), + ("available", 3, "No more results."), ], ) def test_empty_results_explain_availability( @@ -231,13 +300,16 @@ def test_empty_results_explain_availability( assert result.exit_code == 0, result.output assert message in result.stdout assert state in result.stdout + assert ("Missing or expired" in result.stdout) == (state != "available") assert len(job_api.sent("GET", RESULTS_PATH)) == 1 def test_wrong_organization_does_not_fall_back_to_repository_scan(runner: CliRunner, api: FakeApi) -> None: api.reply("GET", JOB_PATH, status=404, json={"detail": "Job not found"}) result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) - assert result.exit_code != 0 + assert result.exit_code == 1 + assert "read the job" in result.stderr + assert "pass --org" in result.stderr assert not result.stdout assert len(api.requests) == 1 @@ -245,7 +317,7 @@ def test_wrong_organization_does_not_fall_back_to_repository_scan(runner: CliRun def test_json_preserves_raw_test_names_and_failure_text(runner: CliRunner, job_api: FakeApi) -> None: row = {**ROW, "name": "line\nbreak", "failure_summary": "\x1b[31merror"} job_api.reply("GET", RESULTS_PATH, json=page([row])) - result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--output", "json"]) + result = runner.invoke(cli, ["tests", JOB, *SCOPE, "--json", "*"]) assert result.exit_code == 0, result.output assert json.loads(result.stdout)["data"] == [row]