From c4bb868b9976f896a2bd81bf89469f0e67b042a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Sat, 5 Sep 2026 11:56:47 +0200 Subject: [PATCH] docs(cli): align guidance with cloud execution --- .github/ISSUE_TEMPLATE/bug_report.yml | 8 +-- .github/ISSUE_TEMPLATE/regression_report.yml | 7 ++- README.md | 28 ++++++--- examples/README.md | 36 ++++++++--- skills/testerarmy-cli/SKILL.md | 64 ++++++++------------ skills/testerarmy-cli/agents/openai.yaml | 4 +- 6 files changed, 81 insertions(+), 66 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index dc1ca9f..757a2f4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -12,7 +12,7 @@ body: attributes: label: Exact command description: Include full `ta ...` command. - placeholder: ta run tests/01-landing-page.md --debug + placeholder: ta tests run --env staging --wait --json validations: required: true - type: input @@ -42,9 +42,9 @@ body: - type: input id: artifacts attributes: - label: Artifact path - description: Path under `.testerarmy//` - placeholder: .testerarmy/2026-03-09T12-10-10-123Z/ + label: Run ID or saved JSON path + description: Include the cloud run ID, or the saved CLI response if no run was created. + placeholder: or result.json validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/regression_report.yml b/.github/ISSUE_TEMPLATE/regression_report.yml index 1802f39..9cbdd2a 100644 --- a/.github/ISSUE_TEMPLATE/regression_report.yml +++ b/.github/ISSUE_TEMPLATE/regression_report.yml @@ -7,7 +7,7 @@ body: id: command attributes: label: Exact command - placeholder: ta run tests/03-project-create.md --json + placeholder: ta tests run --env staging --wait --json validations: required: true - type: input @@ -27,8 +27,9 @@ body: - type: input id: artifacts attributes: - label: Failing artifact path - placeholder: .testerarmy/2026-03-09T12-10-10-123Z/ + label: Failing run ID or saved JSON path + description: Include the cloud run ID, or the saved CLI response if no run was created. + placeholder: or result.json validations: required: true - type: textarea diff --git a/README.md b/README.md index 13ba39c..46fd53c 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,12 @@ ## AI QA agent that clicks through your website like a real human. -TesterArmy CLI (`testerarmy` / `ta`) is an agent-first QA runner. +TesterArmy CLI (`testerarmy` / `ta`) manages QA coverage in the TesterArmy dashboard +and queues runs that execute in TesterArmy cloud. -- Run browser checks from plain prompts. -- Run reusable markdown scenarios (`tests/*.md`). -- Get deterministic pass/fail output plus local artifacts. +- Create and organize saved tests with plain-language steps. +- Run browser and mobile checks against saved environments. +- Wait for results and inspect run transcripts through JSON output. - Feed concrete validation back to coding agents. Start here: @@ -61,12 +62,19 @@ Or run without install: npx testerarmy --help ``` -Run a test scenario: +Find a project and its saved tests, then wait for a cloud run: ```bash -ta run examples/tests/01-landing-page.md --url "http://localhost:3000" +ta projects list --json +ta tests list --project --json +ta tests run --wait --json ``` +Use IDs returned by the list commands. Without `--wait`, a successful command +only confirms that the run was queued. To validate development changes, save a +cloud-reachable preview or tunnel URL as a project environment and select it +with `--env `. + ## Skill Installation Use skills CLI install: @@ -77,7 +85,8 @@ npx skills add tester-army/cli ## Examples -Use starter examples in `examples/`. +Use the markdown scenarios in [`examples/`](examples/README.md) as authoring +references when creating saved dashboard tests with `ta tests create`. - `examples/TESTER.md` - `examples/tests/01-landing-page.md` @@ -85,10 +94,11 @@ Use starter examples in `examples/`. - `examples/tests/03-project-create.md` - `examples/prompts/ad-hoc-regression.md` -Run a full smoke batch: +Run a saved smoke group: ```bash -ta run examples/tests/ --url "http://localhost:3000" --parallel 3 +ta groups list --project --json +ta tests run --group --project --wait --json ``` ## Contributing diff --git a/examples/README.md b/examples/README.md index 72b736f..df9f218 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,8 @@ # Examples -Starter examples for real `ta` usage. +Starter scenarios for creating saved tests with `ta`. The markdown files are +authoring references: translate their actions and assertions into dashboard test +steps, then run the saved tests in TesterArmy cloud. ## Layout @@ -15,17 +17,33 @@ examples/ └── 03-project-create.md ``` -## Run +## Create and run saved coverage + +Find your project, then create a test based on a scenario: + +```bash +ta projects list --json +echo '{"title":"Landing page","steps":[{"title":"Open the landing page","type":"act"},{"title":"The main heading and primary call to action are visible","type":"assert"}]}' | ta tests create --project --json +``` + +Use IDs returned by the commands. For development validation, save a +cloud-reachable preview or tunnel URL as an environment: ```bash -export TESTERARMY_TARGET_URL="http://localhost:3000" +ta projects environments-create --name Development --url https://dev.example.com --json +ta tests run --env development --wait --json +``` -# one test -ta run examples/tests/01-landing-page.md +For a saved suite, find its group and run the group: -# whole directory -ta run examples/tests/ --parallel 3 +```bash +ta groups list --project --json +ta tests run --group --project --env development --wait --json +``` -# ad hoc prompt -ta run "$(cat examples/prompts/ad-hoc-regression.md)" --url "$TESTERARMY_TARGET_URL" +`--wait` waits for validation to finish; without it, success only confirms that +the run was queued. Save JSON output locally when an artifact is useful: + +```bash +ta tests run --env development --wait --json > result.json ``` diff --git a/skills/testerarmy-cli/SKILL.md b/skills/testerarmy-cli/SKILL.md index f565f5f..a91264b 100644 --- a/skills/testerarmy-cli/SKILL.md +++ b/skills/testerarmy-cli/SKILL.md @@ -1,6 +1,6 @@ --- name: testerarmy-cli -description: Use TesterArmy CLI to create, organize, and run dashboard-managed QA tests. Prefer saved tests, groups, project context, credentials, and remote runs over one-off local prompts. Trigger when defining regression coverage or adding QA flows. +description: Use TesterArmy CLI to create, organize, and run dashboard-managed QA tests in TesterArmy cloud. Trigger when defining regression coverage, validating a flow, or inspecting remote test results. license: MIT metadata: author: TesterArmy @@ -12,10 +12,9 @@ metadata: Create dashboard-managed QA coverage with `ta` / `testerarmy`. The CLI is a control plane for the TesterArmy dashboard: manage projects, -environments, credentials, and saved tests, and queue remote runs that execute -in TesterArmy cloud. `ta tests run` queues remote dashboard runs by default. -Use local execution (`ta run "..."` or `ta tests run --local`) only for quick -exploration. +environments, credentials, and saved tests, and queue runs that execute in +TesterArmy cloud. Use a cloud-reachable preview or tunnel environment to +validate development changes. ## When to Use @@ -233,12 +232,8 @@ Common groups: `Smoke`, `Auth`, `Core journeys`, `Mobile smoke`. ## Runs -Modes: - -- Default/remote: queues the saved test in TesterArmy cloud; results show up in the dashboard. -- `--local`: fetches the saved test, then runs it on this machine. - -Remote validation (default): +`ta tests run` queues the saved test in TesterArmy cloud. Add `--wait` for +validation; without it, exit code 0 only confirms that the run was queued. ```bash ta tests run --wait --json @@ -248,24 +243,28 @@ ta tests run --platform ios --app-id --wait --json ta tests run --platform android --app-id --wait --json ``` -Local debugging: +Development targets must be reachable from the cloud browser. Save a preview +or tunnel URL as an environment, then select it on the run: ```bash -ta tests run --local --url http://localhost:3000 --json -ta tests run --group --project --local --url http://localhost:3000 --parallel 3 --json +ta projects environments-create --name Development --url https://dev.example.com --json +ta tests run --env development --wait --json ``` -Defaults: +Run options: -- No flag: remote cloud execution (dashboard run). -- `--local`: local browser execution on this machine. -- `--wait`: wait for remote results. -- `--env `: target a saved project environment (remote runs pass its ID; local runs use its URL). -- Local-only flags such as `--headed`, `--browser`, `--timeout`, and `--system-prompt-file` are ignored for remote runs. -- Local runs can use `--headed`, `--browser chrome|firefox|safari`, `--timeout`, `--output`, `--debug`, and `--system-prompt-file`. -- Remote runs can use `--wait-timeout`, `--wait-interval`, `--output`, `--platform web|ios|android`, `--app-id`, and `--project-environment-id`. -- Remote group runs can use `--environment production|staging|preview`. -- Remote single-test runs can use `--mode fast|deep`. +- `--env ` or `--project-environment-id `: select a saved project environment; do not combine them. +- `--url `: target URL override for group runs; do not combine it with `--env`. +- `--wait-timeout ` and `--wait-interval `: control how long to wait and the initial polling interval. +- `--platform web|ios|android`, `--app-id`, and `--device-model`: select the runtime and mobile build/device. +- Group runs can use `--environment production|staging|preview` to label the run origin. +- Single-test runs can use `--mode fast|deep`. + +Save a local result artifact with shell redirection: + +```bash +ta tests run --env development --wait --json > result.json +``` Runs: @@ -273,6 +272,8 @@ Runs: ta runs list --project --json ta runs get --json ta runs wait --timeout 600000 --json +ta runs messages --json +ta runs telemetry --json ta runs cancel --json ``` @@ -290,21 +291,6 @@ ta tests run --platform android --app-id --wait --json Supported uploads: `.app`, `.app.zip`, `.zip` for iOS Simulator apps and `.apk` for Android. Use `--remove-after ` to auto-delete uploaded apps. -## Local Prompt - -`ta run ` runs an ad hoc local browser test: - -```bash -ta run "check pricing CTA" --url https://example.com --json -``` - -Useful flags: `--headed`, `--browser chrome|firefox|safari`, `--timeout`, -`--output`, `--debug`, and `--system-prompt-file`. - -Use only to explore before creating or updating dashboard tests. Use -`ta tests create` and `ta tests run --group` for durable workflows that run -remotely on the dashboard. - ## Reporting Report: diff --git a/skills/testerarmy-cli/agents/openai.yaml b/skills/testerarmy-cli/agents/openai.yaml index d88cd81..9269e14 100644 --- a/skills/testerarmy-cli/agents/openai.yaml +++ b/skills/testerarmy-cli/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "TesterArmy CLI" - short_description: "Validate changes with ta markdown tests, artifacts, and strict pass/fail reporting" - default_prompt: "Use $testerarmy-cli to run targeted qa validation after code changes. Default to ta run with --json, add --output when needed for reuse, and report exact command + outcome." + short_description: "Run saved QA tests in TesterArmy cloud" + default_prompt: "Use $testerarmy-cli to validate this flow with saved cloud tests, run ta tests run with --wait --json against a saved environment, and report the command, run ID, and outcome."