From aa7e1a6edb20de950627b6405759404cfed9203a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E4=B8=80=E4=B9=8B?= Date: Thu, 13 Aug 2026 14:13:23 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20e2e:=20=E6=9C=AC=E5=9C=B0=E9=AA=8C?= =?UTF-8?q?=E8=AF=81=E5=A2=9E=E5=8A=A0=E6=89=8B=E5=B7=A5=E9=A9=B1=E5=8A=A8?= =?UTF-8?q?=E8=BD=A8=E9=81=93=EF=BC=88pnpm=20serve=20+=20pnpm=20drive?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一次性验证原来只有一条路：写一条 scratch spec，再整轮冷启跑一遍。改一个选择器就 要重付一次构建 + 播种，而且 spec 只断言事先想到的那几件事——上一轮控制台验证就是 这么只证明了元素存在、没证明视觉一致（docs/specs/2026-08-12-console-design-fidelity.md）。 新轨道与桌面端 agentre/e2e/drive.mjs 同形：两仓独立、不共享代码，但同一套命令、 同一套选择器 DSL、同一份 drive.log。 - pnpm serve：run-e2e-web.mjs 起完环境后撑住不退，不跑 spec。这套环境（真 server + 播种账号 + 在线 agentred）原来只活在一次 spec 运行的生命周期里，看完就随 workDir 一起消失。Ctrl-C 仍精确删掉本次播的行。 - pnpm drive up/…/down：一次调用一个动作，打在活过单次调用的浏览器上。snapshot 列出 屏幕上可定位的元素，sql 是只读的独立 oracle，logs 读 server 与 agentred 自己的日志； 每次调用（含失败）当场追加进 scratch/<场景>/logs/drive.log。 四条护栏是机械的，不靠记：只驱动本次 target 自己的 origin；oracle 只读；证据只落在 场景目录内；默认无头。库口令只经 MYSQL_PWD 传给 mysql 客户端，不进 argv——否则同机 ps 就能看到整条口令。 WEBE2E_CONFIG_DIR 是 source: etcd 那类配置的出口：cago 在 source 非 file 时会把整个 配置源换掉，http.address 也来自 etcd，runner 起的 server 会去绑 8443 而不是它挑的空闲 端口，隔离直接作废。 44 条 harness 单测挂进 playwright.runner.config.ts，随 make test-e2e 进 CI。 --- .gitignore | 2 + AGENTS.md | 4 +- docs/README.md | 2 +- .../verification-report-template.md | 138 ++-- docs/testing.md | 6 +- docs/verification.md | 135 ++-- e2e/.prettierignore | 3 + e2e/README.md | 159 ++++- e2e/drive.mjs | 603 ++++++++++++++++++ e2e/lib/drive-target.mjs | 201 ++++++ e2e/package.json | 4 +- e2e/playwright.runner.config.ts | 8 +- e2e/run-e2e-web.mjs | 145 ++++- e2e/web/drive-cli.spec.ts | 248 +++++++ e2e/web/runner-config-dir.spec.ts | 45 ++ e2e/web/runner-serve.spec.ts | 103 +++ 16 files changed, 1642 insertions(+), 164 deletions(-) create mode 100644 e2e/drive.mjs create mode 100644 e2e/lib/drive-target.mjs create mode 100644 e2e/web/drive-cli.spec.ts create mode 100644 e2e/web/runner-config-dir.spec.ts create mode 100644 e2e/web/runner-serve.spec.ts diff --git a/.gitignore b/.gitignore index e227542b..0ee92a2d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ # 目录不存在时 `pnpm scratch` 直接报错。 /e2e/scratch/* !/e2e/scratch/.gitkeep +# drive 的浏览器状态与 serve 的交接件（含一次性会话 Cookie）。都是本次运行私有的。 +/e2e/.drive/ /e2e/node_modules/ /e2e/test-results/ /e2e/playwright-report/ diff --git a/AGENTS.md b/AGENTS.md index 99ebbe42..9f55f3f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,7 @@ Workspace-wide facts live in [`../AGENTS.md`](../AGENTS.md). | Adding an endpoint / service / repository | [docs/architecture.md](docs/architecture.md) | Layering, dependency direction, "how to add an X" | | Adding a scheduled task, or assuming only one replica is running | [docs/architecture.md](docs/architecture.md) | What you may assume about process-local state, how to add a cron job | | Writing any test | [docs/testing.md](docs/testing.md) | What to write per layer, sqlmock vs mockgen, build tags, the guard tests | -| Confirming a change actually works | [docs/verification.md](docs/verification.md) | The twin e2e tracks, scratch workflow, report rules | +| Confirming a change actually works | [docs/verification.md](docs/verification.md) | Which form a one-off check takes — drive it by hand before writing a spec — scratch workflow, verdicts and report rules | | Touching the frontend | [docs/design.md](docs/design.md) | Colour tokens, dark/light, responsive, i18n, the new-page recipe | | Deploying, or changing the image/chart/workflow | [deploy/README.md](deploy/README.md) | Docker and Kubernetes deployment, chart values, etcd seeding, the Gitea pipeline | | Adding a log line, metric or span | [docs/observability.md](docs/observability.md) | Log levels and fields, metrics, traces | @@ -75,7 +75,7 @@ internal/ web/ embed.FS SPA mount, /v1 passthrough migrations/ gormigrate; append-only frontend/ React 19 + Vite + Tailwind + shadcn -e2e/ twin tracks: committed smoke + gitignored scratch +e2e/ committed smoke + on-demand full chain + `drive` (hand-driven) + gitignored scratch ``` Auth has three shapes, and which one a route uses is visible in `internal/api/router.go`: diff --git a/docs/README.md b/docs/README.md index 1d37ba45..4964510f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ non-negotiables, and the routing table that says which of these to open when. | [develop.md](develop.md) | Commands, repo layout, enforced rules + exemptions, migrations, commit flow | | [architecture.md](architecture.md) | Layering, dependency direction, "how to add an X" | | [testing.md](testing.md) | What to test per layer, sqlmock vs mockgen, build tags, guard tests | -| [verification.md](verification.md) | Twin e2e tracks, scratch workflow, report rules | +| [verification.md](verification.md) | Choosing the form a one-off verification takes, per surface; scratch workflow, verdicts and report rules | | [design.md](design.md) | Tokens and the canvas↔code name mapping, type/spacing/radius scales, the auth shell and page skeleton, dark/light, responsive, i18n, new-page recipe | | [observability.md](observability.md) | Logging, metrics, traces | | [documentation.md](documentation.md) | Who owns which fact, how docs stay true | diff --git a/docs/references/verification-report-template.md b/docs/references/verification-report-template.md index 89cc51e3..80cfddb3 100644 --- a/docs/references/verification-report-template.md +++ b/docs/references/verification-report-template.md @@ -1,77 +1,127 @@ -# Verification report template + -Copy to `e2e/scratch//report.md` and fill in **as you go**, not afterwards. -Delete sections that do not apply — an empty heading is noise. The rules behind this -template are in [../verification.md](../verification.md). +# Verification: ---- +## Mode -# Verification: +`verifying a change` | `reproducing a bug` -- **Date**: -- **Change**: -- **Verdict**: ✅ works / ⚠️ works with caveats / ❌ does not work +## Goal / problem -## What I claimed to have built + -One or two sentences. The claim being tested — not a changelog. +## Environment -## How I verified it + -Environment (mocked / real MySQL + Redis / against staging), and the commands run. +- Form and entry point: `` +- Backend and data: `` +- Build under test: `` +- Form factors driven: `` -```bash -go run ./cmd/server -cd e2e && E2E_SCRATCH_AUTOSTART=1 pnpm scratch -``` +## Verdict + + + +| # | Requirement / bug claim | Verdict | Real / substituted | How observed | Check it yourself | +|---|---|---|---|---|---| +| V1 | `` | holds / does not hold / not observed | real, or `substituted: — ` | `` | `` | + +Summary: . + +| Label | Use it when | Requires | +|---|---|---| +| `holds` | you observed the behaviour at runtime | the deciding observation, and how a reader reaches it | +| `does not hold` | you observed it failing, or the bug reproducing | the failing output, assertion diff or error screenshot | +| `not observed` | you never reached the check | what stopped it | + +An unreached check is never `holds`; a run that verified two of three claims is reported as two of three. + +## Authorization + + + +| # | Substitute or effect | The user's authorization, verbatim | +|---|---|---| +| V1 | `` | `` | + +## Reproduction steps + + + +1. `` + +- [ ] Asserts the **expected** behaviour → currently **red**, turns green when fixed +- [ ] Asserts the **current buggy** behaviour → currently **green**, must be flipped when fixed -## Evidence +## Acceptance evidence -Inline, in whatever form fits what is being verified. One scroll to a verdict. + -### +### V1 · `` -```bash +```console $ curl -s localhost:8443/v1/healthz {"status":"ok","db_ping":true,"redis":true} -# exit 0 +$ echo $? +0 ``` -What this proves: . +. -### +**Independent oracle.** Do not take the UI's word for it — read the data or the logs directly: + +```sql +SELECT id, status, approved_at FROM device_flow_codes WHERE user_code = ''; +-- 1 row, status=approved, approved_at set +``` + + | Before | After | -| --- | --- | -| `![before](screenshots/before.png)` | `![after](screenshots/after.png)` | +|---|---| +| `![before](screenshots/v1-before.png)` | `![after](screenshots/v1-after.png)` | -(Drop the backticks when you fill this in — they are here so the template itself -does not carry two permanently broken image links.) +## Evidence index -What this proves: . +- Commands/logs: `` +- Resources/data snapshots: `` +- Screenshots/video: `` -### Independent oracle +A scenario with no `screenshots/` is the right shape for an API, migration or daemon run. -Do not take the UI's word for it — check the data or the logs directly. +## Persistent data changes -```sql -SELECT id, status, approved_at FROM device_flow_codes WHERE user_code = 'A4F-7Q2'; --- 1 row, status=approved, approved_at set -``` + + +| Change | Forward | Backward/backup | Before/after query | +|---|---|---|---| +| `` | `` | `` | `` | + +Dataset: ``. **Green on an empty database is not evidence.** -## What I could NOT verify +## Execution record -Be specific about which part and why. "Everything checked out" with a gap left unstated is -the failure mode this section exists to prevent. +| Step | Status | Evidence/blocker | +|---|---|---| +| `` | pending / passed / failed / blocked | `` | ## Known issues found along the way -Anything you noticed but did not fix — with enough detail for someone to pick it up. -Per the workspace rules, unrelated problems get reported, not fixed on the side. + -## If this reproduces a bug +## Integrity and cleanup -State the polarity of the assertion explicitly: +- Initial/final HEAD: `` / `` +- Final `git status --porcelain=v1`: `` +- Created artifacts, processes and external data, and how each was cleaned up: `` +- Redaction performed: `` -- [ ] Asserts the **expected** behaviour → currently **red**, turns green when fixed -- [ ] Asserts the **current buggy** behaviour → currently **green**, must be flipped when fixed +## Evidence rules + +- Every `holds` names how the target was driven — command, or launch command plus steps — and the deciding observation. +- Where a claim changes state beyond the driven surface, that observation is an independent read with its own command: the database or the logs, not the UI. +- Embed decisive text and images inline; one scroll should reach a verdict. Bare links are for archives and binaries only, each with a note on what it holds. +- Paste terminal output as text. Screenshotting a terminal manufactures evidence instead of capturing it. +- Keep failed and unchecked steps visible. Redact tokens, secrets, real email addresses and session cookies before saving, and again before embedding. +- Keep every path relative to this file; the scenario directory, not `report.md` alone, is what you hand to a reviewer. diff --git a/docs/testing.md b/docs/testing.md index 9b2fd5fb..135c4809 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -66,7 +66,7 @@ the package, which is the path by which a key would actually leak. `git grep '//go:build'` returns nothing, and it should stay that way. Anything that seems to need a tag is either (a) a test-only asset, which the package-isolation trick above handles, or (b) a test needing external infrastructure — which gets its **own entry point** -(`make test-e2e`, or a scratch script per [verification.md](verification.md)), never a tag. +(`make test-e2e`, or a hand-driven run per [verification.md](verification.md)), never a tag. A target you did not run is obvious; a tagged-out test is invisible. ## Migrations are deliberately untested @@ -87,8 +87,8 @@ make dev # migrations run at startup against db.dsn; watch mysql --host --user --password -e 'SHOW TABLES' # read tables back directly ``` -Write that check up under `e2e/scratch/` per [verification.md](verification.md) — for -migrations the evidence is the table list, not a screenshot. +Write that check up under `e2e/scratch//` per [verification.md](verification.md) — +for migrations the evidence is the table list, not a screenshot, and nothing needs authoring. **What is untested is the DDL, not the runner.** `migrations/migrations_test.go` does use sqlmock, on the named-lock wrapper `withMigrationLock` that serialises concurrently diff --git a/docs/verification.md b/docs/verification.md index 46829bcc..e361ab41 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -1,107 +1,82 @@ # Verification -Tests prove the code does what you told it to. Verification answers a different question: -**"I just finished X — does it actually work?"** This document owns how you answer that, -and how you write down the answer. +Tests prove the code does what you told it to. Verification answers a different question: **"I just finished X — does it actually work?"** This route owns the workflow and the report rules; the harness — configs, ports, hermetic guarantees, what earns a place in the smoke suite — is [`../e2e/README.md`](../e2e/README.md)'s. -The mechanics of the two tracks — configs, ports, hermetic guarantees, what earns a place -in the smoke suite — live in [`../e2e/README.md`](../e2e/README.md). This document owns the -**workflow and the report rules**. +## When to skip this route -## The two tracks +Use targeted committed tests alone when they fully observe the changed logic ([testing.md](testing.md#the-cycle)). Use this route when real HTTP, database, session or cross-process wiring is needed, or when reproducing a runtime-only bug. It does not replace TDD: a reproduction confirms the bug is real and still owes the committed failing test. -| | Smoke (`e2e/*.spec.ts`) | Scratch (`e2e/scratch/`) | -| --- | --- | --- | -| Committed | Yes, runs in CI | No — gitignored | -| Question | "Did anything basic break?" | "Does the thing I just built work?" | -| Lifetime | Permanent | Disposable | -| Bar | Very high | None — write one whenever | - -**When in doubt, use scratch.** Promoting a scratch spec into the smoke suite is a separate, -deliberate decision. Things dropped into smoke because they were handy are what make a smoke -suite slow and flaky, and once it is flaky people stop believing it. +Verification is not how the smoke suite grows. Promotion is a separate, deliberate decision — things dropped into smoke because they were handy are what make it slow and flaky, and once it is flaky people stop believing it ([`../e2e/README.md`](../e2e/README.md#what-earns-a-place-in-the-smoke-track)). ## Workflow -```bash -mkdir -p e2e/scratch/ -# write report.md FIRST — see below -cd e2e && pnpm scratch -``` +1. Run `make lint` and the targeted tests; run `make test` only when the blast radius is not confirmed local or a gate requires it. +2. Build/start the drivable target. Only the target starts here: MySQL and Redis come from the gitignored `configs/config.yaml` (`configs/config.example.yaml` is the template), so `db.dsn` and `redis.addr` decide what a run actually writes to — a service it does not configure is asked for, not arranged around. +3. Choose the cheapest form that observes the contract, and put everything it produces under gitignored `e2e/scratch//`: -`` is a lowercase hyphenated slug. **Where you are verifying against an approved -spec, use that spec's slug**, so the evidence and the spec are findable from each other. + | To reach and observe the target | You author | + |---|---| + | an existing command or entry point suffices, and it neither depends on nor writes your own machine state | nothing — drive it yourself and read the oracle | + | it needs a specific launch, isolated state or real-target configuration, and the observation is one-off | a launcher that stops at the target; drive it yourself | + | the sequence must be replayed, or timing/concurrency is the contract | a full asserting spec | -Needing a real backend (real device flow, migrations, session cookies) — the server -takes its MySQL and Redis from `configs/config.yaml`, so point that at your own -instances: + This project: -```bash -go run ./cmd/server # real backend on :8443 -cd e2e && E2E_SCRATCH_AUTOSTART=1 pnpm scratch -``` + | Change lands in | Reach it with | You author | Oracle | + |---|---|---|---| + | HTTP API, auth, device flow, session cookies | `go run ./cmd/server`, then `curl` against `:8443` | nothing | a read-only SQL query against `db.dsn`, or the server log | + | a migration | the forward command against a database holding real existing rows | nothing | the same query before and after, side by side | + | web UI rendering only — layout, copy, theme, anything reachable logged out | `make dev`, then `pnpm drive up --base http://127.0.0.1:5174` | nothing | the screenshot, in **both** form factors (`drive viewport`) | + | web UI behind auth, or touching real data | `cd e2e && pnpm serve` — a seeded account, already signed in — then `pnpm drive up` ([`../e2e/README.md`](../e2e/README.md#driving-by-hand-pnpm-serve--pnpm-drive)) | nothing | `drive sql`, plus the screenshots and `logs/drive.log` the run wrote | + | replay, timing, or both form factors at once | the scratch track | a full spec | the spec's assertions | + | the desktop app and a browser on one agentred | `pnpm dual` ([`../e2e/README.md`](../e2e/README.md#the-dual-end-run-pnpm-dual--the-desktop-app-and-a-browser-on-one-agentred)) | per that suite | that suite's assertions | -What that env var does is in -[`../e2e/README.md`](../e2e/README.md#needing-a-real-backend). + Two facts decide the rows. The smoke track mocks the API; a scratch run reaches the real backend by simply **not** calling the `mock*` helpers from `fixtures/app.ts` — mocking is opt-in, there is no global switch to turn off, so calling one is a deliberate substitution the verdict row names. And every spec runs against **both** `desktop-chromium` and `mobile-chromium`: a desktop-only pass gives no signal about mobile layout. -## Report rules + Reuse the harness for isolation and the oracle, not its mocks. In every form one observation comes from a path the driven surface does not share — read the database or the logs directly. Asserting the UI says "approved" does not prove the row was written; a failed write behind a cheerful UI is the exact failure this catches. -**Create `report.md` before the run and fill it in as you go.** A report reconstructed -afterwards from memory records what you believe happened, which is exactly the thing under -question. The template is -[references/verification-report-template.md](references/verification-report-template.md). +4. Before running, create `report.md` from [references/verification-report-template.md](references/verification-report-template.md); update it as evidence arrives. A report reconstructed afterwards from memory records what you believe happened, which is the thing under question. +5. Record how the target was driven, exit codes where the form produces them, deciding runtime observations, gaps and shortest user reproduction steps. `drive` already appends every action and its outcome to `e2e/scratch//logs/drive.log` and writes screenshots into `screenshots/` — the report cites those, it does not restate them. +```bash +cd e2e && pnpm serve # seeded real environment, held open until Ctrl-C +export AGENTRE_VERIFY_SCENARIO= # every drive call records into this scenario +pnpm drive up # a browser that outlives each command, already signed in +pnpm drive snapshot # what is on screen, and how to address it +pnpm drive click "testid=nav-devices" && pnpm drive shot 01-devices +pnpm drive sql "select status from device_flow_codes where user_code = ''" + +go run ./cmd/server # real backend on :8443, for curl and the SQL oracle +cd e2e && E2E_SCRATCH_AUTOSTART=1 pnpm scratch # the spec form: frontend on :5199, /v1 proxied to :8443 +cd e2e && pnpm scratch --project=desktop-chromium -g "" ``` -e2e/scratch/<task-name>/ -├── report.md -├── screenshots/ -├── videos/ -└── resources/ -``` - -**Evidence form follows what is being verified.** This is not "there must be pictures": -| Verifying | Evidence | -| --- | --- | -| UI behaviour or layout | Screenshot, one sentence on what it proves | -| A visual change | Two-column before/after table | -| A flow across steps | Recording, plus key still frames inline | -| An API or CLI result | The command in a code block, its exit code, the deciding output lines | -| A data effect | The query and its result — read the database directly, not through the UI | -| Logs / errors | The relevant lines, trimmed to what decides it | +**Drive it before you write a spec.** A spec only asserts what you thought of in advance, and a one-line change costs a whole cold run; driving shows you what the page actually looks like now. That distinction is not theoretical here — the previous console round asserted elements existed and shipped a UI that did not match the design (`docs/specs/2026-08-12-console-design-fidelity.md`). Write a spec when the sequence must be **replayed**, not to look at something once. -A scenario with no UI holding only `report.md`, `logs/` and `resources/` is the right shape. -Screenshotting a terminal manufactures evidence instead of capturing it — paste the text. +For acceptance against a spec, `<scenario>` is that spec's slug, so the evidence and the spec are findable from each other. Extract each requirement into one verdict row and evidence section. Verdict labels are `holds`, `does not hold`, `not observed`, and they live only in the verdict table. -**Evidence goes inline, not linked out.** One scroll should reach a verdict. Bare links are -for archives and binaries only. +For bug reproduction, state whether the reproduction asserts the expected behaviour (stays red until the fix lands) or the current buggy behaviour (green now, must be flipped when you fix it). An assertion whose polarity is undocumented becomes meaningless within a week. Choosing a form that authors nothing does not remove the committed failing test. -**Query an independent oracle.** Asserting the UI says "approved" does not prove the row was -written. Check the database or the logs directly as well — a failed write behind a cheerful -UI is the exact failure this catches. +Never weaken an assertion, skip a failed step or describe red as green. If you could not verify part of it, say which part and why — an unverified claim presented as verified stops anyone else from checking. If you worked around something rather than fixing it, that goes in the report too. Obtain authorization before destructive or external side effects, and before substituting a mock for a real dependency; the verdict row then names what stood in and what it does not cover. -**Redact before pasting.** Tokens, secrets, real email addresses, session cookies. +## What the harness enforces for you -## The honesty clause +These are mechanical, in [`../e2e/lib/drive-target.mjs`](../e2e/lib/drive-target.mjs) — not conventions you have to remember: -**Never describe red as green.** +- **Only this run's own origin is ever driven.** Any other URL is refused, including another port on your own machine — most likely the `make dev` you are actually working in. +- **The oracle is read-only.** `drive sql` takes only `SELECT` / `WITH` / `EXPLAIN` / `SHOW`. +- **Evidence stays in its scenario directory**, so the report's relative links hold. +- **Nothing takes over your screen**: the browser is headless unless you ask for `--headed`. -- If it failed, say so, and show the failure. -- If you could not verify part of it, say which part and why — an unverified claim presented - as verified is worse than an admitted gap, because it stops anyone else from checking. -- When reproducing a bug, state which contract your scratch assertion encodes: - **the expected behaviour** (so it stays red until the fix lands) or **the current buggy - behaviour** (so it is green now and must be flipped when you fix it). An assertion whose - polarity is undocumented becomes meaningless within a week. -- If you worked around something rather than fixing it, that goes in the report too. +The same driver, commands and `drive.log` exist in the desktop repo (`agentre/e2e/drive.mjs`). The repos share no code — deliberately, they are independent — but changing the workflow in one is a reason to look at the other. -## Why this track exists +## Maintaining this route -A green `make test` does not mean the feature works. Unit tests check each piece in -isolation, so the failures they structurally cannot see are the ones where every piece is -correct and the *wiring* is not: a click handler that fires but updates nothing visible, a -write that returns success and lands in the wrong column, a redirect that resolves to the -wrong page. Those surface only when you drive the real thing end to end. +Harness facts are owned by [`../e2e/README.md`](../e2e/README.md). Follow [documentation.md](documentation.md) after path or harness changes. What this route still owns: -That is the question this track answers, and it is why "the tests pass" is not a -verification report. +```bash +grep -n 'e2e/scratch' .gitignore # evidence stays local +grep -n 'testDir' e2e/playwright.scratch.config.ts # the scratch config still targets it +grep -n 'scratch\|serve\|drive' e2e/package.json # the run commands still exist +git ls-files --error-unmatch e2e/drive.mjs e2e/lib/drive-target.mjs # the driver is committed +``` diff --git a/e2e/.prettierignore b/e2e/.prettierignore index d452556b..96b84c1a 100644 --- a/e2e/.prettierignore +++ b/e2e/.prettierignore @@ -1,4 +1,7 @@ node_modules +# drive 的浏览器 profile 与 serve 交接件：本机运行时产物，跑过一次就有上百个 +# Chrome 自带的扩展文件，不进格式检查（否则 `make lint` 会因为你验过一次而变红） +.drive test-results playwright-report scratch diff --git a/e2e/README.md b/e2e/README.md index c3a8d477..828b0374 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,16 +1,20 @@ -# e2e — twin tracks +# e2e — four entry points, one rule -Two tracks. Different purposes, different destinations, **never mixed**. +Everything here drives the real app in a real browser. What differs is **who +decides the steps** — a committed spec, or you, right now. -| | Smoke (`e2e/*.spec.ts`) | Local verification (`e2e/scratch/`) | -| ------------- | ----------------------------------- | -------------------------------------------- | -| Committed | Yes | **No** — the whole directory is gitignored | -| Purpose | Stop basic functionality regressing | "I just finished X — does it actually work?" | -| Lifetime | Permanent | Disposable | -| External deps | All mocked | May hit a real environment | -| Bar for entry | **Very high** — core flows only | Low — write one whenever you want to check | -| Output | A green light in CI | `e2e/scratch/<task-name>/report.md` | -| Config | `playwright.config.ts` | `playwright.scratch.config.ts` | +| | `pnpm smoke` | `pnpm serve` + `pnpm drive` | `pnpm scratch` | `pnpm web` / `pnpm dual` | +| ---------------- | ----------------------------------- | ------------------------------------------ | ---------------------------------------- | --------------------------------------- | +| Committed | Yes, runs in CI | The harness is; the evidence is not | **No** — `scratch/` is gitignored | Yes, on demand — never in CI | +| Steps decided by | the spec | **you, one command at a time** | the spec you just wrote | the spec | +| Purpose | stop basic functionality regressing | "does this actually work / look right?" | replay a sequence, or timing/concurrency | the full chain, incl. a real `agentred` | +| External deps | all mocked | real server + real MySQL/Redis | may hit a real environment | real everything | +| Output | a green light in CI | `scratch/<scenario>/` + what you read back | `scratch/<scenario>/report.md` | a green light, on demand | +| Config | `playwright.config.ts` | none — `drive.mjs` attaches over CDP | `playwright.scratch.config.ts` | `playwright.web.config.ts` / `.dual.` | + +**The rule: drive it before you write a spec.** A spec asserts only what you +thought of in advance, and a one-line change costs a whole cold run. Write one +when the sequence must be **replayed** — not to look at something once. ## Running @@ -19,17 +23,21 @@ cd e2e pnpm install pnpm smoke # committed smoke suite (desktop + mobile) -pnpm scratch # your throwaway checks under scratch/ +pnpm serve # a seeded real environment, held open until Ctrl-C +pnpm drive up # a browser that outlives each command +pnpm scratch # throwaway specs under scratch/ +pnpm web # full chain: real server + real agentred ``` -Both scripts run `playwright install chromium` first. That costs ~2s once the -browser is there, and it is what stops the failure mode where bumping -`@playwright/test` leaves a stale build on disk and **every** spec goes red with -`Executable doesn't exist` — which reads like the app broke, not the browser. +`smoke` and `scratch` run `playwright install chromium` first, and `drive` +installs it on first `open`. That costs ~2s once the browser is there, and it is +what stops the failure mode where bumping `@playwright/test` leaves a stale build +on disk and **every** spec goes red with `Executable doesn't exist` — which reads +like the app broke, not the browser. From the repo root, `make test-e2e` runs the smoke track. -Both tracks run every spec against **two projects — `desktop-chromium` and +Both spec tracks run every spec against **two projects — `desktop-chromium` and `mobile-chromium`**. Both are supported form factors, so a desktop-only pass gives no signal about mobile layout. Narrow to one while iterating: @@ -38,6 +46,101 @@ pnpm smoke --project=desktop-chromium pnpm smoke -g "theme" ``` +## Driving by hand (`pnpm serve` + `pnpm drive`) + +`pnpm drive up` leaves a browser running; every later call performs **one action** +against it and records itself. This is the same driver the desktop repo uses +(`agentre/e2e/drive.mjs`) — same commands, same selector DSL, same `drive.log`. +The repos share no code, only the workflow. + +```bash +export AGENTRE_VERIFY_SCENARIO=2026-08-13-console # every call records into this scenario + +pnpm drive up # target = pnpm serve's URL, signed in +pnpm drive up --base http://127.0.0.1:5174 # or a logged-out target you started yourself +pnpm drive snapshot # what is on screen, and how to address it +pnpm drive click "testid=nav-devices" +pnpm drive fill "label=User code" A4F-7Q2 +pnpm drive text "main" # read the page back +pnpm drive sql "select status from device_flow_codes where user_code = 'A4F-7Q2'" +pnpm drive shot 01-devices +pnpm drive viewport 390x844 # the other form factor +pnpm drive logs 40 # server + agentred logs +pnpm drive down +``` + +**Start with `snapshot`.** It lists every visible interactive element with the +address to reach it (`testid=…`, `role=…`), sorted top-to-bottom — so you write +selectors from what is on screen instead of guessing from source. Prefer +`testid=`: visible text is i18n'd here and moves. + +Each call connects over CDP, acts, and disconnects; the browser and its cookies, +`localStorage` and current page stay put. `up` seeds the session cookie from +`pnpm serve`'s handoff, so you start signed in rather than at GitHub OAuth. + +Four guards are mechanical, not conventions to remember: + +- **Only this run's own origin is ever driven.** Any other URL is refused — + including another port on your own machine, which is likely the `make dev` you + are actually working in. +- **The oracle is read-only.** `drive sql` accepts only `SELECT` / `WITH` / + `EXPLAIN` / `SHOW`: a verification observes state, it does not manufacture it. + `--db agentred` reads that run's `agentred.db` instead of MySQL. +- **Evidence stays in its scenario.** `shot` writes under + `scratch/<scenario>/screenshots/`; a name containing `..` is refused. +- **Nothing takes over your screen.** The browser is headless; `--headed` when + you want to watch. + +Every call — including the failures — is appended to +`scratch/<scenario>/logs/drive.log` as it happens, so the report cites a record +written during the run rather than one reconstructed after it. + +`sql` and `logs` need no browser, and keep working after `down` — which is when +you are writing the report up. + +## What `pnpm serve` gives you + +`pnpm serve` is `run-e2e-web.mjs --serve`: everything `pnpm web` builds and +seeds, and then **nothing** — no spec runs, the environment simply stays up: + +``` +pnpm serve + ├─ builds frontend + server + webe2e + agentred, starts the server on a free port + ├─ seeds ONE throwaway account + a browser session + an online agentred + ├─ writes .drive/serve-env.json ← `pnpm drive up` reads it and lands signed in + └─ waits. Ctrl-C deletes exactly the rows it seeded, then stops everything. +``` + +Without it, that environment existed only for the lifetime of a spec run, so +"look at this page once" cost a full cold start and then vanished. + +The handoff file carries the session cookie **and the database DSN** (that is what +`drive sql` queries), so treat it as a credential; it is gitignored and removed on +Ctrl-C. If a stale one survives a crash, `pnpm drive up` will land on a dead +address — delete `.drive/` and re-run `pnpm serve`. + +### When your `configs/config.yaml` is `source: etcd` + +It will not work, and not only because the runner reads `db.dsn` out of the file. +With a non-`file` source, cago **replaces the whole config source** — the local +file is used only to find etcd (`configs/config.go`, `init`). So everything comes +from etcd, including `http.address`, and the server this runner starts would bind +whatever that says (`0.0.0.0:8443`) instead of the free port the run picked. The +isolation the harness is built on is gone, and it collides with your own +`make dev`. The etcd copy also tends to describe a _deployed_ box — ours carries +`/root/...` JWT key paths that do not exist on a laptop. + +Point the run at a file-based config instead: + +```bash +WEBE2E_CONFIG_DIR=/path/to/dir pnpm serve # dir/configs/config.yaml, source: file +``` + +Compose that file from the etcd values (`db`, `redis`, `server`), set +`source: file`, and point the JWT paths at this checkout's `runtime/keys/`. The +runner assigns the port itself. Everything else — seeding, the online `agentred`, +cleanup — then works unchanged. + ## Why two config files and not just .gitignore `.gitignore` stops scratch being **committed**. It does nothing about scratch @@ -55,6 +158,16 @@ dies at module load on `WEBE2E_SERVER_URL` before a single assertion runs — that is exactly how CI bit once (web/ added without the smoke config learning about it). +Three specs under `web/` are the exception: `runner-config.spec.ts`, +`runner-serve.spec.ts` and `drive-cli.spec.ts` drive pure functions out of +`run-e2e-web.mjs` and `lib/drive-target.mjs`, so they need no environment at +all. They live there because what they guard is this harness itself. Run them +alone: + +```bash +pnpm exec playwright test --config playwright.web.config.ts web/drive-cli.spec.ts +``` + ## Hermetic guarantees - **Dedicated port** (`5199`, see `fixtures/ports.ts`), deliberately not the dev @@ -74,6 +187,11 @@ about it). _every_ navigation, so `page.reload()` wipes what the test just set — turning working persistence into a red test. +`drive` shares none of this on purpose: it is pointed at whatever you tell it to +drive, and it keeps one browser across commands precisely so state accumulates. +That is the opposite of what a suite needs and exactly what looking at something +needs. + ## What earns a place in the smoke track Only: the app boots, routing and auth redirects, core device-flow UI, theme and @@ -87,7 +205,8 @@ separate, deliberate decision, not something that happens because it was handy. The smoke track runs against the vite dev server with the API mocked, so it needs no MySQL or Redis. Full-stack flows (real device flow, migrations, session -cookies) need both, plus the Go server: +cookies) need both, plus the Go server. `pnpm serve` arranges all of it; to drive +it with a spec instead: ```bash go run ./cmd/server # real backend on :8443 @@ -166,9 +285,9 @@ evidence inline, the honesty clause — live in is [`docs/references/verification-report-template.md`](../docs/references/verification-report-template.md). ``` -e2e/scratch/<task-name>/ +e2e/scratch/<scenario>/ ├── report.md -├── screenshots/ +├── screenshots/ ← where `pnpm drive shot` writes ├── videos/ └── resources/ ``` diff --git a/e2e/drive.mjs b/e2e/drive.mjs new file mode 100644 index 00000000..d0af1a55 --- /dev/null +++ b/e2e/drive.mjs @@ -0,0 +1,603 @@ +// 验证驱动器:一次调用一个动作,打在 `pnpm drive up` 留在那儿的浏览器上。 +// 它替掉的是「为看一眼而写一条一次性 spec」—— 你看、你动、再看,每一次调用都当场 +// 追加到本场景的 logs/drive.log 里。 +// +// node drive.mjs snapshot # 屏幕上有什么,以及怎么定位它 +// node drive.mjs click "testid=nav-devices" +// node drive.mjs shot 01-devices +// node drive.mjs sql "select status, count(*) from device_flow_codes group by status" +// node drive.mjs logs 40 +// +// 隔离不是这个文件的判断:它连的是 `pnpm serve`(或 --base)定下的那个 target, +// 每个碰到的 URL 都过 assertSanctionedURL(lib/drive-target.mjs)。 +// +// 与桌面端 `agentre/e2e/drive.mjs` 同形 —— 两个仓独立,不共享代码,但同一套命令、 +// 同一套选择器 DSL、同一份 drive.log。区别只在这边的 target 是 server + 浏览器, +// oracle 是 MySQL/agentred.db 而不是桌面端那个 SQLite。 +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + IsolationError, + assertReadOnlySQL, + assertSanctionedURL, + mysqlInvocation, + parseArgs, + parseRoleSpec, + resolveTargetURL, + scenarioPaths, + sessionCookie, + shotPath, + splitLocator, +} from "./lib/drive-target.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); // e2e/ +const stateDir = join(here, ".drive"); +const sessionPath = join(stateDir, "session.json"); +const serveEnvPath = join(stateDir, "serve-env.json"); +const scratchRoot = join(here, "scratch"); +const DEFAULT_TIMEOUT = 15_000; +const DEFAULT_CDP_PORT = 9333; + +function readJSON(path) { + return existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : null; +} + +/** 本次运行的证据落点。一个场景一个目录 —— docs/verification.md。 */ +function scenarioDirs(flags) { + const slug = + flags.scenario || process.env.AGENTRE_VERIFY_SCENARIO || "_unscoped"; + const dirs = scenarioPaths(scratchRoot, slug); + mkdirSync(dirs.logs, { recursive: true }); + mkdirSync(dirs.screenshots, { recursive: true }); + return dirs; +} + +function record(dirs, line) { + appendFileSync(join(dirs.logs, "drive.log"), `${line}\n`); +} + +function requireLiveSession() { + const session = readJSON(sessionPath); + if (!session) { + throw new IsolationError( + "no verification browser is up. Start one: `pnpm drive up` " + + "(after `pnpm serve`, or with --base for a logged-out target)", + ); + } + return session; +} + +function locate(page, spec, flags) { + const { kind, value } = splitLocator(spec); + let locator; + switch (kind) { + case "testid": + locator = page.getByTestId(value); + break; + case "role": { + const { role, name } = parseRoleSpec(value); + locator = page.getByRole(role, name === undefined ? {} : { name }); + break; + } + case "text": + locator = page.getByText(value); + break; + case "label": + locator = page.getByLabel(value); + break; + case "placeholder": + locator = page.getByPlaceholder(value); + break; + default: + locator = page.locator(spec); + } + return Number.isInteger(Number(flags.nth)) + ? locator.nth(Number(flags.nth)) + : locator.first(); +} + +async function attach(session) { + // 应用死掉和空白页透过 CDP 长得一模一样:浏览器答应、页面是空的,snapshot 报 + // 「0 elements」像是界面本来就空。先问 target 自己,让失败自报家门。 + try { + await fetch(session.baseURL, { signal: AbortSignal.timeout(3000) }); + } catch { + throw new Error( + `nothing is serving ${session.baseURL} any more — the server or vite behind ` + + "it stopped. `pnpm drive logs` for why, then bring it back up.", + ); + } + const { chromium } = await import("@playwright/test"); + let browser; + try { + browser = await chromium.connectOverCDP(session.cdpURL, { + timeout: 15_000, + }); + } catch (err) { + throw new Error( + `cannot reach the verification browser at ${session.cdpURL} — still up? ` + + `(\`pnpm drive status\`)\n${err.message}`, + ); + } + const context = browser.contexts()[0]; + if (!context) throw new Error("the verification browser has no context"); + const pages = context.pages(); + const page = + pages.find((p) => p.url().startsWith(session.baseURL)) || + pages[0] || + (await context.newPage()); + return { browser, page }; +} + +// ── 生命周期 ──────────────────────────────────────────────────────────────── + +async function urlReady(url, timeoutMs) { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const res = await fetch(url, { signal: AbortSignal.timeout(2000) }); + if (res.ok) return await res.json(); + } catch { + /* 还没起来 */ + } + if (Date.now() > deadline) return null; + await new Promise((r) => setTimeout(r, 150)); + } +} + +/** + * 起一个**活过单次调用**的浏览器,并把这次的 target 定下来。 + * 无头是默认:一次验证不该抢走屏幕和焦点。--headed 是给你想看着它走的时候用的。 + */ +async function up(flags) { + const serveEnv = readJSON(serveEnvPath); + const baseURL = flags.base ?? serveEnv?.serverURL; + if (!baseURL) { + throw new IsolationError( + "no target: run `pnpm serve` first (seeded, signed in), or name one with " + + "--base http://127.0.0.1:5174 for a logged-out UI check", + ); + } + const cdpPort = Number(flags.port ?? DEFAULT_CDP_PORT); + const cdpURL = `http://127.0.0.1:${cdpPort}`; + const existing = readJSON(sessionPath); + if ( + existing && + !flags.fresh && + (await urlReady(`${cdpURL}/json/version`, 300)) + ) { + console.log(`already up: ${existing.baseURL} (cdp ${existing.cdpURL})`); + return 0; + } + if (existing) await down({}); + + const { chromium } = await import("@playwright/test"); + const executable = chromium.executablePath(); + if (!existsSync(executable)) { + console.log("[drive] installing chromium …"); + spawnSync("pnpm", ["exec", "playwright", "install", "chromium"], { + cwd: here, + stdio: "inherit", + }); + } + mkdirSync(stateDir, { recursive: true }); + const browserDir = join(stateDir, "browser"); + rmSync(browserDir, { recursive: true, force: true }); + const [w, h] = String(flags.viewport ?? "1440x900").split("x"); + const args = [ + `--remote-debugging-port=${cdpPort}`, + `--user-data-dir=${browserDir}`, + `--window-size=${w},${h}`, + "--no-first-run", + "--no-default-browser-check", + "--no-service-autorun", + "--password-store=basic", + "--use-mock-keychain", + ]; + if (!flags.headed) args.push("--headless=new"); + args.push(baseURL); + + const proc = spawn(executable, args, { detached: true, stdio: "ignore" }); + proc.unref(); + const version = await urlReady(`${cdpURL}/json/version`, 30_000); + if (!version) throw new Error(`chromium did not expose CDP on :${cdpPort}`); + + const session = { + baseURL, + cdpPort, + cdpURL, + browserPid: proc.pid ?? null, + headless: !flags.headed, + seeded: !!serveEnv, + serverLog: serveEnv?.serverLog ?? null, + agentredLog: serveEnv?.agentredLog ?? null, + agentredDB: serveEnv?.agentredDB ?? null, + dsn: serveEnv?.dsn ?? null, + startedAt: new Date().toISOString(), + }; + writeFileSync(sessionPath, `${JSON.stringify(session, null, 2)}\n`); + + // 播种的会话 Cookie 要落在浏览器自己的 context 上,后面每次 attach 都带着它。 + if (serveEnv) { + const { browser, page } = await attach(session); + try { + await browser.contexts()[0].addCookies([sessionCookie(serveEnv)]); + await page.goto(baseURL, { waitUntil: "domcontentloaded" }); + } finally { + await browser.close().catch(() => {}); + } + } + console.log( + [ + `${session.headless ? "headless" : "headed"} ${version.Browser} on ${cdpURL}`, + `target ${baseURL}${session.seeded ? " (signed in as the seeded account)" : ""}`, + "", + "drive it (one action per call, every call recorded):", + " pnpm drive snapshot --scenario <slug>", + ' pnpm drive click "testid=…" pnpm drive shot 01-name', + " pnpm drive down", + ].join("\n"), + ); + return 0; +} + +async function down() { + const session = readJSON(sessionPath); + if (!session) { + console.log("nothing is up"); + return 0; + } + // detached 起的 chromium 是**进程组组长**,先打整组;组不在了再打单个 pid。 + // 只打 pid 的话主进程会活下来占着 CDP 端口,下一次 up 就「已经有人在了」。 + const signal = (sig) => { + for (const target of [-session.browserPid, session.browserPid]) { + try { + process.kill(target, sig); + return true; + } catch { + /* 不是组长 / 已经没了 */ + } + } + return false; + }; + if (session.browserPid) { + signal("SIGTERM"); + // 没走干净就升级:说「停了」而它还在,下一次 up 会连上一个上轮的浏览器。 + if (await urlReady(`${session.cdpURL}/json/version`, 3000)) + signal("SIGKILL"); + } + rmSync(sessionPath, { force: true }); + const stillUp = await urlReady(`${session.cdpURL}/json/version`, 1000); + console.log( + stillUp + ? `WARNING: something still answers on ${session.cdpURL} — kill it by hand` + : `stopped the verification browser (${session.baseURL})`, + ); + return stillUp ? 1 : 0; +} + +async function status() { + const session = readJSON(sessionPath); + if (!session) { + console.log("down"); + return 1; + } + const version = await urlReady(`${session.cdpURL}/json/version`, 500); + const serving = await fetch(session.baseURL, { + signal: AbortSignal.timeout(2000), + }) + .then((r) => `${r.status}`) + .catch(() => "unreachable"); + console.log( + [ + `target ${session.baseURL} (${serving})`, + `browser ${version ? version.Browser : "NOT RESPONDING"} on ${session.cdpURL}`, + `seeded ${session.seeded ? "yes — signed in" : "no — logged-out target"}`, + `since ${session.startedAt}`, + ].join("\n"), + ); + return version ? 0 : 1; +} + +// ── 动作 ──────────────────────────────────────────────────────────────────── + +const COMMANDS = { + async goto({ page, session, rest }) { + const url = assertSanctionedURL( + session, + resolveTargetURL(rest[0] ?? "/", session.baseURL), + ); + await page.goto(url, { waitUntil: "domcontentloaded" }); + return `at ${page.url()}`; + }, + + async click({ page, rest, flags }) { + await locate(page, rest[0], flags).click({ + timeout: Number(flags.timeout ?? DEFAULT_TIMEOUT), + }); + return `clicked ${rest[0]} (now at ${page.url()})`; + }, + + async fill({ page, rest, flags }) { + await locate(page, rest[0], flags).fill(rest.slice(1).join(" "), { + timeout: Number(flags.timeout ?? DEFAULT_TIMEOUT), + }); + return `filled ${rest[0]}`; + }, + + async press({ page, rest, flags }) { + // `press <key>` 打给当前焦点;`press <locator> <key>` 打给某个元素。 + if (rest.length === 1) { + await page.keyboard.press(rest[0]); + return `pressed ${rest[0]}`; + } + await locate(page, rest[0], flags).press(rest[1], { + timeout: Number(flags.timeout ?? DEFAULT_TIMEOUT), + }); + return `pressed ${rest[1]} on ${rest[0]}`; + }, + + async wait({ page, rest, flags }) { + await locate(page, rest[0], flags).waitFor({ + state: flags.state ?? "visible", + timeout: Number(flags.timeout ?? DEFAULT_TIMEOUT), + }); + return `${rest[0]} is ${flags.state ?? "visible"}`; + }, + + /** 屏幕上有什么、怎么定位它。看不懂发生了什么时第一个跑的就是它。 */ + async snapshot({ page, rest, flags }) { + const items = await page.evaluate( + ({ rootSel, limit }) => { + const root = rootSel ? document.querySelector(rootSel) : document.body; + if (!root) return null; + const SELECTOR = + 'button, a[href], input, select, textarea, [role], [data-testid], [contenteditable="true"], h1, h2, h3, [aria-label]'; + const out = []; + const seen = new Set(); + for (const el of root.querySelectorAll(SELECTOR)) { + if (out.length >= limit) break; + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + if (rect.width === 0 || rect.height === 0) continue; + if ( + style.visibility === "hidden" || + style.display === "none" || + style.opacity === "0" + ) + continue; + const testid = el.getAttribute("data-testid") || ""; + const role = el.getAttribute("role") || el.tagName.toLowerCase(); + const name = ( + el.getAttribute("aria-label") || + el.getAttribute("placeholder") || + el.innerText || + el.value || + "" + ) + .trim() + .replace(/\s+/g, " ") + .slice(0, 70); + const key = `${role}|${testid}|${name}`; + if (seen.has(key)) continue; + seen.add(key); + const state = []; + if (el.disabled) state.push("disabled"); + if (el.getAttribute("aria-selected") === "true") + state.push("selected"); + const expanded = el.getAttribute("aria-expanded"); + if (expanded !== null) state.push(`expanded=${expanded}`); + out.push({ + role, + testid, + name, + state: state.join(","), + y: Math.round(rect.top), + x: Math.round(rect.left), + }); + } + return out; + }, + { rootSel: rest[0] ?? null, limit: Number(flags.limit ?? 120) }, + ); + if (items === null) throw new Error(`no element matches ${rest[0]}`); + items.sort((a, b) => a.y - b.y || a.x - b.x); + const lines = items.map((it) => { + const addr = it.testid ? `testid=${it.testid}` : `role=${it.role}`; + return ` ${addr.padEnd(42)} [${it.role}] ${it.name}${it.state ? ` (${it.state})` : ""}`; + }); + console.log(`${page.url()}\n${lines.join("\n")}`); + return `${items.length} elements`; + }, + + async text({ page, rest, flags }) { + const locator = rest[0] + ? locate(page, rest[0], flags) + : page.locator("body"); + const value = ( + await locator.innerText({ + timeout: Number(flags.timeout ?? DEFAULT_TIMEOUT), + }) + ).trim(); + console.log(value.slice(0, 4000)); + return `${value.length} chars`; + }, + + async shot({ page, rest, flags, dirs }) { + const name = (rest[0] || "shot").replace(/[^\w.-]/g, "-"); + const path = shotPath(dirs.root, name); + const shooter = rest[1] ? locate(page, rest[1], flags) : page; + await shooter.screenshot({ + path, + fullPage: rest[1] ? undefined : Boolean(flags.full), + }); + console.log(path); + return `saved ${path}`; + }, + + async viewport({ page, rest }) { + const [w, h] = String(rest[0] ?? "1440x900") + .split("x") + .map(Number); + await page.setViewportSize({ width: w, height: h }); + return `viewport ${w}x${h}`; + }, + + async eval({ page, rest }) { + const result = await page.evaluate(rest.join(" ")); + console.log(JSON.stringify(result, null, 2)); + return "evaluated"; + }, + + /** + * 独立 oracle:读应用写进去的数据,而不是它渲染出来的界面。只读 —— + * 一次验证观察状态,不制造状态。 + * + * --db mysql(默认,serve 播种的那套库)/ --db agentred(那台 agentred 的 SQLite) + */ + async sql({ session, rest, flags }) { + const query = assertReadOnlySQL(rest.join(" ")); + if ((flags.db ?? "mysql") === "agentred") { + if (!session.agentredDB || !existsSync(session.agentredDB)) { + throw new Error( + `no agentred database at ${session.agentredDB ?? "(unset)"}`, + ); + } + const { DatabaseSync } = await import("node:sqlite"); + const db = new DatabaseSync(session.agentredDB, { readOnly: true }); + try { + const rows = db.prepare(query).all(); + console.log(JSON.stringify(rows, null, 2)); + return `${rows.length} rows`; + } finally { + db.close(); + } + } + if (!session.dsn) { + throw new Error( + "this session has no database DSN — it was started with --base, not by " + + "`pnpm serve`. Query your own database directly, or re-run through serve.", + ); + } + const { args, env } = mysqlInvocation(session.dsn, query); + try { + const out = execFileSync("mysql", args, { + encoding: "utf8", + // 口令只从环境进去,不进 argv —— 否则同机 `ps` 就能看到它。 + env: { ...process.env, ...env }, + }); + console.log(out.trim()); + return "queried"; + } catch (err) { + if (err.code === "ENOENT") { + throw new Error( + "the mysql client is not on PATH — install it, or query the DSN yourself", + ); + } + throw new Error( + (err.stderr || err.message).toString().trim().split("\n")[0], + ); + } + }, + + /** 服务器与 agentred 自己的日志。浏览器没了也照样能看 —— 写报告时要用。 */ + async logs({ session, rest, flags }) { + const lines = Number(rest[0]) || Number(flags.limit) || 40; + const files = [session.serverLog, session.agentredLog].filter(Boolean); + if (!files.length) { + throw new Error( + "this session has no logs — it was started with --base, not by `pnpm serve`", + ); + } + for (const file of files) { + if (!existsSync(file) || !statSync(file).isFile()) continue; + const tail = readFileSync(file, "utf8") + .split("\n") + .slice(-lines) + .join("\n"); + console.log(`--- ${file}\n${tail}`); + } + return `tailed ${files.length} files`; + }, +}; + +const LIFECYCLE = { up, down, status }; + +async function main() { + let parsed; + try { + parsed = parseArgs(process.argv.slice(2)); + } catch (err) { + console.error(err.message); + return 2; + } + const { command, rest, flags } = parsed; + + if (LIFECYCLE[command]) return LIFECYCLE[command](flags); + if (!command || !COMMANDS[command]) { + console.error( + `usage: pnpm drive <up|down|status|${Object.keys(COMMANDS).join("|")}> [args] ` + + "[--scenario <slug>] [--base URL] [--headed] [--nth N] [--state visible|hidden] " + + "[--timeout ms] [--limit N] [--db pg|agentred] [--full]", + ); + return 2; + } + + const session = requireLiveSession(); + const dirs = scenarioDirs(flags); + const started = new Date(); + const printable = [command, ...rest].join(" "); + + let ctx = null; + try { + // sql 与 logs 读的是应用自己写下的文件,不需要浏览器,而且浏览器关掉之后 + // 还得能用 —— 写报告的时候正是这个时候。 + if (command !== "sql" && command !== "logs") ctx = await attach(session); + const summary = await COMMANDS[command]({ + ...ctx, + session, + rest, + flags, + dirs, + }); + record( + dirs, + `${started.toISOString()} drive ${printable}\n ok: ${summary}`, + ); + // 回执也打到 stdout:否则 click / fill / wait 成功时**一声不响**,分不清是做了 + // 还是没做。命令自己的产出(snapshot / text / sql)已经先打过了。 + console.log(`ok: ${summary}`); + return 0; + } catch (err) { + record( + dirs, + `${started.toISOString()} drive ${printable}\n FAILED: ${err.message.split("\n")[0]}`, + ); + console.error( + err instanceof IsolationError + ? `${err.name}: ${err.message}` + : err.message, + ); + if (ctx) + console.error( + "hint: `pnpm drive snapshot` shows what is actually on screen", + ); + return 1; + } finally { + // 只断开这一个客户端;浏览器和被驱动的应用继续活着,等下一次调用。 + if (ctx) await ctx.browser.close().catch(() => {}); + } +} + +main().then((code) => process.exit(code)); diff --git a/e2e/lib/drive-target.mjs b/e2e/lib/drive-target.mjs new file mode 100644 index 00000000..1cc8bcc8 --- /dev/null +++ b/e2e/lib/drive-target.mjs @@ -0,0 +1,201 @@ +// 手工驱动这条轨道的契约:命令行怎么解析、什么地址可以驱动、证据落在哪、 +// oracle 能做什么。drive.mjs 与 web/drive-cli.spec.ts 都从这里取。 +// +// 形状与桌面端 `agentre/e2e/lib/target.mjs` 对齐 —— 两个仓各自独立、不共享代码, +// 但同一套流程、同一套词汇和同样几条机械护栏。这里的规则是**机械的**,不是靠记的: +// +// 1. **只驱动本次 target 自己的 origin。** 线上地址、同机上别的端口(你正在手调的 +// 那套 `make dev`)一律拒绝 —— 驱动错了对象,报告里是看不出来的。 +// 2. **oracle 只读。** 一次验证观察状态,不制造状态。 +// 3. **证据只落在本场景目录里。** 报告里的相对链接靠的就是这一点。 +import { join, resolve, sep } from "node:path"; + +export class IsolationError extends Error { + constructor(message) { + super(message); + this.name = "IsolationError"; + } +} + +/** 带值的开关。其余以 -- 开头的都是布尔;两张表都没有的当场失败。 */ +const VALUE_FLAGS = new Set([ + "base", + "db", + "limit", + "nth", + "port", + "scenario", + "state", + "timeout", + "viewport", +]); +const BOOL_FLAGS = new Set(["headed", "full", "fresh", "json"]); + +/** + * 拼错的开关**当场失败**,不被当成布尔悄悄吞掉 —— `--headles` 什么都不做而命令 + * 仍然返回成功,是这一条要挡的。 + */ +export function parseArgs(argv) { + const positional = []; + const flags = {}; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (!arg.startsWith("--")) { + positional.push(arg); + continue; + } + const name = arg.slice(2); + if (VALUE_FLAGS.has(name)) flags[name] = argv[++i]; + else if (BOOL_FLAGS.has(name)) flags[name] = true; + else throw new Error(`unknown flag: ${arg}`); + } + return { command: positional[0], rest: positional.slice(1), flags }; +} + +const LOCATOR_KINDS = ["testid", "role", "text", "label", "placeholder"]; + +/** + * `testid=x` / `role=…` / `text=…` / `label=…` / `placeholder=…`,其余原样当 + * Playwright 选择器。只认这几个前缀:按 indexOf("=") 切的话, + * `button[aria-label^='Theme']` 会被切成 kind="button[aria-label",整条就废了。 + */ +export function splitLocator(spec) { + for (const kind of LOCATOR_KINDS) { + if (spec.startsWith(`${kind}=`)) { + return { kind, value: spec.slice(kind.length + 1) }; + } + } + return { kind: "", value: spec }; +} + +/** `button[name="Save"]` → 角色 + 可选的无障碍名。 */ +export function parseRoleSpec(value) { + const m = /^([a-z]+)(?:\[name=(?:"([^"]*)"|'([^']*)')\])?$/.exec(value); + if (!m) { + throw new Error( + `bad role locator: ${value} (expected role=button[name="Save"])`, + ); + } + return { role: m[1], name: m[2] ?? m[3] }; +} + +/** 命令行上的目标变成绝对地址。裸词当路径,不当协议。 */ +export function resolveTargetURL(target, baseURL) { + if (/^https?:\/\//i.test(target)) return target; + if (!baseURL) { + throw new Error( + `no baseURL to resolve "${target}" against — pass --base, or start the ` + + "seeded environment with `pnpm serve`", + ); + } + return new URL(target.startsWith("/") ? target : `/${target}`, baseURL).href; +} + +/** 除了本次 target 自己的 origin,哪儿都不去。 */ +export function assertSanctionedURL(session, url) { + let parsed; + try { + parsed = new URL(url); + } catch { + throw new IsolationError(`not a URL: ${url}`); + } + const base = new URL(session.baseURL); + const loopback = + parsed.hostname === "localhost" || parsed.hostname === "127.0.0.1"; + if (parsed.protocol !== "http:" || !loopback || parsed.port !== base.port) { + throw new IsolationError( + `refusing to drive ${url}: this run's target is only ${session.baseURL}`, + ); + } + return parsed.toString(); +} + +/** 一个场景一个目录。报告、日志、截图、资源各归各位。 */ +export function scenarioPaths(scratchRoot, slug) { + const root = join(scratchRoot, slug); + return { + slug, + root, + logs: join(root, "logs"), + screenshots: join(root, "screenshots"), + resources: join(root, "resources"), + }; +} + +/** 截图的落点。只能落在本场景目录里。 */ +export function shotPath(scenarioDir, name) { + const shots = resolve(scenarioDir, "screenshots"); + const file = resolve(shots, name.endsWith(".png") ? name : `${name}.png`); + if (file !== shots && !file.startsWith(shots + sep)) { + throw new Error(`"${name}" escapes the scenario directory ${shots}`); + } + return file; +} + +/** + * Go 的 MySQL DSN(`user:pass@tcp(host:port)/db?params`)→ mysql 客户端怎么调。 + * + * 两处非要机械化不可: + * - **口令走 MYSQL_PWD,不进 argv。** 放命令行的话,同机上任何人 `ps` 都能看到 + * 整条库口令,而这一路上没有任何东西会报错。 + * - **按最后一个 `@tcp(` 切,不是第一个 `@`。** 口令里带 @ 或 : 很常见,按第一个 + * 切会把口令截断 —— 连上去的是另一套凭据,或者干脆连不上。 + */ +export function mysqlInvocation(dsn, query) { + const m = /^([^:@]+)(?::(.*))?@tcp\(([^:)]+)(?::(\d+))?\)\/([^?]+)/.exec(dsn); + if (!m) { + throw new Error( + `not a MySQL DSN: ${dsn.replace(/:[^:@]*@/, ":<redacted>@")}`, + ); + } + const [, user, password = "", host, port = "3306", db] = m; + return { + args: [ + "-h", + host, + "-P", + port, + "-u", + user, + "--batch", + "--table", + "-e", + query, + db, + ], + env: { MYSQL_PWD: password }, + }; +} + +const READ_ONLY_SQL = /^\s*(select|with|explain|show|describe|desc)\b/i; + +/** 验证观察状态,不制造状态。 */ +export function assertReadOnlySQL(query) { + if (!READ_ONLY_SQL.test(query)) { + throw new Error( + "only SELECT / WITH / EXPLAIN / SHOW are allowed — the oracle is read-only", + ); + } + return query; +} + +/** + * serve 播种的那条浏览器会话 → 一个 Playwright Cookie。 + * domain 取 serve 出来的 host:写死 localhost 不会报错,只是这条 Cookie 不会被 + * 带上,表现成「明明播种了账号却一直跳登录页」。 + */ +export function sessionCookie(serveEnv) { + if (!serveEnv?.sid) { + throw new Error("serve environment has no sid — re-run `pnpm serve`"); + } + const url = new URL(serveEnv.serverURL); + return { + name: serveEnv.cookieName, + value: serveEnv.sid, + domain: url.hostname, + path: "/", + httpOnly: true, + secure: url.protocol === "https:", + sameSite: "Lax", + }; +} diff --git a/e2e/package.json b/e2e/package.json index ee7fc816..2683d531 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -4,12 +4,14 @@ "version": "0.0.0", "type": "module", "packageManager": "pnpm@10.33.0", - "description": "双轨 e2e：committed 冒烟 + gitignored scratch 本地验证。见 README.md", + "description": "e2e 与本地验证的入口：committed 冒烟 + 按需全链路 + 手工驱动（drive）。见 README.md", "scripts": { "smoke": "playwright install chromium && playwright test", "scratch": "playwright install chromium && playwright test --config playwright.scratch.config.ts", "web": "node run-e2e-web.mjs", "dual": "node run-e2e-web.mjs --dual", + "serve": "node run-e2e-web.mjs --serve", + "drive": "node drive.mjs", "runner-test": "playwright test --config playwright.runner.config.ts", "format": "prettier --write .", "lint": "prettier --check ." diff --git a/e2e/playwright.runner.config.ts b/e2e/playwright.runner.config.ts index 1421e46d..649c6384 100644 --- a/e2e/playwright.runner.config.ts +++ b/e2e/playwright.runner.config.ts @@ -1,7 +1,13 @@ +// Harness 自身的单测:不开浏览器、不碰服务器,只驱动 run-e2e-web.mjs 与 +// lib/drive-target.mjs 里那些纯函数。`make test-e2e` 会跑它,所以它是进 CI 的。 +// +// testMatch 要跟着新的 harness spec 一起长:漏掉一个,那个文件就成了「本地能跑、 +// CI 看不见」—— 和加了 build tag 是同一种假绿。 import { defineConfig } from "@playwright/test"; export default defineConfig({ testDir: "web", - testMatch: "runner-config.spec.ts", + testMatch: + /(runner-config|runner-config-dir|runner-serve|drive-cli)\.spec\.ts/, reporter: "list", }); diff --git a/e2e/run-e2e-web.mjs b/e2e/run-e2e-web.mjs index 738dc82c..c952e734 100644 --- a/e2e/run-e2e-web.mjs +++ b/e2e/run-e2e-web.mjs @@ -16,7 +16,9 @@ // agentred state.json, and run the real agentred daemon against the server // 6. wait until the agentred is online on the relay // 7. seed the agentred.db session + journal rows (node:sqlite oracle) -// 8. run playwright with the harness env exported +// 8. run playwright with the harness env exported — or, with `--serve`, run +// nothing and hold the whole environment open for `pnpm drive` to drive by +// hand (see serve()) // 9. ALWAYS delete exactly the rows this run seeded, then kill agentred + server // // With `--dual` it additionally seeds a kind=desktop device and hands the real @@ -59,6 +61,9 @@ const agentredDataDir = join(workDir, "ad"); const agentredDB = join(agentredDataDir, "agentred.db"); const projectDir = join(workDir, "project"); const webserverLog = join(workDir, "webserver.log"); +// serve 档交给 `pnpm drive` 的交接件。放在 e2e/.drive/ 下(gitignored), +// 和浏览器自己的状态挨着,收尾时一起删。 +const serveEnvPath = join(here, ".drive", "serve-env.json"); // 「桌面端 + 浏览器」那一档才用到的东西:真 Wails 应用的数据目录、钥匙串与它自己的 // SQLite(spec 的独立 oracle)。 const desktopDataDir = join(workDir, "desktop"); @@ -88,15 +93,85 @@ const SEEDED_SESSIONS = { agent_sync_id: "", }; -// --dual 才编排桌面端那一侧(e2e/dual/,playwright.dual.config.ts)。默认档保持 -// 今天的样子:只有浏览器,不需要 wails 工具链,也不多跑一个 Wails 进程。 -// 该标志由本 runner 自己消费,不能混进转发给 playwright 的参数里。 -const dualMode = process.argv.slice(2).includes("--dual"); -const playwrightArgs = process.argv.slice(2).filter((a) => a !== "--dual"); +const { + mode: runMode, + dual: dualMode, + playwrightArgs, +} = parseRunnerArgs(process.argv.slice(2)); // 桌面端 Wails IPC 桥的专用端口。**不是** 34216:那个属于 agentre 仓自己的 e2e, // 两边同时跑时才不会互相复用(或对着对方假绿)。 const DESKTOP_DEVSERVER = "localhost:34217"; +/** + * 本 runner 自己消费的开关,一个都不能混进转发给 playwright 的参数里 —— + * 漏过去的话 playwright 死在 "unknown option",看起来像 runner 坏了。 + * + * --dual 连桌面端那一侧一起编排(e2e/dual/,playwright.dual.config.ts); + * 默认档只有浏览器,不需要 wails 工具链。 + * --serve 起完环境就**撑住不退**,不跑任何 spec:留给 `pnpm drive` 手工驱动。 + */ +export function parseRunnerArgs(argv) { + const consumed = new Set(["--dual", "--serve"]); + return { + mode: argv.includes("--serve") ? "serve" : "spec", + dual: argv.includes("--dual"), + playwrightArgs: argv.filter((a) => !consumed.has(a)), + }; +} + +/** + * serve 档交给 drive 的那份交接件。少一样都开不出「已登录的浏览器对着这次播种的 + * 账号」,而缺失不会报错 —— 只会一路验的都是登录页,所以这里当场校验。 + */ +/** + * 这次 run 的 configs/config.yaml 从哪个检出取。 + * + * 默认:本检出有就用本检出的(worktree 的 configs/ 只有 example,真 DSN 在主检出)。 + * `WEBE2E_CONFIG_DIR` 显式指一份 —— 本仓的配置可以是 `source: etcd`,那时整份配置 + * 由 etcd 给:runner 读不到 db.dsn 去播种,它起的 server 也会照 etcd 绑 + * 0.0.0.0:8443,把「跑在空闲端口上」这条隔离作废。那种环境下开发者得指一份本机 + * 可用的文件配置,而不是让 runner 去猜、或者反过来改写共享的 etcd。 + */ +export function configCheckoutFor(serverDir, env = process.env) { + const override = env.WEBE2E_CONFIG_DIR; + if (override) { + const dir = resolve(override); + if (!existsSync(join(dir, "configs", "config.yaml"))) { + throw new Error( + `WEBE2E_CONFIG_DIR=${dir} has no configs/config.yaml — point it at a ` + + "directory shaped like a checkout (…/configs/config.yaml)", + ); + } + return dir; + } + return existsSync(join(serverDir, "configs", "config.yaml")) + ? serverDir + : mainCheckout(serverDir); +} + +export function serveEnvPayload(env, extra = {}) { + for (const key of [ + "WEBE2E_SERVER_URL", + "WEBE2E_COOKIE_NAME", + "WEBE2E_SESSION_SID", + ]) { + if (!env[key]) throw new Error(`serve handoff is missing ${key}`); + } + return { + serverURL: env.WEBE2E_SERVER_URL, + cookieName: env.WEBE2E_COOKIE_NAME, + sid: env.WEBE2E_SESSION_SID, + // `drive sql` 的 oracle 连的就是它。交接件因此带凭据 —— gitignored,收尾即删。 + dsn: extra.dsn ?? null, + runID: env.WEBE2E_RUN_ID, + serverLog: env.WEBE2E_SERVER_LOG, + agentredLog: env.WEBE2E_AGENTRED_LOG, + agentredDB: env.WEBE2E_AGENTRED_DB, + workspace: env.WEBE2E_WORKSPACE ? JSON.parse(env.WEBE2E_WORKSPACE) : null, + sessions: env.WEBE2E_SESSIONS ? JSON.parse(env.WEBE2E_SESSIONS) : null, + }; +} + let serverProc = null; let agentredProc = null; let toolBin = ""; @@ -125,11 +200,7 @@ async function main() { const serverDir = resolve(serverRoot); agentreDir = locateAgentreCheckout(serverDir); if (dualMode) requireWailsToolchain(agentreDir); - // The worktree's configs/ only has config.example.yaml (the real DSN is - // gitignored); the developer's real config.yaml lives in the main checkout. - const configCheckout = existsSync(join(serverDir, "configs", "config.yaml")) - ? serverDir - : mainCheckout(serverDir); + const configCheckout = configCheckoutFor(serverDir); dsn = readDSN(configCheckout); redis = readRedis(configCheckout); @@ -256,6 +327,7 @@ async function main() { ? { WEBE2E_SESSIONS: JSON.stringify(seededSessions) } : {}), }); + if (runMode === "serve") return serve(); if (dualMode) prepareDesktop(agentreDir, agentredFP); const require = createRequire(import.meta.url); @@ -274,6 +346,49 @@ async function main() { child.on("exit", (code) => void finish(code ?? 1)); } +/** + * serve 档:环境已经起好了,不跑 spec,把它**留在那里**给人手工驱动。 + * + * 为什么要有这一档:这套环境(真 server + 播种账号 + 真 agentred)原来只在 + * 一次 spec 运行的生命周期里存在,`finish()` 一到就连进程带 workDir 一起收掉。 + * 于是「看一眼这个页面现在什么样」的代价是整轮冷启 —— 前端构建 + 三个 go build + + * 播种 + agentred 上线,而且看完就没了。撑住不退之后,一次冷启可以驱动很多步。 + * + * 交接件写到 e2e/.drive/serve-env.json:`pnpm drive open` 读它,直接开出一个 + * 已登录这次播种账号的浏览器。收尾时删掉 —— 留着会让下一次 drive 拿着一个 + * 早就没人监听的地址和一条死会话,表现成「登录态莫名其妙失效」。 + */ +async function serve() { + const payload = serveEnvPayload(process.env, { dsn }); + mkdirSync(dirname(serveEnvPath), { recursive: true }); + writeFileSync(serveEnvPath, `${JSON.stringify(payload, null, 2)}\n`); + + const w = payload.workspace; + console.log( + [ + "", + "[web-e2e] serve — 环境已就绪,不会自己退出。Ctrl-C 收尾(会清账号)。", + ` server ${payload.serverURL}`, + ` account webe2e-${payload.runID}(会话 Cookie 已写进交接件)`, + ...(w + ? [ + ` agent ${w.agent_ready_name}(在线) / ${w.agent_blocked_name}(不可用)`, + ` project ${w.project_name} → ${w.project_path}`, + ] + : []), + ` server log ${payload.serverLog}`, + ` agentred log ${payload.agentredLog}`, + ` handoff ${serveEnvPath}`, + "", + " 下一步:cd e2e && pnpm drive up # 已登录,target 自动取上面这个地址", + " pnpm drive snapshot --scenario <slug>", + "", + ].join("\n"), + ); + // 不做任何事,只是不退出:进程活着,上面那些子进程就活着。 + await new Promise(() => {}); +} + // prepareDesktop 备好真 Wails 桌面端那一侧要用的一切,并把它交给 // playwright.dual.config.ts(webServer 由 Playwright 起停,和 e2e/sync 一样)。 // @@ -382,6 +497,10 @@ async function finish(code) { } } + // 交接件必须和环境同生共死:留着的话下一次 `pnpm drive open` 会拿着一个早就 + // 没人监听的地址和一条已被清掉的会话,表现成「登录态莫名其妙失效」。 + rmSync(serveEnvPath, { force: true }); + if (agentredProc && agentredProc.exitCode === null) agentredProc.kill("SIGTERM"); if (serverProc && serverProc.exitCode === null) serverProc.kill("SIGTERM"); @@ -400,9 +519,11 @@ async function finish(code) { // 只有编排进程才装信号处理:被 import 时装上去,Playwright 的 worker 会跟着在 // Ctrl-C 时跑 finish() —— 那里会 process.exit 并打印「kept <workDir>」,把真正的 // 收尾盖掉。import 时什么都不做,这条也算在内。 +// serve 档的 Ctrl-C 是**正常收尾**,不是中断一次运行:按 1 退会留下 workDir 并 +// 报一个假失败。 if (isEntrypoint) { for (const sig of ["SIGINT", "SIGTERM"]) - process.on(sig, () => void finish(1)); + process.on(sig, () => void finish(runMode === "serve" ? 0 : 1)); } // reapOrphanVite 收掉 `wails dev` 关停时遗留的 vite 子进程(它在另一个进程组里, diff --git a/e2e/web/drive-cli.spec.ts b/e2e/web/drive-cli.spec.ts new file mode 100644 index 00000000..f4c65479 --- /dev/null +++ b/e2e/web/drive-cli.spec.ts @@ -0,0 +1,248 @@ +/** + * drive 的纯函数与几条机械护栏(命令行解析、选择器 DSL、可驱动地址、只读 oracle、 + * 证据落盘路径、会话 Cookie)。不开浏览器:这里驱动的是「一条命令怎么变成一次动作」 + * 以及「什么动作会被当场拒绝」。 + * + * 形状与桌面端 `agentre/e2e/lib/target.mjs` + `drive.mjs` 对齐(两个仓独立,不共享代码, + * 但同一套流程和词汇)。守的是六处会静默走偏的地方: + * 1. 拼错的开关被当成布尔吞掉 —— `--headles` 什么都不做,而命令仍然「成功」; + * 2. CSS 属性选择器里的 `=` 被当成 DSL 前缀切开 → 选择器变成一段废话; + * 3. 驱动到不该驱动的地址上(线上、别人的端口),而这在报告里看不出来; + * 4. oracle 写数据 —— 验证只观察状态,不制造状态; + * 5. 截图落到场景目录之外 —— 报告里的相对链接全断; + * 6. 会话 Cookie 的 domain 与 serve 出来的 origin 不一致时不会报错,只是没带上。 + */ +import { test, expect } from "@playwright/test"; + +import { + assertReadOnlySQL, + assertSanctionedURL, + mysqlInvocation, + parseArgs, + parseRoleSpec, + resolveTargetURL, + scenarioPaths, + sessionCookie, + shotPath, + splitLocator, +} from "../lib/drive-target.mjs"; + +const SESSION = { baseURL: "http://127.0.0.1:41234" }; + +test("裸命令解析成命令名与原样的位置参数", () => { + expect(parseArgs(["goto", "/devices"])).toEqual({ + command: "goto", + rest: ["/devices"], + flags: {}, + }); +}); + +test("带值的开关吃掉它的值,不把值留成位置参数", () => { + const { command, rest, flags } = parseArgs([ + "shot", + "v1-before", + "--scenario", + "2026-08-13-console", + ]); + expect(command).toBe("shot"); + expect(rest).toEqual(["v1-before"]); + expect(flags.scenario).toBe("2026-08-13-console"); +}); + +test("拼错的开关当场失败,而不是被当成布尔悄悄吞掉", () => { + // --headles 被吞掉时命令照样返回成功,而你以为自己开了有头浏览器 + expect(() => parseArgs(["up", "--headles"])).toThrow( + /unknown flag: --headles/, + ); +}); + +test("位置参数里的空格与特殊字符原样保留", () => { + expect(parseArgs(["fill", "#user_code", "A4F 7Q2"]).rest).toEqual([ + "#user_code", + "A4F 7Q2", + ]); +}); + +test("选择器 DSL 认前缀", () => { + expect(splitLocator("testid=nav-devices")).toEqual({ + kind: "testid", + value: "nav-devices", + }); + expect(splitLocator("text=批准")).toEqual({ kind: "text", value: "批准" }); +}); + +test("CSS 属性选择器里的 = 不被当成 DSL 前缀", () => { + // 按 indexOf("=") 切会得到 kind="button[aria-label",整条选择器就废了 + expect(splitLocator("button[aria-label^='Theme']")).toEqual({ + kind: "", + value: "button[aria-label^='Theme']", + }); +}); + +test("role 选择器解析出角色与可选的无障碍名", () => { + expect(parseRoleSpec('button[name="Save"]')).toEqual({ + role: "button", + name: "Save", + }); + expect(parseRoleSpec("button")).toEqual({ role: "button", name: undefined }); +}); + +test("写坏的 role 选择器当场失败并给出正确形状", () => { + expect(() => parseRoleSpec("button[Save]")).toThrow( + /role=button\[name="Save"\]/, + ); +}); + +test("只驱动本次 target 自己的 origin", () => { + expect( + assertSanctionedURL(SESSION, "http://127.0.0.1:41234/devices"), + ).toContain("/devices"); +}); + +test("拒绝驱动外网地址", () => { + expect(() => + assertSanctionedURL(SESSION, "https://console.example.com/devices"), + ).toThrow(/refusing to drive/); +}); + +test("拒绝驱动同机上别的端口——那可能是你正在手调的另一套", () => { + expect(() => assertSanctionedURL(SESSION, "http://127.0.0.1:5174/")).toThrow( + /refusing to drive/, + ); +}); + +test("以 / 开头的目标按 baseURL 解析", () => { + expect(resolveTargetURL("/devices", SESSION.baseURL)).toBe( + "http://127.0.0.1:41234/devices", + ); +}); + +test("裸词当成路径,不当成协议或主机", () => { + // new URL("devices:8443") 会解析出协议 "devices:",浏览器停在一个不存在的地址上 + expect(resolveTargetURL("devices", SESSION.baseURL)).toBe( + "http://127.0.0.1:41234/devices", + ); +}); + +test("没有 baseURL 又给了相对路径时当场失败并说清缺什么", () => { + expect(() => resolveTargetURL("/devices", "")).toThrow(/baseURL/i); +}); + +test("oracle 只读:SELECT / WITH / EXPLAIN 放行", () => { + expect(() => assertReadOnlySQL(" select 1 ")).not.toThrow(); + expect(() => + assertReadOnlySQL("WITH x AS (select 1) select * from x"), + ).not.toThrow(); +}); + +test("oracle 拒绝一切写操作——验证观察状态,不制造状态", () => { + for (const q of [ + "delete from users", + "UPDATE devices set status='approved'", + "insert into users values (1)", + "truncate users", + ]) { + expect(() => assertReadOnlySQL(q)).toThrow(/read-only/); + } +}); + +test("Go 的 MySQL DSN 拆成 mysql 客户端的参数", () => { + const { args } = mysqlInvocation( + "server:secret@tcp(db.invalid:3306)/server_dev?charset=utf8mb4&parseTime=True", + "select 1", + ); + expect(args).toEqual([ + "-h", + "db.invalid", + "-P", + "3306", + "-u", + "server", + "--batch", + "--table", + "-e", + "select 1", + "server_dev", + ]); +}); + +test("口令走环境变量,不进命令行——否则 ps 能看到整条库口令", () => { + const { args, env } = mysqlInvocation( + "u:s3cr3t@tcp(127.0.0.1:3306)/db", + "select 1", + ); + expect(env.MYSQL_PWD).toBe("s3cr3t"); + expect(args.join(" ")).not.toContain("s3cr3t"); +}); + +test("口令里带 @ 或 : 也能正确切开", () => { + // 按第一个 @ 切会把口令截断,连上去的是另一套凭据(或者直接连不上) + const { args, env } = mysqlInvocation( + "root:p@ss:word@tcp(127.0.0.1:3306)/db", + "select 1", + ); + expect(env.MYSQL_PWD).toBe("p@ss:word"); + expect(args).toContain("root"); +}); + +test("库名后面的查询参数不算库名的一部分", () => { + const { args } = mysqlInvocation( + "u:p@tcp(h:3306)/mydb?parseTime=True", + "select 1", + ); + expect(args).toContain("mydb"); + expect(args.join(" ")).not.toContain("parseTime"); +}); + +test("不是 MySQL DSN 时当场说清,而不是把一串垃圾丢给客户端", () => { + expect(() => + mysqlInvocation("postgres://u:p@127.0.0.1:5432/db", "select 1"), + ).toThrow(/MySQL DSN/i); +}); + +test("场景目录把日志、截图、资源各归各位", () => { + const dirs = scenarioPaths("/tmp/scratch", "2026-08-13-console"); + expect(dirs.root).toBe("/tmp/scratch/2026-08-13-console"); + expect(dirs.logs).toBe("/tmp/scratch/2026-08-13-console/logs"); + expect(dirs.screenshots).toBe("/tmp/scratch/2026-08-13-console/screenshots"); +}); + +test("截图落在场景目录的 screenshots/ 下并自动补扩展名", () => { + expect(shotPath("/tmp/scn", "v1-before")).toBe( + "/tmp/scn/screenshots/v1-before.png", + ); + expect(shotPath("/tmp/scn", "v1-before.png")).toBe( + "/tmp/scn/screenshots/v1-before.png", + ); +}); + +test("名字里的 .. 逃逸被拒绝,证据只能留在场景目录里", () => { + expect(() => shotPath("/tmp/scn", "../../etc/passwd")).toThrow(/scenario/i); +}); + +test("会话 Cookie 的 domain 取 serve 出来的那个 host,不写死 localhost", () => { + expect( + sessionCookie({ + serverURL: "http://127.0.0.1:41234", + cookieName: "server_session", + sid: "seeded-sid", + }), + ).toEqual({ + name: "server_session", + value: "seeded-sid", + domain: "127.0.0.1", + path: "/", + httpOnly: true, + secure: false, + sameSite: "Lax", + }); +}); + +test("serve 环境缺 sid 时当场失败,而不是开一个没登录的浏览器", () => { + expect(() => + sessionCookie({ + serverURL: "http://127.0.0.1:41234", + cookieName: "server_session", + }), + ).toThrow(/sid/i); +}); diff --git a/e2e/web/runner-config-dir.spec.ts b/e2e/web/runner-config-dir.spec.ts new file mode 100644 index 00000000..869cfeb1 --- /dev/null +++ b/e2e/web/runner-config-dir.spec.ts @@ -0,0 +1,45 @@ +/** + * runner 从哪儿取 configs/config.yaml。 + * + * 守的是一处真实的死路:本仓的 `configs/config.yaml` 可以是 `source: etcd` —— + * 那时整份配置(含 http.address、db、redis、jwt 路径)都由 etcd 给,cago 会把 + * 本地文件整个换掉(cago configs/config.go 的 init:source 非 file 时替换 source)。 + * 于是 runner 既读不到 db.dsn 去播种,它起的 server 也会照 etcd 去绑 0.0.0.0:8443, + * 把「跑在一个空闲端口上」这条隔离直接作废。 + * + * 出口是让开发者显式指一份**本机可用**的配置,而不是让 runner 去猜或去改写 etcd。 + */ +import { test, expect } from "@playwright/test"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { configCheckoutFor } from "../run-e2e-web.mjs"; + +function checkoutWithConfig(): string { + const dir = mkdtempSync(join(tmpdir(), "webe2e-cfgdir-")); + mkdirSync(join(dir, "configs"), { recursive: true }); + writeFileSync(join(dir, "configs", "config.yaml"), 'db:\n dsn: "x"\n'); + return dir; +} + +test("默认用本检出自己的 configs/config.yaml", () => { + const checkout = checkoutWithConfig(); + expect(configCheckoutFor(checkout, {})).toBe(checkout); +}); + +test("WEBE2E_CONFIG_DIR 指到哪就用哪 —— etcd 档下的唯一出口", () => { + const checkout = checkoutWithConfig(); + const override = checkoutWithConfig(); + expect(configCheckoutFor(checkout, { WEBE2E_CONFIG_DIR: override })).toBe( + override, + ); +}); + +test("指了一个没有 configs/config.yaml 的目录时当场失败,并说清期望的形状", () => { + const checkout = checkoutWithConfig(); + const empty = mkdtempSync(join(tmpdir(), "webe2e-empty-")); + expect(() => + configCheckoutFor(checkout, { WEBE2E_CONFIG_DIR: empty }), + ).toThrow(/configs\/config\.yaml/); +}); diff --git a/e2e/web/runner-serve.spec.ts b/e2e/web/runner-serve.spec.ts new file mode 100644 index 00000000..6bd20be0 --- /dev/null +++ b/e2e/web/runner-serve.spec.ts @@ -0,0 +1,103 @@ +/** + * run-e2e-web.mjs 的运行模式选择与 serve 交接件。不起服务器:这里驱动的是 + * 「这次运行到底要不要跑 spec」以及「撑住不退时把什么交给 drive」两个纯函数。 + * + * 守的是两处: + * 1. 未知开关会原样漏给 playwright —— 原来的写法只滤掉 --dual,新增的 --serve + * 会被当成 playwright 的参数,运行时死在 "unknown option",看起来像 runner 坏了; + * 2. serve 交接件少一样东西(尤其 sid)时不会报错,drive 会开出一个没登录的 + * 浏览器,然后一路验的都是登录页。 + */ +import { test, expect } from "@playwright/test"; + +import { parseRunnerArgs, serveEnvPayload } from "../run-e2e-web.mjs"; + +test("默认跑 spec,不进 serve 档", () => { + expect(parseRunnerArgs([])).toEqual({ + mode: "spec", + dual: false, + playwrightArgs: [], + }); +}); + +test("--dual 仍然只切换双端,并且不漏给 playwright", () => { + expect(parseRunnerArgs(["--dual"])).toEqual({ + mode: "spec", + dual: true, + playwrightArgs: [], + }); +}); + +test("--serve 切到撑住不退档,并且不漏给 playwright", () => { + // 漏过去的话 playwright 会死在 unknown option —— 而 serve 档根本不该起 playwright + expect(parseRunnerArgs(["--serve"])).toEqual({ + mode: "serve", + dual: false, + playwrightArgs: [], + }); +}); + +test("--serve 与 --dual 同时给时两个都认,仍不漏参数", () => { + expect(parseRunnerArgs(["--serve", "--dual"])).toEqual({ + mode: "serve", + dual: true, + playwrightArgs: [], + }); +}); + +test("其余参数原样透传给 playwright", () => { + expect(parseRunnerArgs(["-g", "device flow", "--headed"])).toEqual({ + mode: "spec", + dual: false, + playwrightArgs: ["-g", "device flow", "--headed"], + }); +}); + +test("serve 交接件带齐 drive 登录所需的三样,以及排错要看的日志", () => { + const payload = serveEnvPayload({ + WEBE2E_SERVER_URL: "http://127.0.0.1:41234", + WEBE2E_COOKIE_NAME: "server_session", + WEBE2E_SESSION_SID: "seeded-sid", + WEBE2E_SERVER_LOG: "/tmp/aw1/server.log", + WEBE2E_AGENTRED_LOG: "/tmp/aw1/agentred.log", + WEBE2E_RUN_ID: "abc123", + }); + expect(payload).toMatchObject({ + serverURL: "http://127.0.0.1:41234", + cookieName: "server_session", + sid: "seeded-sid", + serverLog: "/tmp/aw1/server.log", + agentredLog: "/tmp/aw1/agentred.log", + runID: "abc123", + }); +}); + +test("交接件带上 DSN —— 没有它 `drive sql` 这个独立 oracle 就用不了", () => { + const payload = serveEnvPayload( + { + WEBE2E_SERVER_URL: "http://127.0.0.1:41234", + WEBE2E_COOKIE_NAME: "server_session", + WEBE2E_SESSION_SID: "seeded-sid", + }, + { dsn: "u:p@tcp(127.0.0.1:3306)/agentre" }, + ); + expect(payload.dsn).toBe("u:p@tcp(127.0.0.1:3306)/agentre"); + // 没传就是 null,而不是 undefined —— 交接件里这一栏必须存在,drive 才能给出 + // 「这次没有 DSN,自己去查库」而不是一句 undefined + expect( + serveEnvPayload({ + WEBE2E_SERVER_URL: "http://127.0.0.1:41234", + WEBE2E_COOKIE_NAME: "server_session", + WEBE2E_SESSION_SID: "seeded-sid", + }).dsn, + ).toBeNull(); +}); + +test("serve 交接件缺 sid 时当场失败,而不是交出一份开不出登录态的环境", () => { + expect(() => + serveEnvPayload({ + WEBE2E_SERVER_URL: "http://127.0.0.1:41234", + WEBE2E_COOKIE_NAME: "server_session", + }), + ).toThrow(/WEBE2E_SESSION_SID/); +});