diff --git a/apps/console/src/app/(console)/grants/client-caption.ts b/apps/console/src/app/(console)/grants/client-caption.ts index cec53ebd9..938bda233 100644 --- a/apps/console/src/app/(console)/grants/client-caption.ts +++ b/apps/console/src/app/(console)/grants/client-caption.ts @@ -1,6 +1,13 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 +/** + * Shared "client {…}" caption logic for anywhere a raw `client_id` would + * otherwise render verbatim (e.g. `cli_8f3a2b1c`). `/grants` had this logic + * inline to avoid a bare technical id; the grant-packages pages didn't reuse + * it and printed the raw id straight through. + */ + const TECHNICAL_CLIENT_ID_RE = /^cli_[a-z0-9]+$/i; const WWW_PREFIX_RE = /^www\./; @@ -18,6 +25,11 @@ export function clientOriginCaption(value: string): string | null { } } +/** + * Caption for a client identified only by `client_id` (no display name on + * the record) — an OAuth client origin renders as its host, a technical id + * (`cli_…`) renders as "registered client", anything else renders verbatim. + */ export function technicalClientCaption(clientId: string | null | undefined): string | null { const trimmed = clientId?.trim(); if (!trimmed) { @@ -28,6 +40,11 @@ export function technicalClientCaption(clientId: string | null | undefined): str ); } +/** + * Caption for a client that may carry a display name (`client.client_name`) + * in addition to `client_id` — prefers the name, falls back to + * `technicalClientCaption`. + */ export function clientCaption(client: { client?: { client_name?: string | null } | null; client_id?: string | null; diff --git a/docs/reference/connector-authoring-guide.md b/docs/reference/connector-authoring-guide.md index c7b9c3c60..b9207927a 100644 --- a/docs/reference/connector-authoring-guide.md +++ b/docs/reference/connector-authoring-guide.md @@ -2,6 +2,8 @@ This guide is for first-party connector manifests. Keep it open while adding or reviewing a connector. +To run and prove out a connector's `collect()` behavior locally — the `connector-dev`/`scenario-record`/`scenario-verify` loop — see [`connector-verification-workflow.md`](./connector-verification-workflow.md). + The goal is not "valid JSON." The goal is an honest, useful record surface: clients can search what should be searched, filter what should be filtered, group what should be grouped, display records without guessing, and ask for bounded follow-up reads before full fetch/export. ## Required Review diff --git a/docs/reference/connector-evidence-claims.md b/docs/reference/connector-evidence-claims.md new file mode 100644 index 000000000..f698d6279 --- /dev/null +++ b/docs/reference/connector-evidence-claims.md @@ -0,0 +1,110 @@ +# Connector Evidence Claims + +This document defines the vocabulary connectors use to describe what has been established about them, and what has not. Keep it open when generating evidence metadata, reviewing a publish, or writing anything that touches connector status. + +## Purpose + +"Verified" is banned as a connector status word. The word hides at least four different propositions — that a connector speaks the protocol correctly, that it correctly processed one real interaction, that it contacted the provider on some date, and that it works against the provider right now — and collapsing them into one word is how verification labels rot. WHOOP shipped with a live-run claim that was prose in a PR comment and passed every automated gate anyway, because no gate tested provider contact. A single word cannot carry four different guarantees without eventually being read as the strongest one. + +The fix is to report claims separately. Each claim below is machine-readable, dated where it applies, and asserted or withheld on its own — never merged into a composite score or a bronze/silver/gold rung. A connector can legitimately show three claims passing and two withheld; that is an honest status, not a partial failure. Labels are written by tooling from observed evidence, not typed by connector authors, so a claim can never be stronger than what actually produced it. + +## Functional evidence claims + +Five claims describe what a connector has been shown to do. They stack in the sense that later claims are harder to obtain, but they are reported independently — passing one does not imply another. + +| Claim | Definition | What establishes it | What it does NOT establish | Who can assert it | +|---|---|---|---|---| +| `protocol_conformant` | The connector speaks the Collection Profile correctly: manifest shape, process state machine, JSONL message contract. | Wire/conformance tests run against the built package. | Provider compatibility. A connector can be perfectly conformant and never successfully contact a real provider. | Tool only. | +| `recorded_replay` | The connector correctly processes a specific, dated, recorded provider interaction. | Black-box replay of a connector-verification scenario against the connector source bound by declaration and source-tree digests. (Binding to the built distributable package arrives with the publication pipeline; until then the claim binds source, and says so.) | That the provider still behaves this way. Replay proves faithful reprocessing of the past, not current compatibility. Replay's network denial covers the connector process (fetch, http/https, raw sockets) and — where OS namespace isolation is available — its descendants; when only process-local denial is active, the status says `network isolation: process-local only`. | Tool only. | +| `author_live` | The connector contacted the real provider successfully, on a specific date, from the author's own account. | **Withheld by all current tooling.** Establishing it requires tool-observed contact matching a per-connector provider-authority policy (accepted origins), which is designed but not built; today's recorder proves only `non_loopback_contact_observed` (any remote endpoint qualifies — a synthetic server or proxy would pass), and `connector-dev` observes protocol output, not network authority. No tool prints this claim until the authority policy exists. | Independent verification. The author's own run is not checked by anyone else. | Tool only (the run is tool-generated; the author supplies the account). | +| `independent_live` | The connector contacted the provider successfully, on a specific date, verified by a second party with their own account. | A live run performed and reported by someone other than the author. | Future behavior. A pass today says nothing about tomorrow. | Second party. | +| `scheduled_live` (future tier) | The connector is currently working, within a defined monitoring window. | A recurring, scheduled live probe against the provider. | Anything outside the probe window — universal account coverage, data-shape coverage, or behavior for accounts unlike the probe account. | Tool only, on a schedule. | + +`protocol_conformant` and `recorded_replay` can be established with no provider account at all. `author_live` and `independent_live` require an account. `scheduled_live` is not built in v1; the evidence format is designed so it can be added later without redefining the other four. + +## Disclosure classes + +Disclosure is orthogonal to functional evidence. A connector can have strong functional evidence and still disclose nothing publicly — evidence generation is default-on, but sharing is always explicit opt-in, never default. + +| Class | Definition | What may leave the author's machine | +|---|---|---| +| `local_only` | Evidence exists only on the author's machine. | Nothing. No artifact, summary, or count is published or sent to a reviewer. | +| `private_reviewer` | Evidence is shared with a trusted maintainer or reviewer, not published. | Raw or lightly-redacted evidence, sent to a specific named reviewer under the same handling rules as personal data. Not public. | +| `public_synthetic` | Evidence is published, built from synthetic (non-real) data. | Synthetic request/response pairs and outputs. No real personal data of any kind. | +| `public_derived` | Evidence is published, derived from a real run but transformed before publication. | Derived fixtures: real structure and behavior, with real values replaced or generalized. Never pattern-preserving for pattern-identifying classes (see below). | +| `public_scrubbed_real` | Evidence is published, built from a real run with deterministic and LLM-assisted redaction applied. | Scrubbed real records: real shape and largely real values, with credentials, identifiers, and sensitive fields removed or replaced. Prohibited outright for the sensitive classes listed below. | + +A connector's status can honestly read "local-only evidence, replay pass, author-live 2026-08-13, independent-live not available." That is a complete, publishable status — not a placeholder for something better later. + +## Recency fields and the aging rule + +Two fields track how current a claim is: + +- `captured_at` — when the underlying scenario or artifact was recorded. +- `live_verified_at` — when a live claim (`author_live`, `independent_live`, `scheduled_live`) was last confirmed. + +The aging rule has two halves, and they do not share a threshold: + +- **Replay scenarios never expire as regression evidence.** A `recorded_replay` pass from a year ago is still a valid regression signal — it proves the connector still processes that dated interaction correctly. Its age is always displayed alongside the claim, so a reader can judge staleness themselves, but the claim itself does not lapse. +- **Live claims age separately and independently.** `author_live` and `independent_live` are claims about a specific date, not standing facts. Their age is displayed the same way, but nothing here defines a global cutoff after which a live claim becomes invalid. + +There is no universal freshness threshold in v1. A stable public API and a scraped browser session age at different rates, and picking one number for both would be arbitrary. Source-specific live-check policies are left for a later support tier once real aging data exists. + +## Scenario-coverage flags + +A `recorded_replay` claim carries flags describing which behaviors the underlying scenario actually exercised. These are not pass/fail on their own — they scope what the replay pass means. + +| Flag | What it covers | +|---|---| +| `empty_state_run` | A run from empty state with real interactions and expected records. (Renamed from `full_refresh`: the producer does not prove every declared stream was exercised or accounted for, so the flag names only what it observes.) | +| `state_seeded_second_run_with_changed_requests` | A later run seeded from an earlier run's non-trivial committed state whose recorded requests differ. (Renamed from `incremental_two_run`: this proves state seeding changed request planning — not overlap handling, duplicate suppression, or safe failure behavior, which need dedicated scenario fixtures.) | +| `pagination` | Multi-page responses and page-to-page continuation. | +| `retry` | Recovery from a transient failure (rate limit, timeout, transient server error) within a run. | +| `partial_failure` | Recovery when part of a run fails without over-advancing committed state. | +| `auth_reuse` | Reuse of an existing authenticated session across requests or runs, without re-authenticating live. | + +A connector with only `empty_state_run` coverage has a narrower, honestly-scoped replay claim than one with all six flags set. Coverage flags are reported, not averaged into a single score. + +Producer status (kept honest, per this document's own rule): today's tooling computes `empty_state_run` and `state_seeded_second_run_with_changed_requests` under exactly the conditions their names state, and captures/compares the normalized protocol trace — SKIP_RESULT with continuation evidence, DETAIL_COVERAGE, DETAIL_GAP with digested locator/pressure evidence, DETAIL_GAP_ATTEMPTED/RECOVERED, DETAIL_GAPS_PAGE_REQUEST, and terminal DONE semantics — under a compile-time-exhaustive policy over the runtime message union: a new message kind cannot be added without being dispositioned, and a run exercising an unsupported evidence surface (ASSISTANCE) has the canonical replay claim withheld. The remaining four flags — `pagination`, `retry`, `partial_failure`, `auth_reuse` — are defined vocabulary with **no producer yet**; nothing sets them, and any status displaying them before a producer exists is lying. They arrive with fault-variant scenarios. + +Exactness note: `derived-from-real` is NOT currently produced by any tool. Captures with observed remote contact earn `non_loopback_contact_observed` — the exact observed fact — because any remote endpoint (a synthetic server, a proxy) satisfies the observation. `derived-from-real` becomes producible only when a per-connector provider-authority policy (accepted origins) exists to check contact against. + +## Provenance classes + +Every claim also carries a provenance class describing where the label came from: + +- `tool_generated` — produced mechanically by tooling from an observed run or replay, with no author input into the label text. +- `author_asserted` — a claim the author states but that tooling cannot independently observe (used sparingly; prefer `tool_generated` wherever possible). +- `independently_observed` — produced by a second party's tooling-generated run, not the author's. + +Labels are written by tooling, never typed by authors. An author does not get to write "author-live: pass" in a manifest or PR description; the `dev`/run-and-watch command generates that line from an actual run. Enforcement today: the fixture-provenance test suite requires every pilot fixture set to carry a tool-written provenance label of valid shape, and `scenario-record` computes `evidence_class` from observed provider contact rather than accepting an author-supplied value. A fuller CI lint — cross-checking every displayed label against the evidence artifact that must have produced it — is designed but not yet built; until it exists, that check is review discipline, not a gate. This is what keeps the WHOOP failure mode — a real live run reduced to unverifiable prose — from recurring. + +## Sensitive-class defaults + +Health, biometric, financial, messages, location, and contacts connectors default to `local_only` or `private_reviewer` disclosure. An author must take an explicit, separate action to move evidence for these classes to any public disclosure class. + +Pattern-preserving scrubbed recordings are prohibited for pattern-identifying classes — the classes above, plus any stream where record counts, timing, or category distribution could identify the author or people connected to them. The reason: for these classes, the pattern *is* the fingerprint. Redacting a value while preserving its shape (a constant timestamp shift, a token-for-token substitution) still preserves cadence, weekly structure, counts, and distributions, and those are frequently as identifying as the redacted values themselves. Scrubbing a body but leaving 340 messages sent every weekday between 9pm and 11pm intact does not protect the author. + +Before any evidence artifact is shared beyond the author's machine — `private_reviewer` or higher — a mandatory third-party-data check runs first. An author's export routinely contains other people who did not consent to appearing in it: message senders, calendar attendees, transaction counterparties, contacts. This check is not optional and is not satisfied by the author's own consent alone. + +All shared evidence, at every disclosure class above `local_only`, is pseudonymized personal data in the GDPR sense. It is never described as anonymized. Pseudonymization reduces risk; it does not remove the data from personal-data handling obligations, because it can still be linked back to an individual — directly through retained structure, or indirectly through pattern. + +## What no combination of claims ever means + +No combination of the claims above, at any coverage or disclosure level, ever means: + +- **That the connector works against the provider right now.** Even `scheduled_live`, when it exists, only covers its probe window and probe account — not every account shape, not the exact moment a reader looks at the status. +- **That the provider hasn't changed since capture.** `recorded_replay`, `author_live`, and `independent_live` are all claims about a specific date. Providers change endpoints, response shapes, and auth flows without notice, and no claim here detects that on its own. +- **That a recording proves the semantic correctness of the mapping.** This is the candidate-oracle rule: a recorded scenario is generated by the same connector implementation being evaluated. If the connector maps a field wrong, drops a nested value, or mislabels a timestamp, replay of that recording reproduces the bug faithfully rather than catching it. A `recorded_replay` pass proves the connector processes that dated interaction the same way it did when captured — not that the processing was correct in the first place. + +## Lifecycle + +A scenario starts as a **candidate oracle**, not a trusted one. It was produced by the implementation under test, so by default it can only prove regression safety and faithful reprocessing — not that the original mapping was right. + +Promotion from candidate to a scenario that can support stronger claims requires, proportionate to what will be shared or relied on: + +- **Declaration-to-output coverage** — every declared stream in the scenario is exercised by a run, or explicitly marked skipped. A stream the scenario never touches cannot be silently assumed correct. +- **Negative controls** — the scenario is deliberately broken (a mapping altered, a request corrupted) and replay is confirmed to fail. A scenario that cannot fail is not evidence of anything. +- **Human mapping review, when evidence is shared** — a person checks the response-to-record mapping by hand before the scenario supports any disclosure class above `local_only`. This is the step that catches what the connector's own code cannot catch about itself. + +A scenario that has not gone through this lifecycle can still back a `recorded_replay` claim for local regression use. It cannot back a claim that leaves the author's machine, and it never backs a claim of semantic correctness regardless of disclosure class. diff --git a/docs/reference/connector-verification-workflow.md b/docs/reference/connector-verification-workflow.md new file mode 100644 index 000000000..fae9ce551 --- /dev/null +++ b/docs/reference/connector-verification-workflow.md @@ -0,0 +1,78 @@ +# Connector Verification Workflow + +This guide is for connector authors who need to run and prove out a connector locally, without reading the source. Keep it open while developing or debugging a connector's `collect()` behavior. + +It covers three commands that form one loop: `connector-dev` (watch a connector run), `scenario-record` (capture what it did against your real account), and `scenario-verify` (replay that capture offline and check it still matches). The vocabulary these commands use — `recorded_replay`, `author_live`, coverage flags, disclosure classes — is defined in [`connector-evidence-claims.md`](./connector-evidence-claims.md). Read that document for what each claim does and does not establish; this guide only covers how to run the tools that produce the evidence. + +## The loop + +### 1. Run and watch — `connector-dev` + +``` +pnpm exec tsx bin/connector-dev.ts +pnpm exec tsx bin/connector-dev.ts ynab +pnpm exec tsx bin/connector-dev.ts gmail --summary-out /tmp/gmail-run.json +``` + +Spawns the connector's own entrypoint exactly the way production does, and streams every `RECORD`/`STATE`/`SKIP_RESULT`/`PROGRESS`/`INTERACTION` message live as it arrives. Auth is resolved from your environment, same as production. Nothing is persisted to a Record Store; this is a local dev loop for watching one connector's behavior against its real upstream, not an end-to-end ingest proof. + +If the connector prompts mid-run (OTP, manual action), `connector-dev` renders the prompt in the terminal and sends your answer back; non-interactive runs supply answers with `--answer =` or `--answers `, and fail loudly naming the prompt when no answer is available. + +When the run finishes, it writes a mechanically-generated run summary to `runs//-summary.json` (or the path given to `--summary-out`) and prints per-stream record counts, `state_emitted`, and `latest_record_emitted_at`. (Those names are deliberate: no Record Store durability path runs here, so nothing is "committed," and `emitted_at` is connector processing time, not source freshness.) This run summary backs an `author_live` claim **only when the run showed observed, non-loopback provider contact** — a run against a local stub can never earn it. A run that exits nonzero or emits protocol output after DONE is a failure even if DONE said succeeded. + +### 2. Capture a scenario — `scenario-record` + +``` +pnpm exec tsx bin/scenario-record.ts +pnpm exec tsx bin/scenario-record.ts oura +pnpm exec tsx bin/scenario-record.ts oura --runs 1 --out /tmp/oura-run1.json +``` + +Runs the connector against your real account and real upstream, exactly like `connector-dev`, but with a preload that captures every HTTP request/response pair the run makes. By default it captures two runs: run 1 from empty state (full refresh), then run 2 immediately re-run seeded with run 1's actual committed state (incremental narrowing). Pass `--runs 1` to capture only the full-refresh run. + +Mid-run INTERACTION prompts (OTP, manual action) are captured too: the prompt/response pairs ride the scenario and are replayed scripted by `scenario-verify`, so an OTP-gated flow regression-tests with no human present. + +The result is a scenario file: `runs//-scenario.json`. Its `evidence_class` is **computed, never asserted**: `derived-from-real` requires tool-observed non-loopback provider contact; a capture from a loopback provider or a dev entrypoint override is labeled `synthetic-spike` mechanically. The file also carries declaration and source-tree digests binding it to the connector that produced it. This capture is **local-only** — it may contain real response bodies from your account and must not be committed or shared without a scrub pass. It is also a **candidate oracle**: it was produced by the same connector implementation it will later be replayed against, so it can prove faithful reprocessing and regression safety, not that the original field mapping was correct. See "What the evidence does and does not establish" below. + +### 3. Replay it offline — `scenario-verify` + +``` +pnpm exec tsx bin/scenario-verify.ts +pnpm exec tsx bin/scenario-verify.ts oura runs/oura/2026-08-13T00-00-00-000Z-scenario.json +``` + +First validates the scenario strictly (incomplete captures, zero runs, malformed shapes, and identity/digest mismatches are rejected before anything is spawned), then replays every run against the real connector code. Network denial covers the connector process itself — `fetch`, `http`/`https`, and raw sockets are all intercepted — and, where OS namespace isolation is available, its descendant processes too; when only process-local denial is active the output says `network isolation: process-local only` (a spawned external client like `curl` is outside that boundary). It checks that the connector produces exactly the recorded streams (extra streams fail), the same records, ids, content hashes, and final state, emits valid protocol output only, ends with a single final DONE, and exits zero. + +On a pass, it prints the claim and the coverage flags the scenario actually exercised, for example: + +``` +recorded_replay: PASS (captured 2026-08-13T00:00:00.000Z) +coverage: empty_state_run, state_seeded_second_run_with_changed_requests +``` + +If the scenario has a second run but that run's requests are identical to the first run's, `state_seeded_second_run_with_changed_requests` is withheld and a note explains why — see the honesty rule below. + +## Artifacts + +| Artifact | Where it lives | What it is | +|---|---|---| +| Run summary | `runs//-summary.json` | Mechanically generated by `connector-dev`: per-stream record counts, `state_emitted`, `latest_record_emitted_at`, skips. Backs `author_live` only with observed non-loopback provider contact. | +| Scenario file | `runs//-scenario.json` | Written by `scenario-record`. A `pdpp.connector-scenario/1` envelope (`src/scenario/format.ts`): every HTTP request/response pair a run made, plus what the run is expected to produce (per-stream record counts, ids, content hashes, and the final committed state). `verify.ts` replays it offline against the real connector and proves the two match. | +| `provenance.json` | `fixtures//scrubbed/pilot-real-shape/provenance.json` | Labels a committed fixture's origin, e.g. `{"format": "pdpp.fixture-provenance/1", "class": "synthetic", "labeled_by": "tool:provenance-labeler/1", "labeled_at": "2026-08-13"}`. Distinct from `runs/` scenario files: fixtures here are the committed, scrubbed kind, not local captures. | + +`runs/` is listed in `packages/polyfill-connectors/.gitignore` — it is local-only and never committed. Do not hand-copy a file out of `runs/` into a committed fixture without going through a scrub pass (see `scrub-connector-fixtures`). + +## What the evidence does and does not establish + +Full definitions live in [`connector-evidence-claims.md`](./connector-evidence-claims.md). Two rules to hold onto while using these commands: + +**The candidate-oracle rule.** A scenario captured by `scenario-record` is generated by the same connector implementation `scenario-verify` later checks it against. If the connector maps a field wrong or drops a value, replay reproduces that bug faithfully instead of catching it. A `recorded_replay` pass proves the connector processes a dated interaction the same way it did at capture time — not that the original mapping was correct. + +**The state-seeded-run honesty rule.** The `state_seeded_second_run_with_changed_requests` flag is only claimed when a scenario's second run was actually seeded from the first run's committed state *and* that second run's recorded requests differ from the first run's. Two runs existing is not enough — `scenario-verify` checks that state seeding observably changed request planning. If the requests are identical or the seeded state was trivial, the flag is withheld and the tool says why. The replay oracle also compares the normalized protocol trace (skips, coverage, gaps, terminal error semantics), so a change that silently drops completeness evidence fails replay. + +## Current limitations + +- **API-class connectors only.** The capture/replay mechanism patches the subprocess's `fetch`. Browser-navigation connectors (patchright/playwright-driven) do not route their traffic through `fetch` in a way this captures, so they stay on live verification. File-import connectors make no network calls at all and need no scenario. +- **Response bodies are stored verbatim.** Provider-issued values in request params are stored as bindings (references into the response that issued them) rather than raw values, and capture temp files live in a private `0700` workspace — but response *bodies* are persisted as received, minus a size cap. Keep scenario files local — this is why `runs/` is gitignored — and do not record connectors that exchange long-lived tokens in their response bodies yet. +- **Auth flows are not captured.** The recorder captures data-collection requests and mid-run INTERACTION prompts, not the login/token-exchange sequence. Auth is resolved from your environment before the run starts, the same way it is in production. +- **Descendant processes escape process-local network denial.** A connector that spawns an external network client (`curl`, a child interpreter) is only contained when OS namespace isolation is available; otherwise replay honestly reports `process-local only` isolation. Connectors that spawn network helpers should not be treated as replay-eligible under process-local isolation. diff --git a/packages/polyfill-connectors/.gitignore b/packages/polyfill-connectors/.gitignore index 6416d0f39..a133ca1f4 100644 --- a/packages/polyfill-connectors/.gitignore +++ b/packages/polyfill-connectors/.gitignore @@ -16,3 +16,7 @@ fixtures/*/scrubbed/*/ # Local data + cache .pdpp-data/ .pdpp-tools/ + +# connector-dev run-summary output (bin/connector-dev.ts) — local proof +# artifacts from a developer's own runs, not reviewed/committable fixtures. +runs/ diff --git a/packages/polyfill-connectors/bin/connector-dev.test.ts b/packages/polyfill-connectors/bin/connector-dev.test.ts new file mode 100644 index 000000000..718b885ec --- /dev/null +++ b/packages/polyfill-connectors/bin/connector-dev.test.ts @@ -0,0 +1,596 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * End-to-end proof for `bin/connector-dev.ts` — the "run and watch it work" + * developer command — driven as a REAL subprocess (not an in-process + * import) against test-only fixture connectors, with no live credentials. + * + * Uses the `--entrypoint` dev/test-only override (see connector-dev.ts's + * module docstring) to point the CLI at + * `src/test-fixtures/connector-dev-cli-fixture.ts` and the existing + * `src/test-fixtures/protocol-subprocess-fails-after-record.ts` fixture + * instead of a registered production connector, so this proves the CLI's + * own spawn/stream/summarize/exit-code behavior without touching + * `src/orchestrator.ts`'s manifest registry. + */ + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import type { RunSummary } from "../src/run-summary.ts"; +import { + type CheckpointEvidence, + classifyFailureEnvironment, + type LastState, + resolveCaptureOnFailureEnv, +} from "./connector-dev.ts"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PACKAGE_ROOT = join(__dirname, ".."); +const CLI_PATH = join(PACKAGE_ROOT, "bin", "connector-dev.ts"); +const fixturePath = (name: string): string => join(PACKAGE_ROOT, "src", "test-fixtures", name); +/** `bin/connector-dev.ts`'s own `lastStatePath` — reimplemented here (not + * imported) so this test asserts on the SAME path convention a real + * developer would compute by hand, rather than trusting the module under + * test to describe its own output location correctly. */ +const lastStatePathFor = (connector: string): string => join(PACKAGE_ROOT, "runs", connector, "last-state.json"); + +function runCli(args: readonly string[]): { code: number | null; stdout: string; stderr: string } { + const result = spawnSync(process.execPath, ["--import", "tsx", CLI_PATH, ...args], { + cwd: PACKAGE_ROOT, + env: { + ...process.env, + PATCHRIGHT_SKIP_BROWSER_DOWNLOAD: "1", + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1", + }, + encoding: "utf8", + timeout: 30_000, + }); + return { code: result.status, stdout: result.stdout, stderr: result.stderr }; +} + +test("connector-dev CLI: succeeding fixture streams RECORD/PROGRESS/STATE lines, exits 0, writes a matching summary", () => { + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-test-")); + const summaryPath = join(tmpDir, "summary.json"); + try { + const result = runCli([ + "connector-dev-cli-fixture", + "--entrypoint", + fixturePath("connector-dev-cli-fixture.ts"), + "--summary-out", + summaryPath, + ]); + + assert.equal(result.code, 0, `expected exit 0; stderr=${result.stderr}`); + + // START echo. + assert.match(result.stdout, /START connector-dev-cli-fixture/); + // Live per-stream RECORD count line (first record prints immediately). + assert.match(result.stdout, /RECORD\s+\[items] 1 record\(s\) so far/); + // PROGRESS line surfaced verbatim. + assert.match(result.stdout, /PROGRESS\s+\[items] collecting synthetic items/); + // STATE commit line. + assert.match(result.stdout, /STATE\s+\[items] checkpoint committed/); + // The intentionally-invalid row becomes a SKIP_RESULT warning. + assert.match(result.stdout, /WARN\s+\[items] skip: shape_check_failed/); + // Terminal summary block. + assert.match(result.stdout, /DONE/); + assert.match(result.stdout, /items\s+3 record\(s\)\s+state_emitted=true/); + assert.match(result.stdout, /skips: 1/); + assert.match(result.stdout, new RegExp(`summary written to: ${summaryPath}`)); + assert.match(result.stdout, /STATUS succeeded/); + + assert.ok(existsSync(summaryPath), "summary file must be written"); + const summary = JSON.parse(readFileSync(summaryPath, "utf8")) as RunSummary; + assert.equal(summary.format, "pdpp.run-summary/1"); + assert.equal(summary.generated_by, "connector-dev"); + assert.equal(summary.connector, "connector-dev-cli-fixture"); + assert.equal(summary.streams.items?.records, 3); + assert.equal(summary.streams.items?.state_emitted, true); + assert.equal(summary.skips, 1); + assert.equal(summary.done.status, "succeeded"); + assert.ok(summary.duration_ms >= 0); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test("connector-dev CLI: failing fixture exits non-zero, prints the failure kind, and writes a failed summary", () => { + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-test-")); + const summaryPath = join(tmpDir, "summary.json"); + try { + const result = runCli([ + "protocol-subprocess-fails-after-record", + "--entrypoint", + fixturePath("protocol-subprocess-fails-after-record.ts"), + "--summary-out", + summaryPath, + ]); + + assert.notEqual(result.code, 0, "a terminal failure must exit non-zero"); + assert.match(result.stdout, /RECORD\s+\[items] 1 record\(s\) so far/); + assert.match(result.stdout, /FAILED\s+retryable: retry budget exhausted/i); + + assert.ok(existsSync(summaryPath), "summary file must still be written on failure"); + const summary = JSON.parse(readFileSync(summaryPath, "utf8")) as RunSummary; + assert.equal(summary.done.status, "failed"); + assert.equal(summary.done.error?.retryable, true); + assert.match(summary.done.error?.message ?? "", /retry budget exhausted/i); + assert.equal(summary.streams.items?.records, 1); + assert.equal(summary.streams.items?.state_emitted, false); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test("connector-dev CLI: default summary path is under runs// when --summary-out is omitted", () => { + const result = runCli(["connector-dev-cli-fixture", "--entrypoint", fixturePath("connector-dev-cli-fixture.ts")]); + + assert.equal(result.code, 0, `expected exit 0; stderr=${result.stderr}`); + const match = /summary written to: (.+runs\/connector-dev-cli-fixture\/.+-summary\.json)/.exec(result.stdout); + assert.ok(match, `expected default summary path in stdout; got: ${result.stdout}`); + const writtenPath = match?.[1]?.trim(); + assert.ok(writtenPath && existsSync(writtenPath), "default-path summary file must exist"); + if (writtenPath) { + rmSync(writtenPath, { force: true }); + } +}); + +// ─── Interaction answering (src/test-fixtures/connector-dev-interaction- +// fixture.ts emits ONE `otp` INTERACTION mid-run, then a record whose +// `otp_value` field is exactly the response value — see that fixture's doc +// comment for why this makes the answering path's effect observable) ────── + +test("connector-dev CLI: --answer = completes an INTERACTION and the run succeeds", () => { + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-interaction-test-")); + const summaryPath = join(tmpDir, "summary.json"); + try { + const result = runCli([ + "connector-dev-interaction-fixture", + "--entrypoint", + fixturePath("connector-dev-interaction-fixture.ts"), + "--answer", + "0=555111", + "--summary-out", + summaryPath, + ]); + + assert.equal(result.code, 0, `expected exit 0; stderr=${result.stderr}`); + assert.match(result.stdout, /PROMPT\s+needs otp: Enter the verification code/); + assert.match(result.stdout, /STATE\s+\[items] checkpoint committed/); + assert.match(result.stdout, /STATUS succeeded/); + assert.doesNotMatch(result.stdout, /PROMPT\s+FAILED/); + + const summary = JSON.parse(readFileSync(summaryPath, "utf8")) as RunSummary; + assert.equal(summary.done.status, "succeeded"); + assert.equal(summary.streams.items?.records, 2); + assert.equal(summary.streams.items?.state_emitted, true); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test("connector-dev CLI: no --answer and no TTY fails loudly, naming the unanswered prompt", () => { + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-interaction-test-")); + const summaryPath = join(tmpDir, "summary.json"); + try { + const result = runCli([ + "connector-dev-interaction-fixture", + "--entrypoint", + fixturePath("connector-dev-interaction-fixture.ts"), + "--summary-out", + summaryPath, + ]); + + assert.notEqual(result.code, 0, "an unanswered interaction with no TTY must fail non-zero"); + assert.match(result.stdout, /PROMPT\s+needs otp: Enter the verification code/); + assert.match( + result.stdout, + /PROMPT\s+FAILED \(no --answer, no TTY\): otp — Enter the verification code shown on your device\./ + ); + assert.match( + result.stdout, + /unanswered prompt: otp — Enter the verification code shown on your device\. \(request_id=/ + ); + assert.match(result.stdout, /FAILED\s+terminal:/); + + const summary = JSON.parse(readFileSync(summaryPath, "utf8")) as RunSummary; + assert.equal(summary.done.status, "failed"); + // The fixture's before-prompt record still made it through — proves the + // failure is specifically the unanswered interaction, not a spawn/crash. + assert.equal(summary.streams.items?.records, 1); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +// ─── DONE-finality honesty: a succeeded DONE is not self-certifying ─────── + +test("connector-dev CLI: a succeeded DONE followed by a nonzero exit is reported as a failure, not a success", () => { + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-done-then-exit1-test-")); + const summaryPath = join(tmpDir, "summary.json"); + try { + const result = runCli([ + "connector-dev-done-then-exit1-fixture", + "--entrypoint", + fixturePath("connector-dev-done-then-exit1-fixture.ts"), + "--summary-out", + summaryPath, + ]); + + assert.notEqual( + result.code, + 0, + `a DONE(succeeded) followed by exit 1 must still fail non-zero; stdout=${result.stdout}` + ); + assert.doesNotMatch(result.stdout, /STATUS succeeded/); + assert.match(result.stdout, /FAILED\s+protocol_violation: nonzero_exit_after_done/); + + // The summary artifact is still written (mirrors the other failure + // paths) and its own DONE.status is honestly "succeeded" — the CLI's + // exit code/printed FAILED line is what carries the real verdict, not a + // rewrite of the connector's own claim. + assert.ok(existsSync(summaryPath), "summary file must still be written"); + const summary = JSON.parse(readFileSync(summaryPath, "utf8")) as RunSummary; + assert.equal(summary.done.status, "succeeded"); + assert.equal(summary.streams.items?.records, 1); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +// ─── --streams and --seed-last-state (src/test-fixtures/connector-dev- +// scope-state-fixture.ts declares two streams, `items` and `extras` — +// matching connector-dev.ts's `ENTRYPOINT_MODE_STREAMS` — and echoes the +// requested stream set plus an incrementing per-stream cursor derived from +// incoming state, so both flags' effects are observable in the run's own +// output/artifacts rather than just exercised as inert plumbing) ───────── + +test("connector-dev CLI: --streams subsets START.scope — the fixture only sees and emits for the named streams", () => { + const connector = `connector-dev-streams-subset-${String(process.pid)}`; + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-streams-test-")); + const summaryPath = join(tmpDir, "summary.json"); + try { + const result = runCli([ + connector, + "--entrypoint", + fixturePath("connector-dev-scope-state-fixture.ts"), + "--streams", + "items", + "--summary-out", + summaryPath, + ]); + + assert.equal(result.code, 0, `expected exit 0; stdout=${result.stdout} stderr=${result.stderr}`); + // START echo names only the scoped stream, not the fixture's full set. + assert.match(result.stdout, new RegExp(`START ${connector} — streams: items$`, "m")); + // The fixture's own PROGRESS line proves `ctx.requested` (built from + // START.scope.streams by connector-runtime.ts) contained ONLY "items" — + // not that the CLI merely printed a narrower banner while still sending + // everything. + assert.match(result.stdout, /PROGRESS\s+\[items] requested streams: items$/m); + // No RECORD/STATE for the scoped-out "extras" stream at all. + assert.doesNotMatch(result.stdout, /\[extras]/); + + const summary = JSON.parse(readFileSync(summaryPath, "utf8")) as RunSummary; + assert.equal(summary.streams.items?.records, 1); + assert.equal(summary.streams.extras, undefined); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + rmSync(join(PACKAGE_ROOT, "runs", connector), { recursive: true, force: true }); + } +}); + +test("connector-dev CLI: --streams naming an unknown stream fails, listing the fixture's actual stream names", () => { + const connector = `connector-dev-streams-unknown-${String(process.pid)}`; + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-streams-test-")); + const summaryPath = join(tmpDir, "summary.json"); + try { + const result = runCli([ + connector, + "--entrypoint", + fixturePath("connector-dev-scope-state-fixture.ts"), + "--streams", + "items,bogus", + "--summary-out", + summaryPath, + ]); + + assert.notEqual(result.code, 0, "an unknown --streams name must fail non-zero"); + assert.match( + result.stdout, + /FAILED\s+--streams named unknown stream\(s\): bogus\. Available streams: items, extras/ + ); + // Fails BEFORE spawning the connector: no START/PROGRESS line at all. + assert.doesNotMatch(result.stdout, /^START/m); + assert.ok(!existsSync(summaryPath), "no summary should be written for a pre-flight arg failure"); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + rmSync(join(PACKAGE_ROOT, "runs", connector), { recursive: true, force: true }); + } +}); + +test("connector-dev CLI: --seed-last-state round-trips a prior run's committed cursor into the next run's START.state", () => { + const connector = `connector-dev-seed-roundtrip-${String(process.pid)}`; + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-seed-test-")); + const summaryPath1 = join(tmpDir, "summary-1.json"); + const summaryPath2 = join(tmpDir, "summary-2.json"); + const lastStatePath = lastStatePathFor(connector); + try { + // Run 1: no prior state, so the fixture's incoming cursor is empty and + // it commits `{ seen: 1 }` for both streams. + const result1 = runCli([ + connector, + "--entrypoint", + fixturePath("connector-dev-scope-state-fixture.ts"), + "--summary-out", + summaryPath1, + ]); + assert.equal(result1.code, 0, `run 1 failed; stdout=${result1.stdout} stderr=${result1.stderr}`); + assert.doesNotMatch(result1.stdout, /^SEEDED/m, "run 1 has no --seed-last-state, so no SEEDED line"); + + assert.ok(existsSync(lastStatePath), "last-state.json must exist after a DONE that emitted STATE"); + const lastStateAfterRun1 = JSON.parse(readFileSync(lastStatePath, "utf8")) as LastState; + assert.equal(lastStateAfterRun1.connector, connector); + assert.deepEqual(lastStateAfterRun1.state, { items: { seen: 1 }, extras: { seen: 1 } }); + + // Run 2: --seed-last-state reads run 1's committed cursor back into + // START.state — the fixture's own increment-from-incoming-state logic + // makes the seed's effect observable: `seen` goes from 1 to 2, which + // could only happen if the seeded value actually reached ctx.state. + const result2 = runCli([ + connector, + "--entrypoint", + fixturePath("connector-dev-scope-state-fixture.ts"), + "--seed-last-state", + "--summary-out", + summaryPath2, + ]); + assert.equal(result2.code, 0, `run 2 failed; stdout=${result2.stdout} stderr=${result2.stderr}`); + assert.match( + result2.stdout, + new RegExp(`^SEEDED state from ${lastStatePath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\(run of .+\\)$`, "m") + ); + + const lastStateAfterRun2 = JSON.parse(readFileSync(lastStatePath, "utf8")) as LastState; + assert.deepEqual(lastStateAfterRun2.state, { items: { seen: 2 }, extras: { seen: 2 } }); + + const summary2 = JSON.parse(readFileSync(summaryPath2, "utf8")) as RunSummary; + assert.equal(summary2.streams.items?.records, 1); + assert.equal(summary2.streams.extras?.records, 1); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + rmSync(join(PACKAGE_ROOT, "runs", connector), { recursive: true, force: true }); + } +}); + +test("connector-dev CLI: --seed-last-state with no prior run fails clearly, naming the missing file", () => { + const connector = `connector-dev-seed-missing-${String(process.pid)}`; + const tmpDir = mkdtempSync(join(tmpdir(), "connector-dev-seed-missing-test-")); + const summaryPath = join(tmpDir, "summary.json"); + const lastStatePath = lastStatePathFor(connector); + assert.ok(!existsSync(lastStatePath), "precondition: no prior last-state.json for this fresh connector name"); + try { + const result = runCli([ + connector, + "--entrypoint", + fixturePath("connector-dev-scope-state-fixture.ts"), + "--seed-last-state", + "--summary-out", + summaryPath, + ]); + + assert.notEqual(result.code, 0, "--seed-last-state with no prior state must fail non-zero"); + assert.match( + result.stdout, + new RegExp( + `FAILED --seed-last-state: no prior run state at ${lastStatePath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}; ` + + "run once without the flag first, then re-run with --seed-last-state\\." + ) + ); + // Fails BEFORE spawning the connector: no START line, no summary. + assert.doesNotMatch(result.stdout, /^START/m); + assert.ok(!existsSync(summaryPath), "no summary should be written for a pre-flight arg failure"); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + rmSync(join(PACKAGE_ROOT, "runs", connector), { recursive: true, force: true }); + } +}); + +// ─── FIX 1: default-on failure-evidence retention ───────────────────────── +// +// Per the grounding research (leading failure-diagnostic tools retain +// evidence by default in their primary run mode), `connector-dev` sets +// `PDPP_CAPTURE_ON_FAILURE=1` for the subprocess unless the developer +// explicitly opts out. `resolveCaptureOnFailureEnv` is pure and covered +// directly below; the CLI-level tests confirm the resolved value actually +// reaches the subprocess by reading it back via +// `connector-dev-env-echo-fixture.ts`'s stderr echo (a real stub the +// production code path writes through, not a parallel assertion route). + +test("resolveCaptureOnFailureEnv: unset environment defaults to on (1)", () => { + assert.equal(resolveCaptureOnFailureEnv(false, {}), "1"); +}); + +test("resolveCaptureOnFailureEnv: --no-capture disables regardless of environment", () => { + assert.equal(resolveCaptureOnFailureEnv(true, {}), undefined); + assert.equal(resolveCaptureOnFailureEnv(true, { PDPP_CAPTURE_ON_FAILURE: "1" }), undefined); +}); + +test("resolveCaptureOnFailureEnv: an explicit 0 already in the environment is respected, not overridden", () => { + assert.equal(resolveCaptureOnFailureEnv(false, { PDPP_CAPTURE_ON_FAILURE: "0" }), "0"); +}); + +test("resolveCaptureOnFailureEnv: an explicit 1 already in the environment stays 1", () => { + assert.equal(resolveCaptureOnFailureEnv(false, { PDPP_CAPTURE_ON_FAILURE: "1" }), "1"); +}); + +test("connector-dev CLI: default run sets PDPP_CAPTURE_ON_FAILURE=1 for the subprocess and prints the policy line", () => { + const result = runCli([ + "connector-dev-env-echo-fixture", + "--entrypoint", + fixturePath("connector-dev-env-echo-fixture.ts"), + ]); + + assert.equal(result.code, 0, `expected exit 0; stderr=${result.stderr}`); + assert.match(result.stdout, /^capture: on-failure \(default; --no-capture to disable\)$/m); + assert.match(result.stderr, /PDPP_CAPTURE_ON_FAILURE_ECHO=1/); +}); + +test("connector-dev CLI: --no-capture disables retention and the subprocess sees it unset", () => { + const result = runCli([ + "connector-dev-env-echo-fixture", + "--entrypoint", + fixturePath("connector-dev-env-echo-fixture.ts"), + "--no-capture", + ]); + + assert.equal(result.code, 0, `expected exit 0; stderr=${result.stderr}`); + assert.match(result.stdout, /^capture: disabled \(--no-capture\)$/m); + assert.match(result.stderr, /PDPP_CAPTURE_ON_FAILURE_ECHO=__unset__/); +}); + +test("connector-dev CLI: an explicit PDPP_CAPTURE_ON_FAILURE=0 already in the environment is passed through untouched", () => { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + CLI_PATH, + "connector-dev-env-echo-fixture", + "--entrypoint", + fixturePath("connector-dev-env-echo-fixture.ts"), + ], + { + cwd: PACKAGE_ROOT, + env: { + ...process.env, + PATCHRIGHT_SKIP_BROWSER_DOWNLOAD: "1", + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1", + PDPP_CAPTURE_ON_FAILURE: "0", + }, + encoding: "utf8", + timeout: 30_000, + } + ); + + assert.equal(result.status, 0, `expected exit 0; stderr=${result.stderr}`); + assert.match(result.stdout, /^capture: disabled \(PDPP_CAPTURE_ON_FAILURE=0 already set in environment\)$/m); + assert.match(result.stderr, /PDPP_CAPTURE_ON_FAILURE_ECHO=0/); +}); + +// ─── FIX 2: evidence + closed taxonomy (pure predicate coverage) ────────── +// +// `classifyFailureEnvironment` is a pure fold over checkpoint evidence, so +// it is exercised directly rather than through a real Playwright capture — +// synthesizing the exact metadata shape `readCheckpointEvidence` would have +// produced from real `pages/