diff --git a/Dockerfile b/Dockerfile index 0fbec5a12..2e6986af8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -272,6 +272,27 @@ FROM browsers AS core-browser ARG PDPP_REFERENCE_REVISION=unknown +# Image provenance. Without these the deployed artifact cannot say what source +# it was built from, and identifying production means md5-diffing files against +# candidate worktrees. Sampling files that way is actively misleading: a file +# unchanged between two commits matches BOTH, so a sample that happens to miss +# the changed files "confirms" the wrong commit. Labels remove the guesswork. +# +# PDPP_BUILD_DIRTY must be set from `git status --porcelain` at build time. A +# silently-dirty build tree is how bad images shipped before, so an unclean +# tree is recorded in the artifact rather than left to memory. +ARG PDPP_BUILD_REVISION=unknown +ARG PDPP_BUILD_SOURCE=unknown +ARG PDPP_BUILD_CREATED=unknown +ARG PDPP_BUILD_DIRTY=unknown +ARG PDPP_BUILD_COMPOSITION=unknown + +LABEL org.opencontainers.image.revision="${PDPP_BUILD_REVISION}" \ + org.opencontainers.image.source="${PDPP_BUILD_SOURCE}" \ + org.opencontainers.image.created="${PDPP_BUILD_CREATED}" \ + pdpp.build.dirty="${PDPP_BUILD_DIRTY}" \ + pdpp.build.composition="${PDPP_BUILD_COMPOSITION}" + # PDPP_LOCAL_TRANSFORMER_SUPERVISOR_RESTART_CONTRACT is baked in (unlike the # root docker-compose.yml `reference` service, which sets it explicitly in # compose env) because this image stage is deployed exclusively through diff --git a/apps/console/src/app/(console)/audit/error.tsx b/apps/console/src/app/(console)/audit/error.tsx index 68406f2e6..5afae5568 100644 --- a/apps/console/src/app/(console)/audit/error.tsx +++ b/apps/console/src/app/(console)/audit/error.tsx @@ -3,16 +3,60 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { SegmentError } from "../components/segment-error.tsx"; +import { useEffect } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts"; +import { ListLoadingSkeleton } from "../components/route-loading.tsx"; + +/** + * Audit-segment error boundary (App Router convention) — SLVP bar: Stripe, + * Linear, Vercel, and Plaid never tell an owner "we hit a transient read + * interruption, retrying." The page renders, or it quietly shows last-known + * state. The owner never learns the backend hiccuped. + * + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`): the read itself is fine — React's Flight/RSC + * streaming writer reacting to the HTTP response closing before the stream + * finished flushing. It is a client-transport race below the data layer, not + * a backend outage — see `sources/error.tsx` for the full original writeup. + * + * `/audit` has no client-cached last-known-read marker, so this boundary + * shows the plain skeleton with no staleness caption rather than fabricate a + * timestamp. + * + * Self-contained on purpose: a `"use client"` boundary must not import + * server-only modules, since the dashboard shell transitively pulls in + * `lib/owner-token.ts` (`server-only`). + */ + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. + */ +const retryCounter = createRetryCounter(); + +export default function AuditError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. + console.error(error); + }, [error]); + + useEffect(() => { + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // counter. There is deliberately no ceiling on the counter itself — a + // persistent failure degrades to a slow quiet heartbeat, never a dead end. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + reset(); + }, delay); + return () => clearTimeout(id); + }, [reset]); -export default function AuditError(props: { error: Error & { digest?: string }; reset: () => void }) { return ( - +
+ +
); } diff --git a/apps/console/src/app/(console)/audit/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/audit/read-resilience.invariants.test.ts new file mode 100644 index 000000000..a712465a5 --- /dev/null +++ b/apps/console/src/app/(console)/audit/read-resilience.invariants.test.ts @@ -0,0 +1,88 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the audit segment, mirroring + * `sources/read-resilience.invariants.test.ts`. See that file and + * `syncs/read-resilience.invariants.test.ts` for the full standard this + * pattern enforces; this file pins the same properties for `/audit`. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g; + +/** + * Strip `/* ... *‍/` block comments before checking for retired owner-facing + * copy. The boundary's doc comment legitimately QUOTES the retired phrases + * (to explain what this pattern replaces and why) — that is documentation, + * not rendered JSX text, so it must not trip the ban. + */ +function withoutBlockComments(src: string): string { + return src.replace(BLOCK_COMMENT_RE, ""); +} + +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_ERROR_HEADING_RE = /Read error/; +const RETIRED_TRY_AGAIN_RE = /Try again/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i; +const RETIRED_BACK_LINK_RE = /Back to audit/; +const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/; + +const USES_LOADING_SKELETON_RE = / { + const rawSrc = await readFile(ERROR_FILE, "utf8"); + const src = withoutBlockComments(rawSrc); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE); + assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BACK_LINK_RE); + assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE); +}); + +test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + // loading.tsx uses ListLoadingSkeleton label="audit events" rows={8}. + assert.match(src, /ListLoadingSkeleton label="audit events" rows=\{8\}/); +}); + +test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, CALLS_RESET_RE); + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.doesNotMatch( + src, + NO_REACT_STATE_COUNTER_RE, + "the retry counter must live at module scope, not React state, or backoff never grows across remounts" + ); + assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded"); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/components/read-resilient-retry.test.ts b/apps/console/src/app/(console)/components/read-resilient-retry.test.ts new file mode 100644 index 000000000..8e715c0d4 --- /dev/null +++ b/apps/console/src/app/(console)/components/read-resilient-retry.test.ts @@ -0,0 +1,41 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Behavioral tests for the shared read-resilient-boundary backoff primitive. + * See `read-resilient-retry.ts` for why the retry counter this module + * supports must be created once per segment boundary at module scope. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { + createRetryCounter, + nextRetryDelayMs, + RETRY_BASE_DELAY_MS, + RETRY_MAX_DELAY_MS, +} from "./read-resilient-retry.ts"; + +test("nextRetryDelayMs starts at the base delay for the first attempt", () => { + assert.equal(nextRetryDelayMs(0), RETRY_BASE_DELAY_MS); +}); + +test("nextRetryDelayMs doubles per attempt until the cap", () => { + assert.equal(nextRetryDelayMs(1), RETRY_BASE_DELAY_MS * 2); + assert.equal(nextRetryDelayMs(2), RETRY_BASE_DELAY_MS * 4); + assert.equal(nextRetryDelayMs(3), RETRY_BASE_DELAY_MS * 8); +}); + +test("nextRetryDelayMs is capped at RETRY_MAX_DELAY_MS and never exceeds it, however large the attempt", () => { + const atCap = nextRetryDelayMs(10); + const wayPastCap = nextRetryDelayMs(1000); + assert.equal(atCap, RETRY_MAX_DELAY_MS); + assert.equal(wayPastCap, RETRY_MAX_DELAY_MS); +}); + +test("createRetryCounter returns an independent counter each call — no shared state between boundaries", () => { + const a = createRetryCounter(); + const b = createRetryCounter(); + a.attempts = 5; + assert.equal(b.attempts, 0, "mutating one boundary's counter must not affect another's"); +}); diff --git a/apps/console/src/app/(console)/components/read-resilient-retry.ts b/apps/console/src/app/(console)/components/read-resilient-retry.ts new file mode 100644 index 000000000..7cf7877a4 --- /dev/null +++ b/apps/console/src/app/(console)/components/read-resilient-retry.ts @@ -0,0 +1,52 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared unbounded-capped-backoff retry primitive for segment error + * boundaries (App Router `error.tsx` convention). + * + * SLVP bar (Stripe, Linear, Vercel, Plaid): none of those products tell a + * user "couldn't load X" with a Try again button because a stream hiccuped. + * When a segment's server read throws `Error: The destination stream closed + * early` — React's RSC streaming writer reacting to the HTTP response closing + * while the Flight stream was still being written, not a failed data read — + * the boundary must retry quietly and indefinitely rather than parking on a + * failure card. See `sources/error.tsx` for the fully-annotated original of + * this pattern; this module factors ONLY the backoff arithmetic, which is + * byte-identical across every segment that adopts it. Each segment's own + * skeleton and copy stay in that segment's `error.tsx` — factoring those out + * too would hide the per-route visual contract behind a generic wrapper. + * + * The retry counter MUST live at module scope, never in React state: Next.js + * remounts the error boundary fresh on every catch (a new error instance + * re-enters the boundary), so a `useState` counter would silently reset to 0 + * on every failure and the backoff would never grow past its base delay. A + * `RetryCounter` created by `createRetryCounter()` at each `error.tsx` + * module's top level (NOT inside this shared module, and NOT shared between + * routes) gives each segment boundary its own independent counter with + * exactly the right lifetime: "how many times has this boundary caught in a + * row since the page was last freshly loaded." + */ + +/** First retry is near-immediate — long enough to dodge a tight synchronous loop. */ +export const RETRY_BASE_DELAY_MS = 300; +/** Backoff ceiling: keep retrying at a calm, bounded cadence forever rather than escalating without limit. */ +export const RETRY_MAX_DELAY_MS = 15_000; + +/** + * A single segment boundary's consecutive-failure counter. Create exactly one + * of these per `error.tsx` module (at module scope, not inside the component) + * and reuse it across every catch that module handles. + */ +export type RetryCounter = { attempts: number }; + +/** Create a fresh, independent module-scoped retry counter for one segment boundary. */ +export function createRetryCounter(): RetryCounter { + return { attempts: 0 }; +} + +/** Capped exponential backoff. Never returns a delay the owner would perceive as "given up". */ +export function nextRetryDelayMs(attempt: number): number { + const scaled = RETRY_BASE_DELAY_MS * 2 ** attempt; + return Math.min(scaled, RETRY_MAX_DELAY_MS); +} diff --git a/apps/console/src/app/(console)/components/source-setup-catalog.tsx b/apps/console/src/app/(console)/components/source-setup-catalog.tsx index 27d6571ec..24bd9afef 100644 --- a/apps/console/src/app/(console)/components/source-setup-catalog.tsx +++ b/apps/console/src/app/(console)/components/source-setup-catalog.tsx @@ -117,7 +117,26 @@ function SourceAcquisitionPaths({ paths }: { paths: readonly ConnectorAcquisitio ); } +/** + * Development-tier method line, resolved before the `not_available_here` + * short-circuit below. `sourceSetupAvailability` reports `not_available_here` + * for every Development entry regardless of whether it renders an add + * action (see `sourceSetupAction`'s scaffold check), so that check alone + * would print "No proven setup path is available" directly above a real + * "Add account" button for a self-testable entry -- contradicting the + * action right next to it. + */ +function developmentMethodLine(entry: ConnectorCatalogEntry): string { + if (entry.isKnownScaffold) { + return "Not implemented yet: this connector cannot collect data."; + } + return "Setup path implemented, not yet proven against a live account."; +} + function sourceMethodLine(entry: ConnectorCatalogEntry, existingSourceCount: number): string { + if (entry.publicTier === "development") { + return developmentMethodLine(entry); + } if (sourceSetupAvailability(entry) === "not_available_here") { return "No proven setup path is available in this dashboard."; } @@ -352,6 +371,41 @@ function ExperimentalSetupSummary({ ); } +/** + * Development-tier connectors are registered and shipped, but this dashboard + * does not offer them in the main list or the Preview disclosure above -- + * either no live run has proven the setup path yet (real, self-testable), or + * the connector is a scaffold with no real collection code yet (never gets + * an add action). Collapsed by default, same precedent as Preview, one tier + * more cautious: an owner running their own instance can see everything that + * exists and tell "not proven yet" apart from "not built yet", instead of a + * connector silently vanishing between "shipped" and "visible". + */ +function DevelopmentSetupSummary({ + entries, + existingSourcesByConnector, +}: { + entries: readonly ConnectorCatalogEntry[]; + existingSourcesByConnector?: Readonly>; +}) { + if (entries.length === 0) { + return null; + } + return ( +
+ Development ({entries.length}) +
+

+ These connectors are registered on this instance but not yet offered above. Some have a real, implemented + setup path with no live-account run yet -- test them with non-critical data. Others are scaffolds with no + collection code yet and have no add action here. +

+ +
+
+ ); +} + function SourceSetupCardList({ entries, existingSourcesByConnector, @@ -393,7 +447,16 @@ export function SourceSetupCatalog({ const filtered = filterSourceCatalog(catalog, query); const available = filtered.filter((entry) => entry.publicTier === "supported" && isRunnableAddOffer(entry)); const experimental = filtered.filter((entry) => entry.publicTier === "preview" && isRunnableAddOffer(entry)); + // Every Development-tier entry belongs here -- real-but-unproven and known + // scaffolds alike. Visibility is the point: an owner running this instance + // must be able to tell a scaffold apart from a connector nobody has tested + // yet, not have either one silently omitted. `SourceSetupCard` itself + // already withholds the add action for a scaffold (`sourceSetupAction` + // returns null), so listing every Development entry here cannot render a + // dead-end "Add" button. + const development = filtered.filter((entry) => entry.publicTier === "development"); const actionable = [...available, ...experimental]; + const anyMatch = actionable.length > 0 || development.length > 0; return (
@@ -405,7 +468,7 @@ export function SourceSetupCatalog({ Search
- {actionable.length > 0 ? ( + {anyMatch ? (
{available.length > 0 ? ( @@ -416,6 +479,7 @@ export function SourceSetupCatalog({ )} +
) : (

diff --git a/apps/console/src/app/(console)/components/views/standing-view-model.test.ts b/apps/console/src/app/(console)/components/views/standing-view-model.test.ts index 68e33a71e..91088d241 100644 --- a/apps/console/src/app/(console)/components/views/standing-view-model.test.ts +++ b/apps/console/src/app/(console)/components/views/standing-view-model.test.ts @@ -379,7 +379,7 @@ test("attention truth: only attention-channel connections with an owner-satisfia { affects: [], audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -440,7 +440,7 @@ test("source issues show non-owner material verdicts without alarming as owner a { affects: [], audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -488,7 +488,7 @@ test("advisory owner actions surface non-urgent Amazon retry work without calm a rendered_verdict: verdict({ channel: "advisory", forward_statement: "Some order detail is still outstanding. Retry this source to collect the missing detail.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: ["orders"], @@ -611,7 +611,7 @@ test("source actionability groups live-shaped rows with scoped counts", () => { rendered_verdict: verdict({ channel: "advisory", forward_statement: "Run a refresh to bring this up to date.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: [], @@ -633,7 +633,7 @@ test("source actionability groups live-shaped rows with scoped counts", () => { rendered_verdict: verdict({ channel: "advisory", forward_statement: "Latest collection completed with known coverage gaps.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: [], @@ -700,7 +700,7 @@ function deferredRecoveryVerdict(): RefConnectorSummary["rendered_verdict"] { return verdict({ channel: "calm", forward_statement: "Catching up on the remaining data.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: [], @@ -775,7 +775,7 @@ test("dashboard cross-surface: every source-work section count equals its render rendered_verdict: verdict({ channel: "advisory", forward_statement: "Run a refresh to bring this up to date.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: [], @@ -796,13 +796,13 @@ test("dashboard cross-surface: every source-work section count equals its render source_work: "system_issue", rendered_verdict: verdict({ channel: "advisory", - forward_statement: "Connector code needs a fix before this can collect again.", + forward_statement: "Some data from this source can't be collected.", pill: { label: "Can't collect", tone: "red" }, required_actions: [ { affects: [], audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -938,7 +938,7 @@ test("reviewable degraded source appears once rather than as review plus source rendered_verdict: verdict({ channel: "advisory", forward_statement: "Retry now to give the recoverable gap another run.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: [], @@ -973,13 +973,13 @@ test("source actionability follows primary-action parity with push policy", () = source_work: "system_issue", rendered_verdict: verdict({ channel: "attention", - forward_statement: "Connector code needs a fix before this can collect again.", + forward_statement: "Some data from this source can't be collected.", pill: { label: "Can't collect", tone: "red" }, required_actions: [ { affects: [], audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -1016,12 +1016,12 @@ test("maintainer-only actions are not advisory owner actions", () => { rendered_verdict: verdict({ channel: "advisory", forward_statement: "This source needs a connector code fix before it can make progress.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: [], audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -1086,7 +1086,7 @@ test("source issues surface attention verdicts that have no owner action, even w { affects: [], audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, diff --git a/apps/console/src/app/(console)/connect/browser-session-login-honesty.invariants.test.ts b/apps/console/src/app/(console)/connect/browser-session-login-honesty.invariants.test.ts new file mode 100644 index 000000000..10c63a299 --- /dev/null +++ b/apps/console/src/app/(console)/connect/browser-session-login-honesty.invariants.test.ts @@ -0,0 +1,96 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * A browser_session connection reaches `first_sync_running` / + * `first_sync_pending` from run evidence ALONE — see `hasDraftSetupProgress` + * in `reference-implementation/runtime/static-secret-setup-status.ts`, which + * returns true for `setupKind === "browser_session"` whenever an active or + * last run row exists. For this setup kind the run IS the login attempt: it + * starts so the owner can sign in inside the streamed browser. + * + * So a run row proves a sign-in was ATTEMPTED, never that it completed. + * `defaultSetupMaterial` pins `present: false` for browser sessions precisely + * because nothing was captured. The console previously rendered "Login is + * complete" from that evidence, telling the owner a session was live while the + * stream still sat on the provider's sign-in form (owner-reported 2026-08-19, + * Reddit). + * + * These are source-text invariants, matching the convention of the sibling + * `*.invariants.test.ts` files: this page is a React server component with no + * DOM harness in this suite, so the rendered copy is asserted against the + * source. See design-notes/browser-stream-status-honesty-2026-08-22.md. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const STATUS_PAGE_FILE = fileURLToPath(new URL("./status/[connectionId]/page.tsx", import.meta.url)); + +// Hoisted to satisfy useTopLevelRegex. +const BROWSER_SESSION_FN = /function describeBrowserSessionState\(/; +const STATIC_SECRET_FN = /function describeConnectionState\(/; +// The claim that must never return: any assertion that the login itself +// succeeded, in any of the phrasings that read as a completed sign-in. +const LOGIN_COMPLETE_CLAIM = /Login is complete|You(?:'re| are) (?:signed|logged) in|Already (?:signed|logged) in/i; +// The honest replacement must point the owner back at the browser, because +// this page genuinely cannot tell whether the sign-in finished. +const FINISH_SIGNING_IN_CUE = /finish signing in there/; +const CANNOT_CONFIRM_CUE = /can't confirm the login by itself/; +// The static-secret claim that must be PRESERVED — it is evidence-backed. +const CREDENTIAL_CAPTURED_CLAIM = /The provider credential is captured/; +// Function-boundary marker used to slice one `describe*State` body. +const NEXT_TOP_LEVEL_FUNCTION = /\nfunction /; + +function readStatusPage(): Promise { + return readFile(STATUS_PAGE_FILE, "utf8"); +} + +/** + * Slices out one `describe*State` function body so an assertion about the + * browser-session branch cannot be satisfied by copy that lives in the + * static-secret or manual-upload branch of the same file. + */ +function sliceFunctionBody(src: string, startPattern: RegExp): string { + const start = src.search(startPattern); + assert.notEqual(start, -1, `expected to find ${String(startPattern)} in the status page`); + const nextFn = src.slice(start + 1).search(NEXT_TOP_LEVEL_FUNCTION); + return nextFn === -1 ? src.slice(start) : src.slice(start, start + 1 + nextFn); +} + +test("the browser-session branch never claims the login completed", async () => { + const src = await readStatusPage(); + const body = sliceFunctionBody(src, BROWSER_SESSION_FN); + + assert.doesNotMatch( + body, + LOGIN_COMPLETE_CLAIM, + "a browser_session run row proves only that a sign-in was attempted — the console has no evidence the login succeeded, so it must not assert that it did" + ); +}); + +test("the browser-session in-flight copy sends the owner back to the browser instead", async () => { + const src = await readStatusPage(); + const body = sliceFunctionBody(src, BROWSER_SESSION_FN); + + assert.match(body, FINISH_SIGNING_IN_CUE, "the owner must be told where to finish an unfinished sign-in"); + assert.match(body, CANNOT_CONFIRM_CUE, "the page must admit it cannot confirm the login itself"); +}); + +test("the static-secret branch keeps its capture claim, which IS backed by setup material", async () => { + const src = await readStatusPage(); + const body = sliceFunctionBody(src, STATIC_SECRET_FN); + + // Guards against over-correcting: unlike browser_session, a static_secret + // connection reaches these states via `hasSetupMaterial` + // (`credential.present === true`), so naming the captured credential is a + // claim the projection actually supports. Scrubbing it would trade one + // dishonesty for a needless loss of information. + assert.match( + body, + CREDENTIAL_CAPTURED_CLAIM, + "static-secret copy cites setup_material.present === true and must be preserved" + ); +}); diff --git a/apps/console/src/app/(console)/connect/status/[connectionId]/page.tsx b/apps/console/src/app/(console)/connect/status/[connectionId]/page.tsx index 25f4bb52a..5242e65c2 100644 --- a/apps/console/src/app/(console)/connect/status/[connectionId]/page.tsx +++ b/apps/console/src/app/(console)/connect/status/[connectionId]/page.tsx @@ -237,9 +237,24 @@ function describeConnectionState(status: ConnectionSetupStatus): StatusDescripti // Browser/SSO connections (ChatGPT and every other browser-bound connector) // have no stored credential at all — copy here must never say "credential" or -// imply a secret was expected. Progress (an active/last run) is real evidence -// even before any material is captured; see `deriveSetupState`'s -// browser_session handling in the RI runtime projection. +// imply a secret was expected. +// +// It must also never claim the login SUCCEEDED. `deriveSetupState` reaches +// `first_sync_running`/`first_sync_pending` for a browser session from +// `hasRunEvidence` alone — the mere existence of an active or last run row +// (`static-secret-setup-status.ts` `hasDraftSetupProgress`). For a +// browser_session connection the run IS the login attempt: it starts so the +// owner can sign in inside the streamed browser. So a run row proves a sign-in +// was ATTEMPTED, never that it completed — and `defaultSetupMaterial` pins +// `present: false` for this kind precisely because no material was captured. +// +// Saying "Login is complete" here told the owner a session was live while the +// stream was still sitting on Reddit's sign-in form. Unlike the static-secret +// and manual-upload branches — whose "credential is captured" / "file is +// captured" claims ARE backed by `setup_material.present === true` — this +// branch has no proof to cite, so it describes only what is observed: a sync +// is running. Owner-reported 2026-08-19; see +// design-notes/browser-stream-status-honesty-2026-08-22.md. function describeBrowserSessionState(status: ConnectionSetupStatus): StatusDescription { const terminalDisposition = describeTerminalSetupDisposition(status); if (terminalDisposition) { @@ -250,13 +265,15 @@ function describeBrowserSessionState(status: ConnectionSetupStatus): StatusDescr return describeActiveConnectionState(status); case "first_sync_running": return { - detail: "Login is complete and the first sync is running. It will continue automatically.", + detail: + "A first sync is running. If the browser is still showing a sign-in page, finish signing in there — this page can't confirm the login by itself.", headline: "First sync running", tone: "pending", }; case "first_sync_pending": return { - detail: "Login is complete and the first sync is queued. It will start automatically.", + detail: + "A first sync is queued and will start automatically. If the browser is still showing a sign-in page, finish signing in there — this page can't confirm the login by itself.", headline: "First sync starting", tone: "pending", }; diff --git a/apps/console/src/app/(console)/deployment/error.tsx b/apps/console/src/app/(console)/deployment/error.tsx index 1533c17b4..bccbaeaeb 100644 --- a/apps/console/src/app/(console)/deployment/error.tsx +++ b/apps/console/src/app/(console)/deployment/error.tsx @@ -3,16 +3,61 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { SegmentError } from "../components/segment-error.tsx"; +import { useEffect } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts"; +import { DetailLoadingSkeleton } from "../components/route-loading.tsx"; + +/** + * Deployment-segment error boundary (App Router convention) — SLVP bar: + * Stripe, Linear, Vercel, and Plaid never tell an owner "we hit a transient + * read interruption, retrying." The page renders, or it quietly shows + * last-known state. The owner never learns the backend hiccuped. + * + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`): the read itself is fine — React's Flight/RSC + * streaming writer reacting to the HTTP response closing before the stream + * finished flushing. It is a client-transport race below the data layer, not + * a backend outage — see `sources/error.tsx` for the full original writeup. + * + * `/deployment` has no client-cached last-known-read marker, so this + * boundary shows the plain skeleton with no staleness caption rather than + * fabricate a timestamp. It reuses `DetailLoadingSkeleton`, matching + * `deployment/loading.tsx` (a detail surface, not a list). + * + * Self-contained on purpose: a `"use client"` boundary must not import + * server-only modules, since the dashboard shell transitively pulls in + * `lib/owner-token.ts` (`server-only`). + */ + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. + */ +const retryCounter = createRetryCounter(); + +export default function DeploymentError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. + console.error(error); + }, [error]); + + useEffect(() => { + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // counter. There is deliberately no ceiling on the counter itself — a + // persistent failure degrades to a slow quiet heartbeat, never a dead end. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + reset(); + }, delay); + return () => clearTimeout(id); + }, [reset]); -export default function DeploymentError(props: { error: Error & { digest?: string }; reset: () => void }) { return ( - +

+ +
); } diff --git a/apps/console/src/app/(console)/deployment/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/deployment/read-resilience.invariants.test.ts new file mode 100644 index 000000000..904531646 --- /dev/null +++ b/apps/console/src/app/(console)/deployment/read-resilience.invariants.test.ts @@ -0,0 +1,95 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the deployment segment, + * mirroring `sources/read-resilience.invariants.test.ts`. See that file and + * `syncs/read-resilience.invariants.test.ts` for the full standard this + * pattern enforces; this file pins the same properties for `/deployment`. + * + * `/deployment` renders a DETAIL surface (one status view), not a list — + * `deployment/loading.tsx` uses `DetailLoadingSkeleton`, not + * `ListLoadingSkeleton`, so this boundary must reuse that skeleton + * specifically to stay visually identical to the route's own loading state. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g; + +/** + * Strip `/* ... *‍/` block comments before checking for retired owner-facing + * copy. The boundary's doc comment legitimately QUOTES the retired phrases + * (to explain what this pattern replaces and why) — that is documentation, + * not rendered JSX text, so it must not trip the ban. + */ +function withoutBlockComments(src: string): string { + return src.replace(BLOCK_COMMENT_RE, ""); +} + +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_ERROR_HEADING_RE = /Read error/; +const RETIRED_TRY_AGAIN_RE = /Try again/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i; +const RETIRED_BACK_LINK_RE = /Back to deployment/; +const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/; + +const USES_DETAIL_SKELETON_RE = / { + const rawSrc = await readFile(ERROR_FILE, "utf8"); + const src = withoutBlockComments(rawSrc); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE); + assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BACK_LINK_RE); + assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE); +}); + +test("the boundary renders the DetailLoadingSkeleton (matching loading.tsx), not the list skeleton or a bespoke banner", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_DETAIL_SKELETON_RE); + assert.doesNotMatch(src, DOES_NOT_USE_LIST_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + // loading.tsx uses DetailLoadingSkeleton label="deployment status". + assert.match(src, /DetailLoadingSkeleton label="deployment status"/); +}); + +test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, CALLS_RESET_RE); + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.doesNotMatch( + src, + NO_REACT_STATE_COUNTER_RE, + "the retry counter must live at module scope, not React state, or backoff never grows across remounts" + ); + assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded"); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts b/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts index 513fb968b..f19f6fdd0 100644 --- a/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts +++ b/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts @@ -19,6 +19,7 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; +import { SUPPORTED_LOCAL_COLLECTOR_CONNECTORS } from "pdpp-reference-implementation/connection-setup-plan"; const ROOT = new URL("../../../../../../", import.meta.url); @@ -37,8 +38,8 @@ const BROWSER_COLLECTOR_MONOREPO_COPY = /PDPP monorepo checkout|pnpm --dir|packages\/polyfill-connectors|browser-collector run command/; const ENROLL_TESTID = /data-testid="collector-enroll-command"/; const RUN_TESTID_CLAUDE = /data-testid={`collector-run-command-/; -const SUPPORTED_CONNECTORS = - /COLLECTOR_RUN_CONNECTORS\s*=\s*\[\s*"claude_code",\s*"codex",\s*"google_takeout",\s*"imessage",\s*"apple_photos",\s*"google_messages",?\s*\]/; +const COLLECTOR_RUN_CONNECTORS_LITERAL_RE = /COLLECTOR_RUN_CONNECTORS\s*=\s*\[([^\]]*)\]/; +const SURROUNDING_QUOTES_RE = /^["']|["']$/g; test("enrollment form derives the canonical local collector commands via shared helpers", async () => { const src = await read(FORM_PATH); @@ -80,11 +81,29 @@ test("enrollment form never renders a local-collector setup invocation", async ( }); test("enrollment form advertises every connector bundled in the published @pdpp/local-collector npx path", async () => { + // Derived, not hardcoded. `LOCAL_COLLECTOR_DEFINITIONS` in + // packages/polyfill-connectors/src/collector-registry.ts is the single source + // of truth for what the published npx bundle actually ships, so the form's + // advertised list is checked against that registry rather than a literal + // roster this test would have to be edited to keep true. A pinned literal + // silently goes stale the moment a connector is bundled (that is exactly how + // `signal` broke this test), and the failure then looks like the FORM is + // wrong when the bundle grew correctly. const src = await read(FORM_PATH); - assert.match( - src, - SUPPORTED_CONNECTORS, - "claude_code, codex, google_takeout, imessage, apple_photos, and google_messages are all bundled in the published @pdpp/local-collector npx path" + const match = src.match(COLLECTOR_RUN_CONNECTORS_LITERAL_RE); + assert.ok(match, "enrollment form must declare COLLECTOR_RUN_CONNECTORS"); + const advertised = (match[1] ?? "") + .split(",") + .map((entry) => entry.trim().replace(SURROUNDING_QUOTES_RE, "")) + .filter(Boolean); + // `SUPPORTED_LOCAL_COLLECTOR_CONNECTORS` is generated from + // `LOCAL_COLLECTOR_DEFINITIONS` (see connection-setup-plan.ts) and is already + // in the enrollment-key (underscore) form the form's literal carries and that + // gets passed to `--connector`, so this compares like for like. + assert.deepEqual( + advertised, + [...SUPPORTED_LOCAL_COLLECTOR_CONNECTORS], + "the form must advertise exactly the connectors bundled in the published @pdpp/local-collector npx path, in bundle order" ); }); diff --git a/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx b/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx index da12d8ac3..3c4a0c674 100644 --- a/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx +++ b/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx @@ -17,9 +17,10 @@ const COLLECTOR_RUN_CONNECTORS = [ "imessage", "apple_photos", "google_messages", + "signal", ] as const; const MACOS_ONLY_LOCAL_COLLECTOR_CONNECTORS = ["imessage", "apple_photos"] as const; -const EXTERNAL_TOOL_LOCAL_COLLECTOR_CONNECTORS = ["google_messages"] as const; +const EXTERNAL_TOOL_LOCAL_COLLECTOR_CONNECTORS = ["google_messages", "signal"] as const; function isMacosOnlyLocalCollectorConnector(connectorId: string): boolean { return (MACOS_ONLY_LOCAL_COLLECTOR_CONNECTORS as readonly string[]).includes(connectorId); diff --git a/apps/console/src/app/(console)/device-exporters/error.tsx b/apps/console/src/app/(console)/device-exporters/error.tsx index feac34bec..c4a41056d 100644 --- a/apps/console/src/app/(console)/device-exporters/error.tsx +++ b/apps/console/src/app/(console)/device-exporters/error.tsx @@ -3,16 +3,66 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { SegmentError } from "../components/segment-error.tsx"; +import { useEffect } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts"; +import { ListLoadingSkeleton } from "../components/route-loading.tsx"; + +/** + * Device-exporters-segment error boundary (App Router convention) — SLVP + * bar: Stripe, Linear, Vercel, and Plaid never tell an owner "we hit a + * transient read interruption, retrying." The page renders, or it quietly + * shows last-known state. The owner never learns the backend hiccuped. + * + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`): the read itself is fine — React's Flight/RSC + * streaming writer reacting to the HTTP response closing before the stream + * finished flushing. It is a client-transport race below the data layer, not + * a backend outage — see `sources/error.tsx` for the full original writeup. + * + * `/device-exporters` has no client-cached last-known-read marker, so this + * boundary shows the plain skeleton with no staleness caption rather than + * fabricate a timestamp. + * + * Self-contained on purpose: a `"use client"` boundary must not import + * server-only modules, since the dashboard shell transitively pulls in + * `lib/owner-token.ts` (`server-only`). + */ + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. + */ +const retryCounter = createRetryCounter(); + +export default function DeviceExportersError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. + console.error(error); + }, [error]); + + useEffect(() => { + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // counter. There is deliberately no ceiling on the counter itself — a + // persistent failure degrades to a slow quiet heartbeat, never a dead end. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + reset(); + }, delay); + return () => clearTimeout(id); + }, [reset]); -export default function DeviceExportersError(props: { error: Error & { digest?: string }; reset: () => void }) { return ( - +
+ +
); } diff --git a/apps/console/src/app/(console)/device-exporters/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/device-exporters/read-resilience.invariants.test.ts new file mode 100644 index 000000000..973e3ff5f --- /dev/null +++ b/apps/console/src/app/(console)/device-exporters/read-resilience.invariants.test.ts @@ -0,0 +1,89 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the device-exporters segment, + * mirroring `sources/read-resilience.invariants.test.ts`. See that file and + * `syncs/read-resilience.invariants.test.ts` for the full standard this + * pattern enforces; this file pins the same properties for + * `/device-exporters`. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g; + +/** + * Strip `/* ... *‍/` block comments before checking for retired owner-facing + * copy. The boundary's doc comment legitimately QUOTES the retired phrases + * (to explain what this pattern replaces and why) — that is documentation, + * not rendered JSX text, so it must not trip the ban. + */ +function withoutBlockComments(src: string): string { + return src.replace(BLOCK_COMMENT_RE, ""); +} + +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_ERROR_HEADING_RE = /Read error/; +const RETIRED_TRY_AGAIN_RE = /Try again/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i; +const RETIRED_BACK_LINK_RE = /Back to device exporters/; +const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/; + +const USES_LOADING_SKELETON_RE = / { + const rawSrc = await readFile(ERROR_FILE, "utf8"); + const src = withoutBlockComments(rawSrc); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE); + assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BACK_LINK_RE); + assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE); +}); + +test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + // loading.tsx uses ListLoadingSkeleton label="device exporters" rows={5}. + assert.match(src, /ListLoadingSkeleton label="device exporters" rows=\{5\}/); +}); + +test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, CALLS_RESET_RE); + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.doesNotMatch( + src, + NO_REACT_STATE_COUNTER_RE, + "the retry counter must live at module scope, not React state, or backoff never grows across remounts" + ); + assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded"); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/error.tsx b/apps/console/src/app/(console)/error.tsx index 3b64340fd..4823a100f 100644 --- a/apps/console/src/app/(console)/error.tsx +++ b/apps/console/src/app/(console)/error.tsx @@ -4,23 +4,91 @@ // SPDX-License-Identifier: Apache-2.0 import { buttonVariants } from "@pdpp/brand-react"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "./components/read-resilient-retry.ts"; +import { ListLoadingSkeleton } from "./components/route-loading.tsx"; /** - * Dashboard error boundary (App Router convention). + * Dashboard ROOT error boundary (App Router convention) — the catch-all for + * anything not caught by a more specific segment boundary (`sources/error.tsx`, + * `syncs/error.tsx`, etc.). + * + * DELIBERATELY DIFFERENT from the leaf-segment boundaries: those all now + * retry unbounded, because their specific throw (`Error: The destination + * stream closed early`) is a known, provenance-checked transport race — the + * underlying read already succeeded, only the RSC stream teardown raced. This + * boundary sits above `page.tsx` (the dashboard overview), which already + * fault-isolates every one of its OWN data reads via `safeRead()` — an + * individual source failing degrades that section to empty inline, it never + * throws up to here. So an error that DOES reach this root boundary is either + * (a) the same stream-teardown race, now unprovable-by-route because this + * boundary is shared by the whole segment, or (b) a genuine unhandled fault + * in render/layout code — precisely the class of bug `safeRead()` was built + * NOT to swallow. There is no error-reporting integration in this codebase + * (no Sentry/equivalent) — `console.error` here is the only diagnostic + * signal an operator has. Retrying an unprovable root-level fault forever, + * silently, would delete that signal for a real crash. + * + * So this boundary retries quietly (same skeleton, no failure copy, capped + * backoff) for a BOUNDED number of attempts — enough to absorb the ordinary + * transient race — and only after that repeatedly fails does it fall back to + * the pre-existing "Something went wrong" / Try again / Sign in again panel. + * That is strictly no worse than the boundary's prior behavior (which showed + * that panel immediately, every time) and materially better for the common + * case: a lone stream hiccup anywhere in the dashboard no longer flashes + * failure copy at the owner. * * Self-contained on purpose: it must not import server-only modules. The * dashboard shell (`RecordroomShellWithPalette`) transitively pulls in * `lib/owner-token.ts`, which is `server-only`; importing it here would break - * the client build. - * Stripe/Linear-style empty state lives below; the user can retry or sign - * back in. See https://nextjs.org/docs/app/getting-started/error-handling. + * the client build. See https://nextjs.org/docs/app/getting-started/error-handling. + */ + +/** Bounded: absorb a handful of quiet retries before conceding this may be a real fault. */ +const MAX_QUIET_ATTEMPTS = 5; + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. */ +const retryCounter = createRetryCounter(); + export default function DashboardError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + const [gaveUp, setGaveUp] = useState(() => retryCounter.attempts >= MAX_QUIET_ATTEMPTS); + useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. console.error(error); }, [error]); + useEffect(() => { + if (gaveUp) { + return; + } + // Bounded, capped backoff: retry quietly like the leaf segment boundaries, + // but stop scheduling further attempts once MAX_QUIET_ATTEMPTS is reached + // so a genuine, persistent fault surfaces instead of looping forever. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + if (retryCounter.attempts >= MAX_QUIET_ATTEMPTS) { + setGaveUp(true); + return; + } + reset(); + }, delay); + return () => clearTimeout(id); + }, [gaveUp, reset]); + + if (!gaveUp) { + return ( +
+ +
+ ); + } + return (

PDPP

@@ -30,7 +98,15 @@ export default function DashboardError({ error, reset }: { error: Error & { dige sign back in if the problem persists.

- diff --git a/apps/console/src/app/(console)/event-subscriptions/error.tsx b/apps/console/src/app/(console)/event-subscriptions/error.tsx index 6dc4005d7..c9315be15 100644 --- a/apps/console/src/app/(console)/event-subscriptions/error.tsx +++ b/apps/console/src/app/(console)/event-subscriptions/error.tsx @@ -3,16 +3,66 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { SegmentError } from "../components/segment-error.tsx"; +import { useEffect } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts"; +import { ListLoadingSkeleton } from "../components/route-loading.tsx"; + +/** + * Event-subscriptions-segment error boundary (App Router convention) — SLVP + * bar: Stripe, Linear, Vercel, and Plaid never tell an owner "we hit a + * transient read interruption, retrying." The page renders, or it quietly + * shows last-known state. The owner never learns the backend hiccuped. + * + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`): the read itself is fine — React's Flight/RSC + * streaming writer reacting to the HTTP response closing before the stream + * finished flushing. It is a client-transport race below the data layer, not + * a backend outage — see `sources/error.tsx` for the full original writeup. + * + * `/event-subscriptions` has no client-cached last-known-read marker, so this + * boundary shows the plain skeleton with no staleness caption rather than + * fabricate a timestamp. + * + * Self-contained on purpose: a `"use client"` boundary must not import + * server-only modules, since the dashboard shell transitively pulls in + * `lib/owner-token.ts` (`server-only`). + */ + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. + */ +const retryCounter = createRetryCounter(); + +export default function EventSubscriptionsError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. + console.error(error); + }, [error]); + + useEffect(() => { + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // counter. There is deliberately no ceiling on the counter itself — a + // persistent failure degrades to a slow quiet heartbeat, never a dead end. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + reset(); + }, delay); + return () => clearTimeout(id); + }, [reset]); -export default function EventSubscriptionsError(props: { error: Error & { digest?: string }; reset: () => void }) { return ( - +
+ +
); } diff --git a/apps/console/src/app/(console)/event-subscriptions/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/event-subscriptions/read-resilience.invariants.test.ts new file mode 100644 index 000000000..1f763e46c --- /dev/null +++ b/apps/console/src/app/(console)/event-subscriptions/read-resilience.invariants.test.ts @@ -0,0 +1,89 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the event-subscriptions segment, + * mirroring `sources/read-resilience.invariants.test.ts`. See that file and + * `syncs/read-resilience.invariants.test.ts` for the full standard this + * pattern enforces; this file pins the same properties for + * `/event-subscriptions`. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g; + +/** + * Strip `/* ... *‍/` block comments before checking for retired owner-facing + * copy. The boundary's doc comment legitimately QUOTES the retired phrases + * (to explain what this pattern replaces and why) — that is documentation, + * not rendered JSX text, so it must not trip the ban. + */ +function withoutBlockComments(src: string): string { + return src.replace(BLOCK_COMMENT_RE, ""); +} + +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_ERROR_HEADING_RE = /Read error/; +const RETIRED_TRY_AGAIN_RE = /Try again/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i; +const RETIRED_BACK_LINK_RE = /Back to event subscriptions/; +const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/; + +const USES_LOADING_SKELETON_RE = / { + const rawSrc = await readFile(ERROR_FILE, "utf8"); + const src = withoutBlockComments(rawSrc); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE); + assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BACK_LINK_RE); + assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE); +}); + +test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + // loading.tsx uses ListLoadingSkeleton label="event subscriptions" rows={6}. + assert.match(src, /ListLoadingSkeleton label="event subscriptions" rows=\{6\}/); +}); + +test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, CALLS_RESET_RE); + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.doesNotMatch( + src, + NO_REACT_STATE_COUNTER_RE, + "the retry counter must live at module scope, not React state, or backoff never grows across remounts" + ); + assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded"); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/grants/error.tsx b/apps/console/src/app/(console)/grants/error.tsx index 2276ef2ce..c226f2df5 100644 --- a/apps/console/src/app/(console)/grants/error.tsx +++ b/apps/console/src/app/(console)/grants/error.tsx @@ -3,16 +3,60 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { SegmentError } from "../components/segment-error.tsx"; +import { useEffect } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts"; +import { ListLoadingSkeleton } from "../components/route-loading.tsx"; + +/** + * Grants-segment error boundary (App Router convention) — SLVP bar: Stripe, + * Linear, Vercel, and Plaid never tell an owner "we hit a transient read + * interruption, retrying." The page renders, or it quietly shows last-known + * state. The owner never learns the backend hiccuped. + * + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`): the read itself is fine — React's Flight/RSC + * streaming writer reacting to the HTTP response closing before the stream + * finished flushing. It is a client-transport race below the data layer, not + * a backend outage — see `sources/error.tsx` for the full original writeup. + * + * `/grants` has no client-cached last-known-read marker, so this boundary + * shows the plain skeleton with no staleness caption rather than fabricate a + * timestamp. + * + * Self-contained on purpose: a `"use client"` boundary must not import + * server-only modules, since the dashboard shell transitively pulls in + * `lib/owner-token.ts` (`server-only`). + */ + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. + */ +const retryCounter = createRetryCounter(); + +export default function GrantsError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. + console.error(error); + }, [error]); + + useEffect(() => { + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // counter. There is deliberately no ceiling on the counter itself — a + // persistent failure degrades to a slow quiet heartbeat, never a dead end. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + reset(); + }, delay); + return () => clearTimeout(id); + }, [reset]); -export default function GrantsError(props: { error: Error & { digest?: string }; reset: () => void }) { return ( - +
+ +
); } diff --git a/apps/console/src/app/(console)/grants/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/grants/read-resilience.invariants.test.ts new file mode 100644 index 000000000..f23328266 --- /dev/null +++ b/apps/console/src/app/(console)/grants/read-resilience.invariants.test.ts @@ -0,0 +1,88 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the grants segment, mirroring + * `sources/read-resilience.invariants.test.ts`. See that file and + * `syncs/read-resilience.invariants.test.ts` for the full standard this + * pattern enforces; this file pins the same properties for `/grants`. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g; + +/** + * Strip `/* ... *‍/` block comments before checking for retired owner-facing + * copy. The boundary's doc comment legitimately QUOTES the retired phrases + * (to explain what this pattern replaces and why) — that is documentation, + * not rendered JSX text, so it must not trip the ban. + */ +function withoutBlockComments(src: string): string { + return src.replace(BLOCK_COMMENT_RE, ""); +} + +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_ERROR_HEADING_RE = /Read error/; +const RETIRED_TRY_AGAIN_RE = /Try again/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure|unchanged/i; +const RETIRED_BACK_LINK_RE = /Back to grants/; +const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/; + +const USES_LOADING_SKELETON_RE = / { + const rawSrc = await readFile(ERROR_FILE, "utf8"); + const src = withoutBlockComments(rawSrc); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE); + assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BACK_LINK_RE); + assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE); +}); + +test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + // loading.tsx uses ListLoadingSkeleton label="grants" rows={6}. + assert.match(src, /ListLoadingSkeleton label="grants" rows=\{6\}/); +}); + +test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, CALLS_RESET_RE); + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.doesNotMatch( + src, + NO_REACT_STATE_COUNTER_RE, + "the retry counter must live at module scope, not React state, or backoff never grows across remounts" + ); + assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded"); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/lib/collection-report.test.ts b/apps/console/src/app/(console)/lib/collection-report.test.ts index 668a95df9..9f3c02698 100644 --- a/apps/console/src/app/(console)/lib/collection-report.test.ts +++ b/apps/console/src/app/(console)/lib/collection-report.test.ts @@ -131,8 +131,10 @@ test("THE HONESTY GATE: collected records with an unknown considered denominator const facts = formatStreamCollectionFacts( entry({ collected: 42, considered: "unknown", coverage_condition: "unknown", stream: "items" }) ); - // The coverage chip stays unknown, never complete. - assert.equal(facts.coverage.value, "unknown"); + // The coverage chip stays unmeasured, never complete. B2: the owner-facing + // word for the `unknown` axis is "not measured"; the honesty claim that it + // must never read as complete is unchanged. + assert.equal(facts.coverage.value, "not measured"); assert.notEqual(facts.coverage.value, "complete"); // The counts line shows the raw count and an EXPLICIT unknown denominator — // never a "42 / 42" fraction that would read as complete. @@ -410,10 +412,17 @@ test("required missing evidence (unknown coverage) stays distinct from accepted // declaration resolves to `unknown` — this must keep reading as missing // evidence, never as a settled accepted-absence policy, so the two states // remain distinguishable on the stream row after the copy-only fix. + // + // B2 (2026-08-22): the owner-facing word for the `unknown` axis is now "not + // measured" — the SAME word the forward disposition already used — so the + // owner does not read two vocabularies for one state. The distinctness the + // rest of this test guards is unchanged. const unmeasured = formatStreamCollectionFacts( entry({ collected: 0, considered: "unknown", coverage_condition: "unknown", forward_disposition: "unmeasured" }) ); - assert.equal(unmeasured.coverage.value, "unknown"); + assert.equal(unmeasured.coverage.value, "not measured"); + // The coverage chip and the disposition line must now agree word-for-word. + assert.equal(unmeasured.disposition?.label, "not measured"); assert.notEqual( unmeasured.coverage.title, formatStreamCollectionFacts(entry({ coverage_condition: "deferred" })).coverage.title diff --git a/apps/console/src/app/(console)/lib/collection-report.ts b/apps/console/src/app/(console)/lib/collection-report.ts index 89d83da40..edd832ecc 100644 --- a/apps/console/src/app/(console)/lib/collection-report.ts +++ b/apps/console/src/app/(console)/lib/collection-report.ts @@ -27,8 +27,10 @@ * unit-testable without a browser harness. */ -import type { AxisChip, EvidenceTone, ForwardDispositionSummary } from "./connection-evidence.ts"; -import { formatCoverageAxis, formatForwardDisposition } from "./connection-evidence.ts"; +import type { AxisChip, EvidenceTone } from "@pdpp/display"; +import { formatCoverageAxis } from "@pdpp/display"; +import type { ForwardDispositionSummary } from "./connection-evidence.ts"; +import { formatForwardDisposition } from "./connection-evidence.ts"; import type { RefCollectionReportEntry } from "./ref-client.ts"; export interface StreamCollectionFacts { diff --git a/apps/console/src/app/(console)/lib/connection-catalog.test.ts b/apps/console/src/app/(console)/lib/connection-catalog.test.ts index 3cc029d8b..55696448b 100644 --- a/apps/console/src/app/(console)/lib/connection-catalog.test.ts +++ b/apps/console/src/app/(console)/lib/connection-catalog.test.ts @@ -270,6 +270,11 @@ test("browser-bound static-secret capability is not enough to create an account" test("non-browser static-secret connectors keep the existing single capture path", () => { const catalog = buildConnectorCatalog([ { + // No public_listing declared -> defaults to Development (the same + // fixture shape this test always used). "gmail" is a real, + // live-proven connector key (STATIC_SECRET_LIVE_PROVEN_KEYS), so the + // shared planner resolves static_secret_connect purely from the + // connector key regardless of this synthetic manifest's declared tier. connector_id: "https://registry.pdpp.dev/connectors/gmail", display_name: "Gmail", runtime_requirements: { bindings: { network: { required: true } } }, @@ -290,7 +295,13 @@ test("non-browser static-secret connectors keep the existing single capture path assert.equal(entry.modality, "api_network"); assert.equal(entry.setupModality, "static_secret"); assert.equal(entry.disposition, "static_secret_connect"); - assert.equal(sourceSetupAction(entry), null); + // "gmail" is real (not a known scaffold) and its disposition + // (static_secret_connect) IS in the Development disclosure's self-test + // allowlist, so it gets a self-test action in the Development disclosure + // even though it is not owner-actionable (the whole Development tier is + // hard-excluded from ownerActionable). + assert.equal(entry.isKnownScaffold, false); + assert.equal(sourceSetupAction(entry) !== null, true, "a real development entry gets a self-test action"); assert.equal(sourceSetupSecondaryAction(entry), null); assert.equal(sourceSetupStatus(entry).label, "Development"); }); @@ -690,8 +701,15 @@ test("configured Google provider readiness exposes the existing owner authorizat assert.equal(entry.supportState, "supported"); assert.equal(entry.disposition, "provider_auth_connect"); assert.equal(sourceSetupStatus(entry).label, "Development"); - assert.match(sourceSetupGuidance(entry), PROVIDER_BROWSER_GUIDANCE_RE); - assert.equal(sourceSetupAction(entry), null); + // google-maps-data-portability is real, not a known scaffold (its own + // manifest documents exactly what is and is not implemented via + // public_listing.proof_gate), and provider_auth_connect IS in the + // Development disclosure's self-test allowlist -- so this configured, + // ready-to-authorize entry gets a self-test action even though it is not + // owner-actionable (the whole Development tier is hard-excluded from + // ownerActionable, and sourceSetupAvailability stays "not_available_here"). + assert.equal(entry.isKnownScaffold, false); + assert.equal(sourceSetupAction(entry) !== null, true, "a real development entry gets a self-test action"); assert.equal(sourceSetupAvailability(entry), "not_available_here"); assert.deepEqual(providerAuthConnectEntries(catalog), [entry]); }); @@ -932,12 +950,17 @@ test("filesystem connectors outside the proven set are local-collector-unproven, } }); -test("owner catalog never offers a development template, whatever its setup state", () => { - // An unlisted connector must not be offered or addable on the OFFER surface, - // and `experimental` is not an exception: the Experimental section presents - // what is already offered rather than acting as a second door into the - // catalog. Both an experimental and a non-experimental unlisted template are - // asserted here so the gate cannot be reopened for one support_state alone. +test("owner catalog never offers a development template as a runnable add offer, whatever its setup state", () => { + // A Development-tier template must never be a runnable OFFER (the main + // list or the Preview disclosure), and `experimental` is not an exception: + // the Experimental section presents what is already offered rather than + // acting as a second door into the catalog. Both an experimental and a + // non-experimental unlisted template are asserted here so the gate cannot + // be reopened for one support_state alone. Unlike the pre-Development- + // disclosure contract, these rows DO now appear in the raw catalog array + // (see connection-catalog.ts) so the owner can see them in the Development + // disclosure -- `isRunnableAddOffer` is the authority for "offered", not + // catalog membership. const catalog = buildOwnerConnectorCatalog( [], [ @@ -979,20 +1002,27 @@ test("owner catalog never offers a development template, whatever its setup stat }), ] ); + const unlistedExperimental = catalog.find((e) => e.connectorKey === "unlisted-experimental"); + assert.ok(unlistedExperimental, "development rows are visible in the catalog for the Development disclosure"); assert.equal( - catalog.find((e) => e.connectorKey === "unlisted-experimental"), - undefined, - "an unlisted experimental template must not be offered" + isRunnableAddOffer(unlistedExperimental), + false, + "an unlisted experimental template must not be a runnable add offer" ); + + const unlistedProofGated = catalog.find((e) => e.connectorKey === "unlisted-proof-gated"); + assert.ok(unlistedProofGated, "development rows are visible in the catalog for the Development disclosure"); assert.equal( - catalog.find((e) => e.connectorKey === "unlisted-proof-gated"), - undefined, - "an unlisted non-experimental template must stay dropped" + isRunnableAddOffer(unlistedProofGated), + false, + "an unlisted non-experimental template must not be a runnable add offer" ); + // The listing gate must not swallow the Experimental section itself: a // connector the operator HAS listed still reaches it. const listedExperimental = catalog.find((e) => e.connectorKey === "listed-experimental"); assert.ok(listedExperimental, "a preview experimental template must still be offered"); + assert.equal(isRunnableAddOffer(listedExperimental), true, "a preview experimental template is a runnable offer"); assert.equal(sourceSetupAvailability(listedExperimental), "experimental_opt_in"); assert.ok(sourceSetupAction(listedExperimental), "a listed experimental template keeps its add action"); }); @@ -1066,6 +1096,24 @@ test("isOwnerActionableEntry respects demo/test fallback rules when ownerActiona assert.equal(isOwnerActionableEntry(ynab), true); }); +/** + * A real (non-scaffold) Development entry whose disposition resolves to one + * of `sourceSetupAction`'s runnable dispositions gets a self-test action even + * though it is not owner-actionable (the server hard-disables + * `ownerActionable` for the whole Development tier). This mirrors the exact + * disposition set `sourceSetupAction` itself special-cases; kept here as an + * independent literal (not an import) so a drift between the two would fail + * this test rather than silently agreeing with itself. + */ +const DEVELOPMENT_SELF_TEST_DISPOSITIONS = new Set([ + "local_collector_enroll", + "static_secret_connect", + "static_secret_experimental", + "manual_upload_connect", + "browser_collector_manual", + "provider_auth_connect", +]); + test("presentation consistency: helper functions agree with ownerActionable authority", async () => { // Every fixture in the presentation test suite must have presentation functions // that agree with isOwnerActionableEntry. This is the core maintainability check. @@ -1076,27 +1124,47 @@ test("presentation consistency: helper functions agree with ownerActionable auth // false is what keeps it out of the calm "available now" list and every // owner-agent REST/actionability surface; the explicit Experimental opt-in // section is the only place its action renders. + // + // Exception: a real (non-scaffold) Development entry with a runnable + // disposition also has a real action -- the Development disclosure's own + // self-test opt-in -- even though isOwnerActionableEntry is false for the + // whole tier. A KNOWN scaffold never gets an action regardless of + // disposition: clicking it can never collect anything. const manifests = await loadCommittedManifests(); const catalog = buildConnectorCatalog(manifests); for (const entry of catalog) { const isActionable = isOwnerActionableEntry(entry); const hasAction = sourceSetupAction(entry) !== null; + const isDevelopmentSelfTestable = + entry.publicTier === "development" && !entry.isKnownScaffold && DEVELOPMENT_SELF_TEST_DISPOSITIONS.has(entry.disposition); if (entry.supportState === "experimental") { assert.equal(isActionable, false, `${entry.connectorKey}: experimental must not be owner-actionable`); - assert.equal(hasAction, entry.publicTier !== "development", `${entry.connectorKey}: only preview experimental entries expose an opt-in action`); + assert.equal( + hasAction, + entry.publicTier !== "development" || isDevelopmentSelfTestable, + `${entry.connectorKey}: only preview experimental entries (or a self-testable development entry) expose an opt-in action` + ); continue; } // The invariant: if isOwnerActionableEntry returns true, sourceSetupAction - // must have a non-null result. Mutations to either would break this. + // must have a non-null result. Mutations to either would break this. A + // development entry is the one deliberate exception: it can have an + // action while isOwnerActionableEntry stays false for the tier. assert.equal( hasAction, - isActionable && entry.publicTier !== "development", - `${entry.connectorKey}: sourceSetupAction must match isOwnerActionableEntry. ` + - `Helper says ${isActionable}, action is ${hasAction ? "set" : "null"}` + (isActionable && entry.publicTier !== "development") || isDevelopmentSelfTestable, + `${entry.connectorKey}: sourceSetupAction must match isOwnerActionableEntry (or the development self-test exception). ` + + `Helper says ${isActionable}, action is ${hasAction ? "set" : "null"}, isDevelopmentSelfTestable=${isDevelopmentSelfTestable}` ); + + // A KNOWN scaffold must NEVER get an action, regardless of disposition: + // it cannot collect anything, so an add button would be a dead end. + if (entry.publicTier === "development" && entry.isKnownScaffold) { + assert.equal(hasAction, false, `${entry.connectorKey}: a known scaffold must never expose an action`); + } } }); @@ -1305,14 +1373,27 @@ test("a connector-key allowlist cannot declare readiness a deployment has not su ); }); -test("console catalog exposes a development connector only with server UAT authority", () => { +test("console catalog uses the manifest tier as its sole listing authority", () => { + // Development-tier entries flow through as catalog rows -- the owner + // running this instance must be able to see what is registered and tell + // "unproven" apart from "unimplemented" (Development disclosure on + // /sources/add). But the manifest tier remains the sole RUNNABLE-OFFER + // authority: a development entry is never in the main list or the Preview + // disclosure, and the obsolete UAT exposure fact cannot promote it there + // either. const uatFalseTemplate = ownerTemplate({ connectorKey: "test-unproven", tier: "development", uat_expose_unlisted_connectors: false, }); let catalog = buildOwnerConnectorCatalog([], [uatFalseTemplate]); - assert.equal(catalog.length, 0, "development must be filtered from Add Source"); + assert.equal(catalog.length, 1, "development must still be visible in the catalog for the Development disclosure"); + let entry = catalog[0]; + assert.ok(entry); + assert.equal(isRunnableAddOffer(entry), false, "development is never a runnable add offer"); + // Real (non-scaffold), self-testable disposition: gets a self-test action + // in the Development disclosure even though it is never a runnable offer. + assert.equal(sourceSetupAction(entry) !== null, true, "a real development entry gets a self-test action"); // The authenticated server can selectively expose one Development connector // without changing its lifecycle tier. @@ -1322,7 +1403,19 @@ test("console catalog exposes a development connector only with server UAT autho uat_expose_unlisted_connectors: true, }); catalog = buildOwnerConnectorCatalog([], [uatTrueTemplate]); - assert.equal(catalog.length, 1, "explicit UAT exposure must admit the named development connector"); - assert.equal(catalog[0]?.publicTier, "development", "UAT exposure must not promote the lifecycle tier"); - assert.equal(catalog[0]?.ownerActionable, true, "the exposed setup path must be actionable in UAT"); + entry = catalog[0]; + assert.ok(entry); + assert.equal(isRunnableAddOffer(entry), false, "UAT exposure must not override development"); + + // A KNOWN scaffold never gets a self-test action, regardless of UAT exposure. + const scaffoldTemplate: OwnerConnectorTemplateLike = { + ...ownerTemplate({ connectorKey: "test-scaffold", tier: "development" }), + is_known_scaffold: true, + }; + catalog = buildOwnerConnectorCatalog([], [scaffoldTemplate]); + entry = catalog[0]; + assert.ok(entry); + assert.equal(entry.isKnownScaffold, true); + assert.equal(isRunnableAddOffer(entry), false, "a scaffold is never a runnable add offer"); + assert.equal(sourceSetupAction(entry), null, "a scaffold never gets a self-test action"); }); diff --git a/apps/console/src/app/(console)/lib/connection-catalog.ts b/apps/console/src/app/(console)/lib/connection-catalog.ts index 52eb4abea..ddc49da8d 100644 --- a/apps/console/src/app/(console)/lib/connection-catalog.ts +++ b/apps/console/src/app/(console)/lib/connection-catalog.ts @@ -26,6 +26,7 @@ import { classifyConnectorIntentModality, connectorKeyFromManifest, enrollmentKeyForCanonicalKey, + isKnownScaffoldConnector, manualUploadSetupFromManifest, type StaticSecretSetupFieldLike, staticSecretCredentialCaptureFromManifest, @@ -49,6 +50,15 @@ export interface CatalogManifestLike { rationale?: string | null; } | null; public_listing?: { + /** + * Owner-facing reason this connector is not offered as a runnable + * "add now"/Preview card yet, when the manifest declares one. Written + * for a development-tier connector, so it is the most honest, + * connector-specific text available for the Development disclosure. + */ + proof_gate?: string | null; + /** Manifest-authored explanation for the current lifecycle tier. */ + rationale?: string | null; tier?: "supported" | "preview" | "development" | null; } | null; } | null; @@ -108,7 +118,18 @@ export interface OwnerConnectorTemplateLike { kind?: string | null; svg?: string | null; } | null; + /** + * Server-owned fact, meaningful only for Development-tier entries: true when + * the connector-conformance roster names this connector a KNOWN scaffold + * (unconditional `SKIP_RESULT`, no real collection). A scaffold must never + * render an add action, even inside a Development disclosure. + */ + is_known_scaffold?: boolean | null; public_listing?: { + /** Owner-facing reason this connector is not offered as a runnable card yet, when declared. */ + proof_gate?: string | null; + /** Manifest-authored explanation for the current lifecycle tier. */ + rationale?: string | null; tier?: "supported" | "preview" | "development" | null; } | null; registration_status?: string | null; @@ -207,6 +228,21 @@ export interface ConnectorCatalogEntry { externalDocs: readonly ConnectorExternalDoc[]; /** Optional manifest-declared brand glyph; absent renders the Monogram fallback (see ConnectorIcon). */ icon?: OwnerConnectorTemplateLike["icon"]; + /** + * Meaningful only for Development-tier entries: true when the connector is + * a KNOWN scaffold (unconditional `SKIP_RESULT`, no real collection) rather + * than real-but-unproven. Drives whether a Development disclosure card may + * ever render an add action for this entry. + */ + isKnownScaffold: boolean; + /** + * Manifest-authored explanation for the current lifecycle tier + * (`public_listing.rationale` or `public_listing.proof_gate`, in that + * order), when the manifest declares one. Most specific, most honest + * per-connector text available for the Development disclosure; a + * connector without one falls back to generic tier copy. + */ + listingNote: string | null; /** Binding-derived modality. */ modality: CatalogModality; /** The next owner step selected by the shared planner. */ @@ -262,6 +298,12 @@ function cleanManifestText(value: unknown): string | null { return typeof value === "string" && value.trim() ? value.trim() : null; } +function listingNoteFromPublicListing( + listing: { proof_gate?: string | null; rationale?: string | null } | null | undefined +): string | null { + return cleanManifestText(listing?.rationale) ?? cleanManifestText(listing?.proof_gate); +} + function setupCopyFromManifest(manifest: CatalogManifestLike): { description: string | null; helpText: string | null; @@ -335,6 +377,8 @@ export function buildConnectorCatalog( displayName: displayNameFor(manifest, connectorKey), disposition: plan.catalogDisposition, externalDocs: externalDocsFromManifest(manifest), + isKnownScaffold: isKnownScaffoldConnector(connectorKey), + listingNote: listingNoteFromPublicListing(manifest.capabilities?.public_listing), modality: plan.connectorModality, nextStepKind: plan.nextStepKind, proofGate: plan.proofGate, @@ -477,13 +521,20 @@ export function buildOwnerConnectorCatalog( for (const template of templates) { const connectorKey = cleanManifestText(template.connector_key); const setupPlan = template.setup_plan; - // Development remains hidden unless the authenticated server explicitly - // exposes this exact template for UAT. - if ( - !connectorKey || - template.registration_status !== "registered" || - (template.public_listing?.tier === "development" && template.uat_expose_unlisted_connectors !== true) - ) { + // Development-tier entries flow through as catalog entries so the owner + // running this instance can see what exists and self-test it -- they are + // never a runnable "add now" or Preview offer (see isRunnableAddOffer and + // the Development disclosure in source-setup-catalog.tsx). The manifest + // tier remains the sole listing-TIER authority; this only stops dropping + // the row outright. + // + // This supersedes the narrower `uat_expose_unlisted_connectors` gate that + // previously guarded this filter: that flag only revealed development rows + // the server explicitly opted in, which still left the owner unable to see + // the other development connectors on his own instance. The flag remains + // authoritative for the owner-actionable/disposition decisions above; it is + // only its use as a LISTING gate here that this replaces. + if (!connectorKey || template.registration_status !== "registered") { continue; } const disposition = setupPlan?.catalog_disposition; @@ -528,6 +579,11 @@ export function buildOwnerConnectorCatalog( disposition, externalDocs: externalDocsFromManifest(manifestForCopy), icon: template.icon ?? null, + isKnownScaffold: + typeof template.is_known_scaffold === "boolean" + ? template.is_known_scaffold + : isKnownScaffoldConnector(connectorKey), + listingNote: listingNoteFromPublicListing(template.public_listing), modality: connectorModality, nextStepKind, ownerActionable: capability.actionable, diff --git a/apps/console/src/app/(console)/lib/connection-control-result.test.ts b/apps/console/src/app/(console)/lib/connection-control-result.test.ts index 3c0eea9c6..9ecc87fde 100644 --- a/apps/console/src/app/(console)/lib/connection-control-result.test.ts +++ b/apps/console/src/app/(console)/lib/connection-control-result.test.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Unit tests for the `revokeConnection` / `deleteConnection` client-wrapper - * outcome mappings. + * Unit tests for the `revokeConnection` / `deleteConnection` / + * `pauseConnection` / `resumeConnection` client-wrapper outcome mappings. * * The pure `(status, body, code)` → outcome classifiers live in * `connection-control-result.ts` (not `operator-runs.ts`) specifically so they @@ -29,6 +29,8 @@ import test from "node:test"; import { fileURLToPath } from "node:url"; import { classifyDeleteConnectionResponse, + classifyPauseConnectionResponse, + classifyResumeConnectionResponse, classifyRevokeConnectionResponse, connectionControlErrorCode, } from "./connection-control-result.ts"; @@ -46,6 +48,14 @@ const WRAPPER_DELETE_PATH_RE = /connectionControlPath\(connectionId, ""\)/; const WRAPPER_DELETE_METHOD_RE = /method: "DELETE"/; const WRAPPER_DELETE_CLASSIFY_RE = /classifyDeleteConnectionResponse\(response\.status, body, connectionControlErrorCode\(body\)\)/; +const PAUSE_THROWS_RE = /nope|connection pause failed/; +const RESUME_THROWS_RE = /nope|connection resume failed/; +const WRAPPER_PAUSE_PATH_RE = /connectionControlPath\(connectionId, "\/pause"\)/; +const WRAPPER_PAUSE_CLASSIFY_RE = + /classifyPauseConnectionResponse\(response\.status, body, connectionControlErrorCode\(body\)\)/; +const WRAPPER_RESUME_PATH_RE = /connectionControlPath\(connectionId, "\/resume"\)/; +const WRAPPER_RESUME_CLASSIFY_RE = + /classifyResumeConnectionResponse\(response\.status, body, connectionControlErrorCode\(body\)\)/; test("revoke 200 maps to revoked", () => { assert.deepEqual(classifyRevokeConnectionResponse(200, { status: "revoked" }, null), { status: "revoked" }); @@ -121,3 +131,72 @@ test("operator-runs deleteConnection DELETEs the shared owner-session connection assert.match(src, WRAPPER_DELETE_METHOD_RE); assert.match(src, WRAPPER_DELETE_CLASSIFY_RE); }); + +// --- Pause / resume --------------------------------------------------------- +// The typed refusals matter as much as the successes: a repeat pause (or a +// resume of an already-active connection) is a no-op the console messages in +// place, NOT an error banner, so each must classify rather than throw. + +test("pause maps 200 to paused", () => { + assert.deepEqual(classifyPauseConnectionResponse(200, { object: "owner_connection_pause" }, null), { + status: "paused", + }); +}); + +test("pause maps 409 connector_instance_not_active to not_active", () => { + const body = { error: { code: "connector_instance_not_active" } }; + assert.deepEqual(classifyPauseConnectionResponse(409, body, connectionControlErrorCode(body)), { + status: "not_active", + }); +}); + +test("pause maps 404 connector_instance_not_found to not_found", () => { + const body = { error: { code: "connector_instance_not_found" } }; + assert.deepEqual(classifyPauseConnectionResponse(404, body, connectionControlErrorCode(body)), { + status: "not_found", + }); +}); + +test("pause on an unexpected status throws a described error", () => { + const body = { error: { code: "api_error", message: "nope" } }; + assert.throws(() => classifyPauseConnectionResponse(500, body, connectionControlErrorCode(body)), PAUSE_THROWS_RE); +}); + +test("resume maps 200 to resumed", () => { + assert.deepEqual(classifyResumeConnectionResponse(200, { object: "owner_connection_resume" }, null), { + status: "resumed", + }); +}); + +test("resume maps 409 connector_instance_not_paused to not_paused", () => { + const body = { error: { code: "connector_instance_not_paused" } }; + assert.deepEqual(classifyResumeConnectionResponse(409, body, connectionControlErrorCode(body)), { + status: "not_paused", + }); +}); + +test("resume maps 404 connector_instance_not_found to not_found", () => { + const body = { error: { code: "connector_instance_not_found" } }; + assert.deepEqual(classifyResumeConnectionResponse(404, body, connectionControlErrorCode(body)), { + status: "not_found", + }); +}); + +test("resume on an unexpected status throws a described error", () => { + const body = { error: { code: "api_error", message: "nope" } }; + assert.throws(() => classifyResumeConnectionResponse(500, body, connectionControlErrorCode(body)), RESUME_THROWS_RE); +}); + +test("operator-runs pauseConnection POSTs the shared owner-session pause route through the classifier", async () => { + const src = await readFile(OPERATOR_RUNS_FILE, "utf8"); + assert.match(src, WRAPPER_PAUSE_PATH_RE); + assert.match(src, WRAPPER_POST_RE); + assert.match(src, WRAPPER_PAUSE_CLASSIFY_RE); +}); + +test("operator-runs resumeConnection POSTs the shared owner-session resume route through the classifier", async () => { + const src = await readFile(OPERATOR_RUNS_FILE, "utf8"); + assert.match(src, WRAPPER_RESUME_PATH_RE); + assert.match(src, WRAPPER_POST_RE); + assert.match(src, WRAPPER_RESUME_CLASSIFY_RE); +}); diff --git a/apps/console/src/app/(console)/lib/connection-control-result.ts b/apps/console/src/app/(console)/lib/connection-control-result.ts index 2e2692b68..ee7b88cab 100644 --- a/apps/console/src/app/(console)/lib/connection-control-result.ts +++ b/apps/console/src/app/(console)/lib/connection-control-result.ts @@ -6,6 +6,7 @@ import { describeError } from "./describe-error.ts"; /** * Pure mappings for the owner-session connection control responses * (`POST /_ref/connections/:id/revoke`, `POST /_ref/connections/:id/reactivate`, + * `POST /_ref/connections/:id/pause`, `POST /_ref/connections/:id/resume`, * and `DELETE /_ref/connections/:id`), factored out of `operator-runs.ts` so * they can be unit tested directly under `node --test` without pulling in the * server-only fetch helpers (`owner-token.ts` imports `server-only`, which @@ -27,6 +28,18 @@ import { describeError } from "./describe-error.ts"; * already active; nothing to reactivate) * - `404 connector_instance_not_found` → `not_found` * + * Pause: + * - `200` → `paused` + * - `409 connector_instance_not_active` → `not_active` (already paused, + * draft, or revoked; nothing to pause) + * - `404 connector_instance_not_found` → `not_found` + * + * Resume: + * - `200` → `resumed` + * - `409 connector_instance_not_paused` → `not_paused` (connection is + * already active; nothing to resume) + * - `404 connector_instance_not_found` → `not_found` + * * Delete: * - `200` → `deleted` * - `409 connection_run_active` → `run_active` (a run is in flight; stop it @@ -69,6 +82,18 @@ export function classifyReactivateConnectionResponse( throw new Error(describeError(body, `connection reactivate failed (${status})`)); } +export type PauseConnectionOutcome = "paused" | "not_active" | "not_found"; + +export interface PauseConnectionResult { + status: PauseConnectionOutcome; +} + +export type ResumeConnectionOutcome = "resumed" | "not_paused" | "not_found"; + +export interface ResumeConnectionResult { + status: ResumeConnectionOutcome; +} + export type DeleteConnectionOutcome = "deleted" | "run_active" | "default_account" | "not_found"; export interface DeleteConnectionResult { @@ -108,6 +133,53 @@ export function classifyRevokeConnectionResponse( throw new Error(describeError(body, `connection revoke failed (${status})`)); } +/** + * Map a pause response `(status, body, errorCode)` to a typed outcome, or throw + * a described error for any status that is not a documented pause outcome. + * `not_active` covers every non-active target (already paused, draft, revoked) + * — the route answers one typed code for all of them, so a repeat pause is a + * clean no-op the console messages in place rather than an error banner. + */ +export function classifyPauseConnectionResponse( + status: number, + body: unknown, + errorCode: string | null +): PauseConnectionResult { + if (status === 200) { + return { status: "paused" }; + } + if (status === 409 && errorCode === "connector_instance_not_active") { + return { status: "not_active" }; + } + if (status === 404 && errorCode === "connector_instance_not_found") { + return { status: "not_found" }; + } + throw new Error(describeError(body, `connection pause failed (${status})`)); +} + +/** + * Map a resume response `(status, body, errorCode)` to a typed outcome, or + * throw a described error for any status that is not a documented resume + * outcome. The inverse of {@link classifyPauseConnectionResponse}: + * `not_paused` covers every non-paused target. + */ +export function classifyResumeConnectionResponse( + status: number, + body: unknown, + errorCode: string | null +): ResumeConnectionResult { + if (status === 200) { + return { status: "resumed" }; + } + if (status === 409 && errorCode === "connector_instance_not_paused") { + return { status: "not_paused" }; + } + if (status === 404 && errorCode === "connector_instance_not_found") { + return { status: "not_found" }; + } + throw new Error(describeError(body, `connection resume failed (${status})`)); +} + /** * Map a delete response `(status, body, errorCode)` to a typed outcome, or throw * a described error for any status that is not a documented delete outcome. diff --git a/apps/console/src/app/(console)/lib/connection-evidence.test.ts b/apps/console/src/app/(console)/lib/connection-evidence.test.ts index 49a01ef7c..2dbc28056 100644 --- a/apps/console/src/app/(console)/lib/connection-evidence.test.ts +++ b/apps/console/src/app/(console)/lib/connection-evidence.test.ts @@ -14,6 +14,7 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { formatCoverageAxis, formatFreshnessAxis, formatOutboxAxis } from "@pdpp/display"; import { deriveAutoPausedBanner, deriveConnectionNextStep, @@ -22,12 +23,9 @@ import { derivePrimaryRowAction, deriveStreakDots, formatCollectionRateReadout, - formatCoverageAxis, formatDominantCondition, formatForwardDisposition, - formatFreshnessAxis, formatLastDurableProgress, - formatOutboxAxis, formatProjectionFreshness, formatSourceHeartbeat, formatSourceOutboxState, @@ -110,7 +108,11 @@ function baseOverview(overrides: Partial = {}): ConnectorOver test("coverage axis never labels 'unknown' as 'complete'", () => { assert.equal(formatCoverageAxis("unknown").tone, "neutral"); - assert.equal(formatCoverageAxis("unknown").label.toLowerCase().includes("unknown"), true); + // B2: the `unknown` axis renders as "not measured" — never as "complete", + // and never as the bare word "unknown" that competed with the disposition + // line's "not measured" for the same underlying state. + assert.equal(formatCoverageAxis("unknown").value, "not measured"); + assert.equal(formatCoverageAxis("unknown").label.toLowerCase().includes("complete"), false); assert.equal(formatCoverageAxis("complete").tone, "success"); assert.equal(formatCoverageAxis("gaps").tone, "warning"); assert.equal(formatCoverageAxis("partial").tone, "warning"); @@ -195,14 +197,23 @@ test("the visible deferred pill reads optional/not-collected, not policy jargon assert.equal(chip.tone, "neutral"); }); -test("inventory_only, unavailable, and unsupported visible labels are unchanged (not demonstrably misleading)", () => { - // Only the deferred pill's visible value/label was demonstrably misleading - // (read as queued work). The sibling accepted-absence labels already read - // as plain, settled facts ("inventory only", "unavailable", "unsupported") - // with no queued-work connotation, so their visible value/label are left - // untouched per the owner-gate scope; only their titles were sharpened. - assert.equal(formatCoverageAxis("inventory_only").value, "inventory only"); - assert.equal(formatCoverageAxis("inventory_only").label, "Coverage · inventory only"); +test("inventory_only says plainly that it is complete by design, and stays neutral", () => { + // B4 (owner ledger 2026-08-22): the owner asked whether the neutral/green tone + // on `inventory only` was intentional. It IS, and it is honest — + // `inventory_only` is an AcceptedAbsencePolicy that `hasOutstandingGap` + // excludes and `deriveForwardDisposition` resolves to `complete`, so the + // connection genuinely owes no further data. The tone therefore stays + // neutral. What changed is the WORD: "inventory only" read as a limitation + // the owner might have to act on, so the value now states that this is a + // finished state by design. + const chip = formatCoverageAxis("inventory_only"); + assert.equal(chip.tone, "neutral"); + assert.match(chip.value, /complete/i); + assert.match(chip.value, /design/i); + assert.equal(chip.label.startsWith("Coverage"), true); + // It must NOT read as an unmeasured or gapped state — those are real defects + // and this is not one. + assert.doesNotMatch(chip.value, /not measured|unknown|gap|missing/i); assert.equal(formatCoverageAxis("unavailable").value, "unavailable"); assert.equal(formatCoverageAxis("unavailable").label, "Coverage · unavailable"); assert.equal(formatCoverageAxis("unsupported").value, "unsupported"); @@ -221,7 +232,7 @@ test("axis chips degrade safely when runtime axes are missing or novel", () => { assert.equal(out.length, 2); assert.deepEqual( out.map((c) => c.label), - ["Coverage · unknown", "Freshness · unknown"] + ["Coverage · not measured", "Freshness · not measured"] ); assert.equal( out.every((c) => c.tone === "neutral"), @@ -243,7 +254,7 @@ test("axis chips: a novel outbox value still degrades to neutral 'unknown' for a // degrades through formatOutboxAxis to the neutral unknown fallback chip; the // "evidence unavailable" sharpening only applies to a literal `unknown` axis. assert.equal(out.length, 3); - assert.equal(out[2]?.label, "Outbox · unknown"); + assert.equal(out[2]?.label, "Outbox · not measured"); assert.equal(out[2]?.tone, "neutral"); }); @@ -1963,7 +1974,7 @@ test("deriveFailureSummary uses the server wait verdict even when raw health say renderedVerdict({ channel: "advisory", forward_statement: "The source is throttling this connection; it will retry automatically.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: [], diff --git a/apps/console/src/app/(console)/lib/connection-evidence.ts b/apps/console/src/app/(console)/lib/connection-evidence.ts index 2d8c4b837..4b6684b66 100644 --- a/apps/console/src/app/(console)/lib/connection-evidence.ts +++ b/apps/console/src/app/(console)/lib/connection-evidence.ts @@ -18,6 +18,8 @@ * without a browser harness. */ +import type { AxisChip, EvidenceTone } from "@pdpp/display"; +import { formatAttentionAxis, formatCoverageAxis, formatFreshnessAxis, formatOutboxAxis } from "@pdpp/display"; import type { DeviceSourceInstance, RefCollectionRateSnapshot, @@ -32,256 +34,11 @@ import type { import type { ConnectorOverview, ConnectorRunRef } from "./rs-client.ts"; import { formatTotalRecordsLabel, isTotalRecordsAuthoritative } from "./total-records-label.ts"; -export type EvidenceTone = "neutral" | "success" | "warning" | "danger"; - -export interface AxisChip { - /** The axis name (e.g. "Coverage", "Freshness"). Rendered muted. */ - dimension: string; - /** Short owner-facing label (e.g. "Coverage · gaps"). Kept for backward compat/tooltips. */ - label: string; - /** Long-form hover/tooltip — describes what the chip means. */ - title: string; - tone: EvidenceTone; - /** The axis state value (e.g. "gaps", "fresh"). Rendered prominent. */ - value: string; -} - -const COVERAGE_LABELS: Record = { - complete: { - dimension: "Coverage", - label: "Coverage · complete", - title: "All required streams have durable evidence of complete coverage.", - tone: "success", - value: "complete", - }, - deferred: { - dimension: "Coverage", - label: "Coverage · optional, not collected", - title: - "The manifest declares this coverage out of scope. This is an accepted, settled state — not a queued task — and does not block connection health.", - tone: "neutral", - // The underlying axis key stays "deferred" (durable manifest/runtime - // contract — see AcceptedCoveragePolicy in connector-coverage-policy.ts). - // "Deferred" read as queued/pending work to owners, contradicting the - // settled, non-degrading semantics this axis actually carries. The - // visible value/label now say plainly that this stream is optional and - // not collected; the manifest-declaration detail moves to the title. - value: "optional, not collected", - }, - gaps: { - dimension: "Coverage", - label: "Coverage · gaps", - title: "Required coverage has known retryable or terminal gaps.", - tone: "warning", - value: "gaps", - }, - inventory_only: { - dimension: "Coverage", - label: "Coverage · inventory only", - title: - "The manifest declares that only inventory/discovery evidence is ever required here, not full detail. This is a settled, complete state for this stream — not partial progress.", - tone: "neutral", - value: "inventory only", - }, - partial: { - dimension: "Coverage", - label: "Coverage · partial", - title: "Some required streams collected only partial data.", - tone: "warning", - value: "partial", - }, - retryable_gap: { - dimension: "Coverage", - label: "Coverage · retryable gap", - title: - "Some required detail is missing, but the runtime expects to fill it on a later run. Records already collected stay valid; no owner action is needed yet.", - tone: "warning", - value: "retryable gap", - }, - terminal_gap: { - dimension: "Coverage", - label: "Coverage · won't backfill", - title: - "Some required detail will not backfill on its own — the connector or source cannot recover it without a change. Records already collected stay valid and usable; this is not current data loss. Open the connection's latest run to see which streams are affected and the recovery step.", - tone: "danger", - // "terminal gap" is jargon. The value stays short for the chip; the title - // carries the three things the owner actually needs (per design-notes/ - // dashboard-health-semantics-and-reliability-2026-06-01.md): what state this - // is, whether current records are safe, and what can recover coverage. The - // reference's coverage condition carries a `Review source coverage gaps` - // remediation but not the specific cause/stream/time — that contract gap is - // noted in the workstream report; the per-stream detail lives in the latest - // run's known_gaps, which the connection detail page links to. - value: "won't backfill", - }, - unavailable: { - dimension: "Coverage", - label: "Coverage · unavailable", - title: - "The manifest accepts that the source does not expose this coverage. This is a settled state, not a temporary gap awaiting a retry.", - tone: "neutral", - value: "unavailable", - }, - unknown: { - dimension: "Coverage", - label: "Coverage · unknown", - title: "No durable coverage evidence is available yet.", - tone: "neutral", - value: "unknown", - }, - unsupported: { - dimension: "Coverage", - label: "Coverage · unsupported", - title: - "The manifest accepts that the connector cannot collect this coverage. This is a settled state, not a temporary gap awaiting a retry.", - tone: "neutral", - value: "unsupported", - }, -}; - -const FRESHNESS_LABELS: Record = { - fresh: { - dimension: "Freshness", - label: "Freshness · fresh", - title: "The last successful run is within policy.", - tone: "success", - value: "fresh", - }, - stale: { - dimension: "Freshness", - label: "Freshness · stale", - title: "The last successful run is outside the configured freshness window.", - tone: "warning", - value: "stale", - }, - unknown: { - dimension: "Freshness", - label: "Freshness · unknown", - title: "Freshness cannot be derived from current evidence.", - tone: "neutral", - value: "unknown", - }, -}; - -const OUTBOX_LABELS: Record = { - active: { - dimension: "Outbox", - label: "Outbox · active", - title: "Outbound work is making progress.", - // `active` means the local-device outbox is draining — a healthy, - // progressing state. It previously shared `neutral` (muted grey) with - // `unknown`, so an operator could not tell a draining outbox from one - // whose evidence we could not read. `success` gives it a distinct, - // non-alarming colour (the same green as `idle`); the value text - // ("active" vs "idle") carries the finer distinction, and the row-level - // pill still escalates an actively-draining outbox to a "Syncing" badge. - tone: "success", - value: "active", - }, - idle: { - dimension: "Outbox", - label: "Outbox · idle", - title: "No retryable outbound work is pending.", - tone: "success", - value: "idle", - }, - stalled: { - dimension: "Outbox", - label: "Outbox · stalled", - title: "Retryable outbound work is stalled and not progressing.", - tone: "danger", - value: "stalled", - }, - unknown: { - dimension: "Outbox", - label: "Outbox · unknown", - title: "Outbox state cannot be read from durable evidence.", - tone: "neutral", - value: "unknown", - }, -}; - -const ATTENTION_LABELS: Record = { - acknowledged: { - dimension: "Attention", - label: "Attention · acknowledged", - title: "Owner action is acknowledged but not yet resolved.", - tone: "warning", - value: "acknowledged", - }, - in_progress: { - dimension: "Attention", - label: "Attention · in progress", - title: "Owner action is in progress.", - tone: "warning", - value: "in progress", - }, - none: null, - open: { - dimension: "Attention", - label: "Attention · open", - title: "Owner action is open.", - tone: "warning", - value: "open", - }, -}; - -export function formatCoverageAxis( - axis: RefConnectionHealthSnapshot["axes"]["coverage"] | null | string | undefined -): AxisChip { - return formatKnownAxis(COVERAGE_LABELS, axis, "unknown", "Coverage"); -} - -export function formatFreshnessAxis( - axis: RefConnectionHealthSnapshot["axes"]["freshness"] | null | string | undefined -): AxisChip { - return formatKnownAxis(FRESHNESS_LABELS, axis, "unknown", "Freshness"); -} - -export function formatOutboxAxis( - axis: RefConnectionHealthSnapshot["axes"]["outbox"] | null | string | undefined -): AxisChip { - return formatKnownAxis(OUTBOX_LABELS, axis, "unknown", "Outbox"); -} - -export function formatAttentionAxis( - axis: RefConnectionHealthSnapshot["axes"]["attention"] | null | string | undefined -): AxisChip | null { - if (axis === null) { - return null; - } - if (axis !== undefined && Object.hasOwn(ATTENTION_LABELS, axis)) { - return ATTENTION_LABELS[axis as RefConnectionHealthSnapshot["axes"]["attention"]]; - } - return { - dimension: "Attention", - label: "Attention · unknown", - title: `Unknown attention axis "${axis}" from the reference server.`, - tone: "neutral", - value: "unknown", - }; -} - -function formatKnownAxis( - labels: Record, - axis: T | null | string | undefined, - fallback: T, - labelPrefix: string -): AxisChip { - if (axis !== null && axis !== undefined && Object.hasOwn(labels, axis)) { - return labels[axis as T]; - } - const fallbackChip = labels[fallback]; - if (axis === null) { - return fallbackChip; - } - return { - ...fallbackChip, - dimension: labelPrefix, - title: `Unknown ${labelPrefix.toLowerCase()} axis "${axis}" from the reference server.`, - value: "unknown", - }; -} +// The owner-facing axis vocabulary (coverage / freshness / outbox / attention) +// lives in `packages/display/src/health/axis-vocabulary.ts`, so the console +// and the headless `sources-report` CLI render the SAME words from the same +// evidence. Import the axis formatters and `AxisChip`/`EvidenceTone` from +// `@pdpp/display` directly — this module deliberately does not re-export them. /** * Whether the outbox axis is meaningful for this connection. @@ -740,7 +497,7 @@ export function summarizeOutboxForRow( case "active": return { label: "Outbox active", tone: "neutral" }; case "unknown": - return { label: "Outbox unknown", tone: "neutral" }; + return { label: "Outbox not measured", tone: "neutral" }; case "idle": return null; default: @@ -1032,10 +789,23 @@ export function formatSourceOutboxState( return { dimension: "Outbox", label: "Outbox · backlog", title: counts, tone: "warning", value: "backlog" }; case "drained": return { dimension: "Outbox", label: "Outbox · drained", title: counts, tone: "success", value: "drained" }; + // B2: "no evidence taken" uses ONE owner-facing word everywhere. case "unknown": - return { dimension: "Outbox", label: "Outbox · unknown", title: counts, tone: "neutral", value: "unknown" }; + return { + dimension: "Outbox", + label: "Outbox · not measured", + title: counts, + tone: "neutral", + value: "not measured", + }; default: - return { dimension: "Outbox", label: "Outbox · unknown", title: counts, tone: "neutral", value: "unknown" }; + return { + dimension: "Outbox", + label: "Outbox · not measured", + title: counts, + tone: "neutral", + value: "not measured", + }; } } @@ -1264,7 +1034,10 @@ export function deriveConnectionStatusDisplay(input: { } const partial = health.axes.coverage === "gaps" || health.axes.coverage === "partial"; return { - label: partial ? "Partial" : "Degraded", + // B3: "Degraded" was jargon. This is the same rollup the server pill + // names "Missing data" (rendered-verdict.ts) — kept identical so the + // two surfaces never speak different words for one state. + label: partial ? "Partial" : "Missing data", shape: "diamond", // biome-ignore lint/suspicious/noUnnecessaryConditions: dominant is string | undefined (formatDominantCondition(health)?.title); tsc rejects removing this guard. title: dominant ?? `Useful data may exist, but coverage or freshness is incomplete${reason}.`, diff --git a/apps/console/src/app/(console)/lib/connection-modality.test.ts b/apps/console/src/app/(console)/lib/connection-modality.test.ts index 1db7d90a2..09b246065 100644 --- a/apps/console/src/app/(console)/lib/connection-modality.test.ts +++ b/apps/console/src/app/(console)/lib/connection-modality.test.ts @@ -30,10 +30,10 @@ import { const COLLECTOR_RUN_CONNECTORS_LITERAL_RE = /COLLECTOR_RUN_CONNECTORS\s*=\s*\[([^\]]*)\]/; const SURROUNDING_QUOTES_RE = /^["']|["']$/g; -test("supported local-collector set is exactly claude_code, codex, google_takeout, imessage, apple_photos, and google_messages", () => { +test("supported local-collector set is exactly claude_code, codex, google_takeout, imessage, apple_photos, google_messages, and signal", () => { assert.deepEqual( [...SUPPORTED_LOCAL_COLLECTOR_CONNECTORS], - ["claude_code", "codex", "google_takeout", "imessage", "apple_photos", "google_messages"] + ["claude_code", "codex", "google_takeout", "imessage", "apple_photos", "google_messages", "signal"] ); }); @@ -45,6 +45,7 @@ test("supported browser-collector set is derived from browser-bound production r "heb", "reddit", "usaa", + "venmo", "whoop", ]); }); @@ -84,6 +85,7 @@ test("isSupportedLocalCollectorConnector narrows only the supported keys", () => assert.equal(isSupportedLocalCollectorConnector("imessage"), true); assert.equal(isSupportedLocalCollectorConnector("apple_photos"), true); assert.equal(isSupportedLocalCollectorConnector("google_messages"), true); + assert.equal(isSupportedLocalCollectorConnector("signal"), true); assert.equal(isSupportedLocalCollectorConnector("amazon"), false); assert.equal(isSupportedLocalCollectorConnector("gmail"), false); assert.equal(isSupportedLocalCollectorConnector(""), false); diff --git a/apps/console/src/app/(console)/lib/data-source.ts b/apps/console/src/app/(console)/lib/data-source.ts index ec39bed25..ebd6f94af 100644 --- a/apps/console/src/app/(console)/lib/data-source.ts +++ b/apps/console/src/app/(console)/lib/data-source.ts @@ -133,6 +133,8 @@ export interface DashboardDataSource { includeFleetHealth?: boolean; limit?: number; profile?: undefined; + /** See `ref-client.ts`'s `listConnectorSummaries` — Sources-page-only opt-in. */ + sourcesVisibility?: boolean; }): Promise; }; /** diff --git a/apps/console/src/app/(console)/lib/fused-source-status.test.ts b/apps/console/src/app/(console)/lib/fused-source-status.test.ts new file mode 100644 index 000000000..eef942827 --- /dev/null +++ b/apps/console/src/app/(console)/lib/fused-source-status.test.ts @@ -0,0 +1,148 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The fused status line's contract, with the emphasis on the one rule that + * matters: the line must never read cheerier than its worst axis. + * + * The interesting cases are all AXIS DISAGREEMENTS — syncing-but-blocked, + * fresh-but-failing, stale-but-syncing — because agreement is trivial and + * disagreement is where the old last-writer-wins behavior fabricated green. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { fuseSourceStatus } from "./fused-source-status.ts"; +import type { SourceStatusFlag } from "./source-actionability.ts"; + +function flag(over: Partial = {}): SourceStatusFlag { + return { dot: "●", freshnessNote: null, kind: "healthy", label: "Working", tone: "success", ...over }; +} + +// Hoisted to satisfy useTopLevelRegex. +const OPENS_WITH_BLOCKED = /^Blocked/; +const ENDS_WITH_SYNCING_NOW = /Syncing now$/; +const MENTIONS_SYNCING = /Syncing/; +const MENTIONS_REFRESHING_NOW = /Refreshing now/; +const PERIOD_BEFORE_SEPARATOR = /\. ·/; + +const SYNCING_COLLAPSE: SourceStatusFlag = { + dot: "◌", + freshnessNote: null, + kind: "pending", + label: "Syncing", + tone: "muted", +}; + +test("a healthy source fuses state, freshness, and activity into one line", () => { + const fused = fuseSourceStatus(flag({ freshnessNote: "Last refreshed 2 hours ago." }), { syncing: true }); + + assert.equal(fused.line, "Working · Last refreshed 2 hours ago · Syncing now"); + assert.equal(fused.state, "Working"); + assert.equal(fused.freshness, "Last refreshed 2 hours ago"); + assert.equal(fused.syncing, true); +}); + +test("an in-flight run never hides a blocked verdict", () => { + // The defect this whole module exists to kill. Today's derivation returns the + // "Syncing" collapse here, erasing "Blocked" — the owner sees a source that + // looks busy and fine while it is actually failing. + const fused = fuseSourceStatus(SYNCING_COLLAPSE, { + syncing: true, + verdictFallback: flag({ + freshnessNote: "Last refreshed 6 days ago.", + kind: "blocked", + label: "Blocked", + tone: "destructive", + }), + }); + + assert.equal(fused.state, "Blocked", "the worst honest verdict must own the state slot"); + assert.equal(fused.tone, "destructive", "tone must follow the worst axis, not the activity"); + assert.equal(fused.line, "Blocked · Last refreshed 6 days ago · Syncing now"); + assert.match(fused.line, OPENS_WITH_BLOCKED, "the line must not open with a reassuring word"); +}); + +test("an in-flight run never hides a needs-attention verdict", () => { + const fused = fuseSourceStatus(SYNCING_COLLAPSE, { + syncing: true, + verdictFallback: flag({ kind: "degraded", label: "Needs attention", tone: "warning" }), + }); + + assert.equal(fused.state, "Needs attention"); + assert.equal(fused.tone, "warning"); + assert.equal(fused.line, "Needs attention · Syncing now"); +}); + +test("syncing survives as its own clause rather than replacing the state", () => { + // Activity is additive: the owner learns BOTH that it is broken and that + // something is being done about it right now. + const fused = fuseSourceStatus(SYNCING_COLLAPSE, { + syncing: true, + verdictFallback: flag({ kind: "blocked", label: "Blocked", tone: "destructive" }), + }); + + assert.equal(fused.syncing, true); + assert.match(fused.line, ENDS_WITH_SYNCING_NOW); + assert.notEqual(fused.state, "Syncing", "'Syncing' describes an action, never a state"); +}); + +test("a healthy verdict does not upgrade a worse rendered state", () => { + // Guards the comparison direction: the fallback wins only when it is no + // BETTER than the flag. A stale-but-green verdict must not overwrite a + // blocked lifecycle state. + const fused = fuseSourceStatus(flag({ kind: "blocked", label: "Blocked", tone: "destructive" }), { + verdictFallback: flag({ kind: "healthy", label: "Working", tone: "success" }), + }); + + assert.equal(fused.state, "Blocked"); + assert.equal(fused.tone, "destructive"); +}); + +test("a source that never refreshed says so instead of omitting freshness", () => { + const fused = fuseSourceStatus(flag({ freshnessNote: null }), { hasEverSucceeded: false }); + + assert.equal(fused.freshness, "Never updated"); + assert.equal(fused.line, "Working · Never updated"); +}); + +test("unknown freshness is omitted rather than guessed", () => { + // A source that HAS succeeded but carries no freshness annotation must not + // have one invented for it; the slot is simply absent. + const fused = fuseSourceStatus(flag({ freshnessNote: null }), { hasEverSucceeded: true }); + + assert.equal(fused.freshness, null); + assert.equal(fused.line, "Working"); +}); + +test("the server's own 'Refreshing now' annotation is not doubled up", () => { + // rendered-verdict.ts already folds activity into the freshness annotation. + // This module owns the activity slot, so that phrasing must be dropped + // rather than printed beside our own clause. + const fused = fuseSourceStatus(flag({ freshnessNote: "Refreshing now." }), { syncing: true }); + + assert.equal(fused.freshness, null); + assert.equal(fused.line, "Working · Syncing now"); + assert.doesNotMatch(fused.line, MENTIONS_REFRESHING_NOW); +}); + +test("a paused source is never shown as syncing even with a stale run flag", () => { + const fused = fuseSourceStatus(flag({ kind: "paused", label: "Paused", tone: "muted" }), { syncing: true }); + + assert.equal(fused.syncing, false); + assert.equal(fused.line, "Paused"); +}); + +test("a revoked source is never shown as syncing", () => { + const fused = fuseSourceStatus(flag({ kind: "revoked", label: "Revoked", tone: "muted" }), { syncing: true }); + + assert.equal(fused.syncing, false); + assert.doesNotMatch(fused.line, MENTIONS_SYNCING); +}); + +test("freshness punctuation is normalized so the separator reads cleanly", () => { + const fused = fuseSourceStatus(flag({ freshnessNote: " Last refreshed 3 days ago. " }), {}); + + assert.equal(fused.freshness, "Last refreshed 3 days ago"); + assert.doesNotMatch(fused.line, PERIOD_BEFORE_SEPARATOR); +}); diff --git a/apps/console/src/app/(console)/lib/fused-source-status.ts b/apps/console/src/app/(console)/lib/fused-source-status.ts new file mode 100644 index 000000000..cc7c40d18 --- /dev/null +++ b/apps/console/src/app/(console)/lib/fused-source-status.ts @@ -0,0 +1,167 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The fused "what it is / when it last updated / whether it's syncing" status + * line for one source. + * + * Owner-raised 2026-08-19: "why is there no product-standard fused + * `Last updated X / Syncing now` status?" The design problem — and why the + * obvious one-line fix is wrong — is written up in + * `design-notes/fused-source-status-2026-08-22.md`. The short version: + * + * Freshness, activity, and the health verdict are THREE INDEPENDENT AXES that + * routinely disagree. A source can be syncing right now and still be broken; it + * can be fresh and have a failing run in flight; it can be stale precisely + * because the sync that would refresh it keeps failing. Today's code resolves + * that disagreement by LAST-WRITER-WINS: `deriveRenderedSourceStatus` returns + * early on `running` with `label: "Syncing"`, `tone: "muted"`, and + * `freshnessNote: null` (`source-actionability.ts`), so an in-flight run erases + * both the freshness note and whatever the verdict said was wrong. That is the + * fabricated-green defect in miniature: the most reassuring axis silently wins. + * + * This module fuses instead of overwriting, under one rule: + * + * ACTIVITY IS ADDITIVE, NEVER SUBSTITUTIVE. + * + * "Syncing" is something a source is DOING, not something it IS. So the state + * slot always keeps the worst honest verdict, and syncing is appended as + * context. The fused line can never read cheerier than its worst axis. + * + * This module deliberately owns no label VALUES that `connection-evidence.ts` + * already owns — it composes the strings that module and + * `deriveRenderedSourceStatus` produce. + */ + +import type { SourceStatusFlag, SourceStatusKind, SourceStatusTone } from "./source-actionability.ts"; + +/** + * How much confidence the fused line is entitled to. Ordered worst-to-best so + * the worst axis can be selected by comparison rather than by branch order. + */ +const SEVERITY_BY_KIND: Record = { + archived: 0, + setup_failed: 0, + blocked: 0, + degraded: 1, + unknown: 2, + revoked: 3, + paused: 4, + pending: 5, + healthy: 6, +}; + +export interface FusedSourceStatus { + /** The freshness slot alone, or null when freshness is genuinely unknown. */ + freshness: string | null; + kind: SourceStatusKind; + /** The whole line, e.g. `"Needs attention · Last refreshed 3 days ago · Syncing now"`. */ + line: string; + /** The state slot alone — the worst honest verdict, never "Syncing". */ + state: string; + /** True when a run is in flight. Drives the animated dot, never the wording of `state`. */ + syncing: boolean; + tone: SourceStatusTone; +} + +/** The activity clause. Additive context, never the state itself. */ +const SYNCING_CLAUSE = "Syncing now"; + +/** Trailing sentence period on a server freshness annotation. */ +const TRAILING_PERIOD = /\.$/; +/** The server's activity-flavored freshness annotation; this module owns activity. */ +const SERVER_REFRESHING_NOW = /^refreshing now$/i; + +/** + * Freshness copy for a source that has never produced a successful refresh. + * "Never" is a real, honest answer; omitting the slot would let the line read + * as though freshness simply wasn't applicable. + */ +const NEVER_UPDATED = "Never updated"; + +/** + * Trims a server freshness annotation into the fused line's slot. + * + * The server's own annotation already fuses activity in some cases — + * `rendered-verdict.ts` returns "Refreshing now." when `badges.syncing` — which + * would double up with our activity clause. Fusing is this module's job, so the + * activity-flavored annotation is dropped here and re-added from the actual + * `syncing` flag, keeping one source of truth for the activity slot. + */ +function freshnessSlot(note: string | null, hasEverSucceeded: boolean): string | null { + if (note === null) { + return hasEverSucceeded ? null : NEVER_UPDATED; + } + const trimmed = note.trim().replace(TRAILING_PERIOD, ""); + if (trimmed === "") { + return hasEverSucceeded ? null : NEVER_UPDATED; + } + // The server's activity-flavored freshness annotation; our own clause covers it. + if (SERVER_REFRESHING_NOW.test(trimmed)) { + return null; + } + return trimmed; +} + +/** + * Picks the state slot. `running` must never overwrite a worse verdict, so when + * a source is both syncing and unhealthy the unhealthy label wins the slot and + * syncing moves to its own clause. + * + * `flag` is what `deriveRenderedSourceStatus` produced. When it already + * collapsed to "Syncing" (its `running` early-return), `verdictFallback` + * carries the verdict label that collapse discarded — that is the honest state, + * and it is used whenever it is no better than what the flag reported. + */ +function stateSlot(flag: SourceStatusFlag, verdictFallback: SourceStatusFlag | null): SourceStatusFlag { + if (!verdictFallback) { + return flag; + } + return SEVERITY_BY_KIND[verdictFallback.kind] <= SEVERITY_BY_KIND[flag.kind] ? verdictFallback : flag; +} + +/** + * Composes the fused status line. + * + * @param flag The status as rendered today (may already be the "Syncing" collapse). + * @param options.syncing Whether a run is actually in flight. + * @param options.verdictFallback The verdict-derived status that the "Syncing" + * collapse discarded, when there was one. Supplying it is what lets a failing + * source keep saying it is failing while it syncs. + * @param options.hasEverSucceeded Whether any successful refresh exists, so a + * missing freshness note can be reported as "Never updated" rather than omitted. + */ +export function fuseSourceStatus( + flag: SourceStatusFlag, + options: { + hasEverSucceeded?: boolean; + syncing?: boolean; + verdictFallback?: SourceStatusFlag | null; + } = {} +): FusedSourceStatus { + const syncing = options.syncing ?? false; + const hasEverSucceeded = options.hasEverSucceeded ?? true; + const state = stateSlot(flag, options.verdictFallback ?? null); + + // Freshness is taken from whichever slot actually carries it: the "Syncing" + // collapse nulls its own note, so the recovered verdict is the only place it + // survives. + const freshness = freshnessSlot(state.freshnessNote ?? flag.freshnessNote, hasEverSucceeded); + + // A paused or revoked source is not syncing in any owner-meaningful sense, + // and a stale in-flight run flag must not make it look like it is. Those + // states already rank ahead of `running` in `deriveRenderedSourceStatus`; + // this keeps the fused line consistent with that ranking. + const showSyncing = syncing && state.kind !== "paused" && state.kind !== "revoked"; + + const line = [state.label, freshness, showSyncing ? SYNCING_CLAUSE : null].filter(Boolean).join(" · "); + + return { + freshness, + kind: state.kind, + line, + state: state.label, + syncing: showSyncing, + tone: state.tone, + }; +} diff --git a/apps/console/src/app/(console)/lib/operator-runs.ts b/apps/console/src/app/(console)/lib/operator-runs.ts index 09bfc0cc0..6a9debe4c 100644 --- a/apps/console/src/app/(console)/lib/operator-runs.ts +++ b/apps/console/src/app/(console)/lib/operator-runs.ts @@ -4,11 +4,15 @@ import { type CancelRunResult, cancelRunErrorCode, classifyCancelRunResponse } from "./cancel-run-result.ts"; import { classifyDeleteConnectionResponse, + classifyPauseConnectionResponse, classifyReactivateConnectionResponse, + classifyResumeConnectionResponse, classifyRevokeConnectionResponse, connectionControlErrorCode, type DeleteConnectionResult, + type PauseConnectionResult, type ReactivateConnectionResult, + type ResumeConnectionResult, type RevokeConnectionResult, } from "./connection-control-result.ts"; import { describeError } from "./describe-error.ts"; @@ -18,8 +22,12 @@ export type { CancelRunOutcome, CancelRunResult } from "./cancel-run-result.ts"; export type { DeleteConnectionOutcome, DeleteConnectionResult, + PauseConnectionOutcome, + PauseConnectionResult, ReactivateConnectionOutcome, ReactivateConnectionResult, + ResumeConnectionOutcome, + ResumeConnectionResult, RevokeConnectionOutcome, RevokeConnectionResult, } from "./connection-control-result.ts"; @@ -458,6 +466,39 @@ export async function reactivateConnection(connectionId: string): Promise { + const response = await fetchAs(connectionControlPath(connectionId, "/pause"), { + method: "POST", + }); + const body = await readBody(response); + return classifyPauseConnectionResponse(response.status, body, connectionControlErrorCode(body)); +} + +/** + * Owner-resume one paused connection via the owner-session + * `POST /_ref/connections/:id/resume` route. The inverse of + * {@link pauseConnection}: flips the connection back to `active` so scheduled + * and manual runs land again. Zero cascade; credential freshness is delegated + * to the next collection run. Returns a typed outcome (`not_paused` if the + * connection was already active) so the console can message in place. + */ +export async function resumeConnection(connectionId: string): Promise { + const response = await fetchAs(connectionControlPath(connectionId, "/resume"), { + method: "POST", + }); + const body = await readBody(response); + return classifyResumeConnectionResponse(response.status, body, connectionControlErrorCode(body)); +} + /** * Owner-delete one configured connection via the owner-session * `DELETE /_ref/connections/:id` route. Erases exactly that connection's diff --git a/apps/console/src/app/(console)/lib/ref-client-verdict-label-parity.test.ts b/apps/console/src/app/(console)/lib/ref-client-verdict-label-parity.test.ts new file mode 100644 index 000000000..2cb8283f7 --- /dev/null +++ b/apps/console/src/app/(console)/lib/ref-client-verdict-label-parity.test.ts @@ -0,0 +1,53 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Owner ledger 2026-08-22, item flagged by the quiet-setup-expiry lane: + * `RefVerdictPill.label` in `ref-client.ts` is a hand-maintained mirror of + * the reference server's `VerdictLabel` union (`runtime/rendered-verdict.ts`) + * — the console app cannot import server runtime code directly (see + * ref-client-pagination.test.ts's header for why `ref-client.ts` cannot be + * imported by node:test either). The server legitimately emits "Archived" + * (`ref-control.ts:6619`) and "Setup never completed" (`ref-control.ts:6629`) + * pills, but the console mirror's union omitted both, which would make a + * strictly-typed caller narrow those two real server values to nothing. + * + * This test pins the mirror at the source-text level so the two unions + * cannot silently re-diverge, matching ref-client-pagination.test.ts's + * established source-scanning pattern for this same file. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const REF_CLIENT_FILE = `${HERE}ref-client.ts`; + +const REQUIRED_PILL_LABELS = [ + "Archived", + "Can't collect", + "Checking", + "Healthy", + "Import complete", + "Missing data", + "Needs refresh", + "Not measured", + "Setup never completed", + "Syncing", +]; + +test("RefVerdictPill.label carries every terminal label the reference server emits", async () => { + const source = await readFile(REF_CLIENT_FILE, "utf8"); + const interfaceMatch = source.match(/export interface RefVerdictPill \{[\s\S]*?\n\}/); + assert.ok(interfaceMatch, "RefVerdictPill interface must exist in ref-client.ts"); + const interfaceBody = interfaceMatch[0]; + for (const label of REQUIRED_PILL_LABELS) { + assert.ok( + interfaceBody.includes(`"${label}"`), + `RefVerdictPill.label is missing "${label}", which the reference server emits (ref-control.ts). ` + + "A caller that switches on this union would silently fail to narrow a real server value." + ); + } +}); diff --git a/apps/console/src/app/(console)/lib/ref-client.ts b/apps/console/src/app/(console)/lib/ref-client.ts index f1bc6e0a9..9278bf80e 100644 --- a/apps/console/src/app/(console)/lib/ref-client.ts +++ b/apps/console/src/app/(console)/lib/ref-client.ts @@ -688,6 +688,29 @@ export interface RefConnectorSummary { * field omits it and the console falls back to connector-level modality. */ source_kind?: string; + /** + * Provider-neutral Sources-list visibility (mirrors server `ConnectorSummary + * .source_visibility`). `"archived"` marks a PURE recovered historical + * fragment — preserved records, no collection, never resuming — and never a + * UAT-transferred/manual-import row or an active promoted connection, both + * of which read `"active"`. + * + * `"hidden_from_sources"` is the retired spelling of `"archived"`, accepted + * so a console deployed ahead of its reference keeps classifying those rows + * as archived instead of silently rendering them as live sources. + * + * `"setup_failed"` marks a revoked retired-setup-shell binding + * (`browser_enrollment_shell`/etc.) that never had a successful run — + * repeated failed setup, zero records, the owner's only evidence a + * connector was ever attempted. A shell whose run DID succeed is not + * classified this way at all; it is excluded from the Sources page + * entirely, because a promoted `"active"` row already represents it. + * + * Optional: a reference predating this field omits it, in which case the + * row is treated as `"active"` (fails open to visible, the pre-existing + * behavior) rather than hiding rows an older reference never classified. + */ + source_visibility?: "active" | "archived" | "hidden_from_sources" | "setup_failed" | null; /** * Server-owned work classification derived from `owner_state.resolver`. * Optional only for references predating this field; the console fails closed @@ -867,7 +890,17 @@ export type RefVerdictTone = "amber" | "green" | "grey" | "red"; export type RefRenderedChannel = "advisory" | "attention" | "calm"; export interface RefVerdictPill { - label: "Can't collect" | "Checking" | "Degraded" | "Healthy" | "Needs refresh" | "Not measured" | "Syncing"; + label: + | "Archived" + | "Can't collect" + | "Checking" + | "Healthy" + | "Import complete" + | "Missing data" + | "Needs refresh" + | "Not measured" + | "Setup never completed" + | "Syncing"; tone: RefVerdictTone; } @@ -1723,6 +1756,7 @@ export function listConnectorSummaries(options?: { includeFleetHealth?: boolean; limit?: number; profile?: undefined; + sourcesVisibility?: boolean; }): Promise; export async function listConnectorSummaries( options: { @@ -1738,6 +1772,15 @@ export async function listConnectorSummaries( includeFleetHealth?: boolean; limit?: number; profile?: ConnectorSummaryProfile; + /** + * Owner Sources page's exclusive opt-in (`sources_visibility=1`): + * excludes a pure recovered historical fragment from this identity page + * BEFORE the reference's `LIMIT`, so `has_more`/the next cursor stay + * authoritative over the rows the Sources list actually renders. Every + * other caller (Explore, Add Source, manual upload) omits this. Mutually + * exclusive with `connectorId`/`profile` server-side. + */ + sourcesVisibility?: boolean; } = {} ): Promise< RefConnectorSummariesResponse | RefConnectorIdentitySummariesResponse | RefConnectorRetainedCountSummariesResponse @@ -1761,6 +1804,7 @@ export async function listConnectorSummaries( include_fleet_health: options.includeFleetHealth ? 1 : undefined, limit: options.limit ?? CONNECTOR_SUMMARY_DEFAULT_PAGE_LIMIT, profile: options.profile, + sources_visibility: options.sourcesVisibility ? 1 : undefined, })) as RefConnectorSummariesResponse; } diff --git a/apps/console/src/app/(console)/lib/rs-client-route-agreement.test.ts b/apps/console/src/app/(console)/lib/rs-client-route-agreement.test.ts new file mode 100644 index 000000000..48ce41f08 --- /dev/null +++ b/apps/console/src/app/(console)/lib/rs-client-route-agreement.test.ts @@ -0,0 +1,66 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Pins the literal route strings the console calls against the literal + * strings the reference server registers them under, so a rename on either + * side fails a test instead of surfacing as a live 404. + * + * `rs-client.ts` and `operator-runs.ts` import `server-only` transitively, so + * their functions cannot execute in a plain `node:test` process (same + * constraint documented in `ref-client-pagination.test.ts`). These tests pin + * the source-level contract instead: read both the caller and the route + * registration as text and assert the same literal path appears in each. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +const CONNECTOR_TEMPLATES_PATH = "/v1/owner/connector-templates"; +const RUN_INTERACTION_STREAM_MINT_PATH = "/_ref/runs/:runId/run-interaction-stream"; + +test("listOwnerConnectorTemplates calls the path owner-connector-templates.ts registers", async () => { + const clientSource = await readFile(new URL("./rs-client.ts", import.meta.url), "utf8"); + assert.match( + clientSource, + /authedFetch\("\/v1\/owner\/connector-templates"\)/, + "rs-client.ts must call the literal /v1/owner/connector-templates path" + ); + + const routeSource = await readFile( + new URL( + "../../../../../../reference-implementation/server/routes/owner-connector-templates.ts", + import.meta.url + ), + "utf8" + ); + assert.match( + routeSource, + /app\.get\(\s*"\/v1\/owner\/connector-templates"/, + "owner-connector-templates.ts must register the literal /v1/owner/connector-templates path" + ); + + assert.ok(clientSource.includes(CONNECTOR_TEMPLATES_PATH) && routeSource.includes(CONNECTOR_TEMPLATES_PATH)); +}); + +test("mintRunInteractionStream calls the path streaming/routes.ts registers", async () => { + const clientSource = await readFile(new URL("./operator-runs.ts", import.meta.url), "utf8"); + assert.match( + clientSource, + /fetchAs\(`\/_ref\/runs\/\$\{encodeURIComponent\(runId\)\}\/run-interaction-stream`, \{\s*\n\s*body: asJson\(payload\)/, + "operator-runs.ts must POST to the literal /_ref/runs/:runId/run-interaction-stream template" + ); + + const routeSource = await readFile( + new URL("../../../../../../reference-implementation/server/streaming/routes.ts", import.meta.url), + "utf8" + ); + assert.match( + routeSource, + /app\.post\("\/_ref\/runs\/:runId\/run-interaction-stream",/, + "streaming/routes.ts must register POST /_ref/runs/:runId/run-interaction-stream" + ); + + assert.ok(routeSource.includes(RUN_INTERACTION_STREAM_MINT_PATH)); +}); diff --git a/apps/console/src/app/(console)/lib/rs-client.ts b/apps/console/src/app/(console)/lib/rs-client.ts index 3b5da5fe3..114c199d6 100644 --- a/apps/console/src/app/(console)/lib/rs-client.ts +++ b/apps/console/src/app/(console)/lib/rs-client.ts @@ -34,6 +34,7 @@ import type { RefRetainedBytesBreakdown, } from "./ref-client.ts"; import { refFetch } from "./ref-client.ts"; +import { aggregateStreamRecordCounts } from "./stream-record-count-aggregate.ts"; import { verifyDashboardSession } from "./verify-session.ts"; export interface StreamSummary { @@ -1446,7 +1447,7 @@ function projectRun( export async function getConnectorOverview(connector: ConnectorManifest): Promise { try { const streams = await listStreams(connector.connector_id); - const totalRecords = streams.reduce((sum, s) => sum + (s.record_count ?? 0), 0); + const { totalRecords, totalRecordsState } = aggregateStreamRecordCounts(streams); // Run data: most-recent run (any status) + most-recent succeeded. // Kept lazy-import to avoid a cycle: ref-client imports from owner-token @@ -1467,6 +1468,7 @@ export async function getConnectorOverview(connector: ConnectorManifest): Promis lastSuccessfulRun, streams, totalRecords, + totalRecordsState, }; } catch (err) { if (err instanceof ReferenceServerUnreachableError) { @@ -1480,6 +1482,10 @@ export async function getConnectorOverview(connector: ConnectorManifest): Promis lastSuccessfulRun: null, streams: [], totalRecords: 0, + // The read FAILED, so nothing is known about how much this connection + // holds. `resolveRecordCountDisplay` short-circuits on `error` first, but + // the state must not claim a measured zero for any other consumer. + totalRecordsState: "unobserved", }; } } diff --git a/apps/console/src/app/(console)/lib/source-actionability.test.ts b/apps/console/src/app/(console)/lib/source-actionability.test.ts index 041d2547c..e75c99a78 100644 --- a/apps/console/src/app/(console)/lib/source-actionability.test.ts +++ b/apps/console/src/app/(console)/lib/source-actionability.test.ts @@ -17,6 +17,7 @@ import { primaryOwnerActionRemediation, primaryRequiredAction, projectSourceActionability, + RESUME_PAUSED_CTA_LABEL, SETUP_IN_PROGRESS_CTA_LABEL, SOURCE_WORK_GROUP_COPY, sourceAttentionHeadline, @@ -186,14 +187,14 @@ test("source actionability ignores non-owner local-device remediation for owner- required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", remediation: { cause: "stalled_unknown", commands: [], kind: "local_collector_recovery", label: "Fix connector code", - summary: "Connector code needs a fix before owner recovery can proceed.", + summary: "A maintainer must repair the collector before owner recovery can proceed.", target: { identity_source: "source_instance_bindings", kind: "local_device" }, }, satisfied_when: { kind: "none" }, @@ -214,11 +215,11 @@ test("source actionability does not convert maintainer-primary work into owner w source_work: "system_issue", rendered_verdict: verdict({ channel: "attention", - forward_statement: "Connector code needs a fix before this can collect again.", + forward_statement: "Some data from this source can't be collected.", required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -312,12 +313,12 @@ test("source actionability keeps a Degraded pill (no wired owner action) in syst source_work: "system_issue", rendered_verdict: verdict({ channel: "advisory", - forward_statement: "Connector code needs a fix before this can collect again.", - pill: { label: "Degraded", tone: "amber" }, + forward_statement: "Some data from this source can't be collected.", + pill: { label: "Missing data", tone: "amber" }, required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -377,7 +378,7 @@ test("source actionability resolves per-stream owner action availability from ac action({ cta: "Retry now", kind: "retry_gap", satisfied_when: { kind: "gap_recovered" } }), action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -492,6 +493,69 @@ test("source actionability: revoked outranks draft — a revoked connection neve assert.equal(actionability.work, null); }); +// A PURE recovered historical fragment — production shape (2026-08-18): a +// spine-events-only reconstruction of an owner-deleted connection, restored +// under a synthetic `restored-historical-archive:` binding key with no +// credential ever captured for the resurrected identity. The server already +// marks this `source_visibility: "hidden_from_sources"` (see `ref-control.ts` +// `deriveSourceVisibility`) and the Sources list already honors it +// (`sources-view-model.ts` `isVisibleOnSourcesList`), but `source-work` derivation +// never consulted the field, so the SAME fragment still landed in the +// needs-you group with "Reconnect this account and collection resumes" — a +// prompt that is false for a connection the owner already deleted and does +// not intend to reconnect. `/syncs` and the dashboard "Needs you" section +// both read `sourceWorkFromConnectors`, so this defect was owner-visible on +// both surfaces. +test("source actionability excludes a hidden_from_sources pure recovered fragment from every work group", () => { + const fragment = connector({ + connection_id: "cin_e4ab231c7d49b8f59e4c80ed", + connector_id: "chatgpt", + display_name: "ChatGPT (historical archive 2 of 2)", + rendered_verdict: verdict({ + forward_statement: "Reconnect this account and collection resumes.", + pill: { label: "Can't collect", tone: "red" }, + required_actions: [action({ cta: "Reconnect this account", kind: "reauth" })], + }), + source_visibility: "hidden_from_sources", + source_work: "needs_owner", + status: "paused", + }); + + const actionability = projectSourceActionability(fragment); + assert.equal(actionability.work, null, "a hidden_from_sources fragment must never produce a work item"); + + const groups = sourceWorkFromConnectors([fragment]); + assert.equal(groups.needsOwner.length, 0); + assert.equal(groups.review.length, 0); + assert.equal(groups.systemIssues.length, 0); + assert.equal(groups.notMeasured.length, 0); + assert.equal(groups.working.length, 0); + assert.equal(groups.unavailable.length, 0); + assert.equal(sourceAttentionHeadline(groups).needsYou, 0); +}); + +// A normal `"active"` visibility connection with the identical needs_owner +// verdict shape must be unaffected — this guards against the fix +// over-suppressing every credential-required connection instead of only the +// hidden fragment. +test("source actionability still surfaces a visible needs_owner connection with the same verdict shape", () => { + const groups = sourceWorkFromConnectors([ + connector({ + connection_id: "cin_live_needs_owner", + rendered_verdict: verdict({ + forward_statement: "Reconnect this account and collection resumes.", + pill: { label: "Can't collect", tone: "red" }, + required_actions: [action({ cta: "Reconnect this account", kind: "reauth" })], + }), + source_visibility: "active", + source_work: "needs_owner", + }), + ]); + + assert.equal(groups.needsOwner.length, 1); + assert.equal(sourceAttentionHeadline(groups).needsYou, 1); +}); + test("source actionability: a non-draft connection with real verdict evidence is never treated as setup_in_progress", () => { const actionability = projectSourceActionability(connector()); @@ -558,12 +622,12 @@ test("source actionability headline counts only needs-owner work and exposes sta source_work: "system_issue", rendered_verdict: verdict({ channel: "advisory", - forward_statement: "Connector code needs a fix before this can collect again.", - pill: { label: "Degraded", tone: "amber" }, + forward_statement: "Some data from this source can't be collected.", + pill: { label: "Missing data", tone: "amber" }, required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -666,7 +730,7 @@ test("Sources grouping follows server source_work for degraded wait, passive coo }, rendered_verdict: verdict({ channel: "calm", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [], }), }), @@ -767,6 +831,7 @@ test("source actionability groups a Needs refresh connection under review, never // ─── Recovery-state grouping (connector-neutral recovery governor UI tranche) ── const RECOVERY_CHECKING_RE = /checking/i; +const RESUME_COPY_RE = /resume/i; const SOURCE_DETAILS_RE = /Open source details/; const RECOVERY_SYNCING_RE = /syncing details/i; const RECOVERY_CATCHING_UP_RE = /catching up/i; @@ -790,7 +855,7 @@ function deferredRecoveryVerdict(overrides: Partial = {}): R return verdict({ channel: "calm", forward_statement: "The next run is expected to fill the remaining data.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, progress: { gaps_drained_last_run: null, headline: "Collecting in the background.", @@ -919,7 +984,7 @@ test("recovery grouping: a connector-defect verdict with recoverable gaps stays required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -963,3 +1028,56 @@ test("recovery grouping: an inactive backlog routes to NAMED recovery before the assert.doesNotMatch(row.what, RECOVERY_CHECKING_RE); assert.match(`${row.statusLabel} ${row.what}`, RECOVERY_CATCHING_UP_RE); }); + +// --- Paused: a first-class, owner-reversible lifecycle state ---------------- +// Pause stops collection without giving anything up. These pin the two +// properties that make it usable: the owner can SEE it, and can get back out. + +test("a paused source renders the Paused status rather than its pre-pause verdict", () => { + const actionability = projectSourceActionability( + connector({ + // A healthy-looking verdict from before the pause must not win: it + // describes collection that has stopped. + rendered_verdict: verdict({ pill: { label: "Healthy", tone: "green" } }), + status: "paused", + }) + ); + + assert.equal(actionability.paused, true); + assert.equal(actionability.renderedStatus.kind, "paused"); + assert.equal(actionability.renderedStatus.label, "Paused"); + assert.equal(actionability.renderedStatus.tone, "muted"); +}); + +test("a paused source outranks an in-flight run flag and never renders as Syncing", () => { + const flag = deriveRenderedSourceStatus(verdict(), false, false, null, true, true); + + assert.equal(flag.kind, "paused", "paused must outrank running"); + assert.equal(flag.label, "Paused"); +}); + +test("a revoked source outranks paused, so the durable state always wins", () => { + const actionability = projectSourceActionability(connector({ revoked_at: "2026-08-01T00:00:00Z", status: "paused" })); + + assert.equal(actionability.revoked, true); + assert.equal(actionability.paused, false, "revoked must suppress paused, never both at once"); + assert.equal(actionability.renderedStatus.kind, "revoked"); +}); + +test("a paused source offers Resume as an available action, not as owner-blocking work", () => { + const summary = connector({ source_work: "needs_owner", status: "paused" }); + const groups = sourceWorkFromConnectors([summary]); + + // Visible, so collection is never silently stopped with no way back... + assert.equal(groups.review.length, 1); + const [row] = groups.review; + assert.ok(row); + assert.equal(row.actionLabel, RESUME_PAUSED_CTA_LABEL); + assert.equal(row.statusLabel, "is paused"); + assert.match(row.what, RESUME_COPY_RE); + // ...but never counted against the "needs you" headline: the owner chose + // this state and is not blocking anything. Note the server said + // `needs_owner` here; the lifecycle check must still win. + assert.equal(groups.needsOwner.length, 0); + assert.equal(sourceAttentionHeadline(groups).needsYou, 0); +}); diff --git a/apps/console/src/app/(console)/lib/source-actionability.ts b/apps/console/src/app/(console)/lib/source-actionability.ts index dfb461e87..d1eadc5ff 100644 --- a/apps/console/src/app/(console)/lib/source-actionability.ts +++ b/apps/console/src/app/(console)/lib/source-actionability.ts @@ -3,6 +3,7 @@ import { deriveFailureSummary, type FailureSummary } from "./connection-evidence.ts"; import { isActiveConnectorRunSummaryStatus } from "./connector-run-summary-status.ts"; +import { type FusedSourceStatus, fuseSourceStatus } from "./fused-source-status.ts"; import type { FormattedNextAction } from "./next-action.ts"; import type { RefActionRemediation, @@ -16,7 +17,16 @@ import type { export type SourceWorkGroupId = "needsOwner" | "notMeasured" | "review" | "systemIssue" | "unavailable" | "working"; -export type SourceStatusKind = "blocked" | "degraded" | "healthy" | "pending" | "revoked" | "unknown"; +export type SourceStatusKind = + | "archived" + | "blocked" + | "degraded" + | "healthy" + | "paused" + | "pending" + | "revoked" + | "setup_failed" + | "unknown"; export type SourceStatusTone = "destructive" | "muted" | "success" | "warning"; @@ -67,10 +77,17 @@ export interface SourceWorkGroups { export interface SourceActionabilityProjection { failureSummary: FailureSummary | null; + /** + * The single owner-facing status line: state, freshness, and activity fused + * under the worst-honest-axis rule. See `fused-source-status.ts`. + */ + fusedStatus: FusedSourceStatus; label: string; nextAction: FormattedNextAction | null; ownerActionByStream: SourceStreamOwnerActionAvailability; ownerActionCue: SourceOwnerActionCue | null; + /** Collection is stopped but fully reversible. Never true when `revoked`. */ + paused: boolean; primaryAction: RefRequiredAction | null; primaryVerdictAction: SourcePrimaryVerdictAction | null; renderedStatus: SourceStatusFlag; @@ -220,6 +237,24 @@ export function isRevokedConnector(connector: RefConnectorSummary): boolean { return connector.status === "revoked" || Boolean(connector.revoked_at); } +/** + * A `paused` connection: collection is stopped, but nothing was given up — + * records, grants, schedule, and the stored credential all survive, and the + * owner can resume from the same detail page they paused on. Like + * {@link isRevokedConnector} this is a LIFECYCLE check independent of the + * verdict, because `rendered_verdict` carries no lifecycle concept: a paused + * row's health/coverage evidence describes the collection that stopped, and + * rendering that as the source's status would tell the owner about a state + * the connection is no longer in. + * + * Deliberately does NOT treat a revoked row as paused — `isRevokedConnector` + * is checked first everywhere the two meet, so the durable state always wins + * over the reversible one. + */ +export function isPausedConnector(connector: RefConnectorSummary): boolean { + return connector.status === "paused"; +} + /** * A `draft` connection has completed neither its credential capture nor its * first ingest — `rendered_verdict`/`connection_health` carry no lifecycle @@ -272,6 +307,34 @@ function freshnessNoteFromVerdict(verdict: RefRenderedVerdict): string | null { return verdict.annotations.find((annotation) => annotation.kind === "freshness")?.text ?? null; } +/** + * The status a verdict alone implies, ignoring lifecycle and activity. + * + * `deriveRenderedSourceStatus` returns early on `running` (and on + * paused/revoked/pending) and throws the verdict away. That is right for the + * single-slot dot, but it means an in-flight run hides a "Needs attention" or + * "Blocked" verdict entirely. The fused status line needs the discarded + * verdict back so activity can be shown ALONGSIDE the real state instead of + * replacing it — see `fuseSourceStatus`. + * + * Returns null when there is no verdict to recover, in which case the caller + * should keep whatever `deriveRenderedSourceStatus` decided. + */ +export function deriveSourceVerdictStatus(verdict: RefRenderedVerdict | null | undefined): SourceStatusFlag | null { + if (!verdict) { + return null; + } + const status = VERDICT_TONE_STATUS[verdict.pill.tone]; + // `label` is the BARE pill label here, unlike `deriveRenderedSourceStatus`, + // which concatenates the freshness note into it. The fused line renders + // freshness in its own slot, so pre-concatenating would print it twice. + return { + ...status, + freshnessNote: freshnessNoteFromVerdict(verdict), + label: verdict.pill.label, + }; +} + function labelWithFreshness(base: string, note: string | null): string { return note ? `${base} · ${note}` : base; } @@ -281,14 +344,48 @@ export function deriveRenderedSourceStatus( revoked: boolean, pending = false, terminalSetupDisposition: RefTerminalSetupDisposition | null = null, - running = false + running = false, + paused = false, + archived = false, + setupFailed = false ): SourceStatusFlag { + // Ranked FIRST, ahead of every verdict-derived tone. An archived source's + // last run may well have succeeded, so its stored verdict can still be + // green — rendering that would tell the owner a source that will never + // collect again is healthy, which is exactly the fabricated-green defect + // class. Archived is terminal and muted: the records are real, the + // collection is over, and no tone implies otherwise. + if (archived) { + return { dot: "⊘", freshnessNote: null, kind: "archived", label: "Archived · not collecting", tone: "muted" }; + } + // Ranked alongside archived, ahead of plain `revoked`: a setup-failed + // source is a MORE SPECIFIC terminal state than "Revoked" — it says the + // connection never worked in the first place, not that a working one was + // taken away. Muted, never a warning/destructive tone: the owner already + // knows this attempt did not finish (server-side `archiveRenderedVerdict` + // built this label), so there is nothing new to flag as a problem here. + if (setupFailed) { + return { dot: "⊘", freshnessNote: null, kind: "setup_failed", label: "Setup never completed", tone: "muted" }; + } if (revoked) { return { dot: "⊘", freshnessNote: null, kind: "revoked", label: "Revoked", tone: "muted" }; } + // Ranked directly after `revoked` and ahead of `running`/`pending`: a paused + // connection is not collecting, so a stale in-flight run flag or a verdict + // tone must never render it as "Syncing" or as a health colour. Muted (not + // a warning tone) because pause is a state the owner chose, not a problem + // to fix — the way back is an action, which the detail page offers. + if (paused) { + return { dot: "⏸", freshnessNote: null, kind: "paused", label: "Paused", tone: "muted" }; + } if (running) { return { dot: "◌", freshnessNote: null, kind: "pending", label: "Syncing", tone: "muted" }; } + // NOTE: the `running` collapse above intentionally discards the verdict, so a + // caller that wants the fused status line must recover it via + // `deriveSourceVerdictStatus` and pass it to `fuseSourceStatus` as the + // fallback. See `fused-source-status.ts` for why activity must not overwrite + // a worse verdict. if (terminalSetupDisposition) { return { dot: "◐", @@ -480,11 +577,80 @@ function itemFromConnector( }; } +/** + * An ARCHIVED source: preserved records, no collection, never resuming. + * Server-derived in `deriveSourceVisibility` (`ref-control.ts`) as + * `source_visibility: "archived"`. + * + * `"hidden_from_sources"` is the retired spelling, still accepted so a + * console deployed ahead of its reference keeps treating those rows as + * archived rather than as live sources — failing toward the safe reading, + * since the dangerous error is showing a dead source as collecting. + * + * Such a source must never generate owner-facing work. Its ONLY durable + * content is records from past runs; it has no schedule, no stored + * credential, and the owner has already acted on it (by deleting the + * connection it came from). A "Reconnect this account and collection + * resumes" prompt built from its `CredentialsValid: false` condition is + * technically correct (no credential exists) but not actionable in the way + * the copy implies — reconnecting does not "resume" anything, because + * nothing here was ever a live collection the owner intends to continue. + * + * The Sources list now RENDERS these rows (in a distinct Archived group) + * rather than dropping them, but the no-work rule is unchanged and applies + * to every owner-facing work surface (`/syncs`, the dashboard "Needs you" + * section) that reads `sourceWorkFromConnectors`. + */ +export function isArchivedSource(connector: RefConnectorSummary): boolean { + return connector.source_visibility === "archived" || connector.source_visibility === "hidden_from_sources"; +} + +/** + * A SETUP-FAILED source: a revoked retired-setup-shell binding + * (`browser_enrollment_shell`/etc.) that never had a successful run. + * Server-derived in `deriveSourceVisibility` (`ref-control.ts`) as + * `source_visibility: "setup_failed"`. + * + * Holds zero records by construction — no run against this shell ever + * emitted any. Distinct from {@link isArchivedSource}: an archived source + * once collected and is now terminal; a setup-failed source never collected + * at all. It must never generate owner-facing work on `/syncs`/the + * dashboard for the same reason an archived source does not — there is no + * live connection behind it, only spent setup mechanics. + */ +export function isSetupFailedSource(connector: RefConnectorSummary): boolean { + return connector.source_visibility === "setup_failed"; +} + +/** The one owner-facing CTA label for resuming a paused connection. */ +export const RESUME_PAUSED_CTA_LABEL = "Resume"; + export function sourceWorkItemFromConnector(connector: RefConnectorSummary): SourceWorkItem | null { - if (isRevokedConnector(connector)) { + if (isRevokedConnector(connector) || isArchivedSource(connector) || isSetupFailedSource(connector)) { return null; } + // A paused source is surfaced in `review` ("Available actions"), never in + // `needsOwner`. Nothing is broken and nothing is waiting on the owner — the + // owner already decided to stop collecting — so counting it as "needs you" + // would inflate the one number that is supposed to mean "you are blocking + // something" (see `sourceAttentionHeadline`). But it must not vanish either: + // a paused row that produced no work item at all (the `revoked` treatment) + // would leave collection silently stopped with no path back on any list + // surface. `review` is exactly the group for an optional action the owner + // may take, so the source stays visible and carries its own way out. + // + // Checked before the verdict for the same reason `deriveRenderedSourceStatus` + // ranks paused early: a paused row's verdict describes collection that has + // stopped. + if (isPausedConnector(connector)) { + return itemFromConnector(connector, "review", { + actionLabel: RESUME_PAUSED_CTA_LABEL, + statusLabel: "is paused", + what: "Collection is paused. Your existing records, schedule, and sign-in are kept — resume to start collecting again.", + }); + } + const terminalSetupDisposition = connector.terminal_setup_disposition ?? null; if (isSetupInProgressConnector(connector) && terminalSetupDisposition) { const copy = TERMINAL_SETUP_DISPOSITION_COPY[terminalSetupDisposition]; @@ -536,34 +702,79 @@ export function projectSourceActionability(connector: RefConnectorSummary): Sour const routeId = connectionRouteId(connector); const label = connectorLabel(connector); const revoked = isRevokedConnector(connector); + const archived = isArchivedSource(connector); + const setupFailed = isSetupFailedSource(connector); + const paused = !revoked && isPausedConnector(connector); const terminalSetupDisposition = connector.terminal_setup_disposition ?? null; const pending = !revoked && isSetupInProgressConnector(connector) && terminalSetupDisposition === null; const running = connector.last_run !== null && isActiveConnectorRunSummaryStatus(connector.last_run.status); - const primaryAction = pending ? null : primaryRequiredAction(connector.rendered_verdict); - const primaryVerdictAction = formatPrimaryVerdictAction( - connector.rendered_verdict, - pending, - terminalSetupDisposition - ); + // An archived or setup-failed source offers NO action ON THIS ROW. An + // archived source's stored verdict still carries the required actions from + // when it was live — typically "Reconnect this account and collection + // resumes", which leads nowhere: reconnecting mints a new connection and + // resumes nothing here. A setup-failed source never had a credential or + // schedule to begin with, so its detail page has nothing actionable either + // — `NextActionCta` (`sources-view.tsx`) always links to THIS row's detail + // page, and that would be a dead end. The honest next step for both is a + // fresh attempt, which the page's own "add a source" link already offers. + // Suppressing at the projection root keeps every surface (list cue, + // passport foot, /syncs) consistent, the same intent `dfbbb8843` + // established for archived rows. + const noVerdictAction = archived || setupFailed; + const primaryAction = pending || noVerdictAction ? null : primaryRequiredAction(connector.rendered_verdict); + const primaryVerdictAction = noVerdictAction + ? null + : formatPrimaryVerdictAction(connector.rendered_verdict, pending, terminalSetupDisposition); return { // A failure summary is display formatting for a server verdict. Never use // the raw health snapshot as a classifier when the verdict is absent. failureSummary: - pending || !connector.rendered_verdict + pending || noVerdictAction || !connector.rendered_verdict ? null : deriveFailureSummary(connector.connection_health, connector.rendered_verdict), label, - nextAction: formatRenderedRequiredAction(connector.rendered_verdict, pending, terminalSetupDisposition), + nextAction: noVerdictAction + ? null + : formatRenderedRequiredAction(connector.rendered_verdict, pending, terminalSetupDisposition), ownerActionByStream: pending ? {} : ownerActionAvailabilityByStream(connector.rendered_verdict ?? null), ownerActionCue: ownerActionCueFromVerdictAction(primaryVerdictAction), + paused, primaryAction, primaryVerdictAction, + fusedStatus: fuseSourceStatus( + deriveRenderedSourceStatus( + connector.rendered_verdict, + revoked, + pending, + terminalSetupDisposition, + running, + paused + ), + { + hasEverSucceeded: connector.last_successful_run !== null, + syncing: running, + // Hand back the verdict the `running` collapse discards, so a source + // that is syncing AND failing still says it is failing. + // + // Only for the `running` collapse. `revoked`/`paused`/`pending` are + // LIFECYCLE facts that outrank any verdict — a revoked source is + // revoked no matter how its last verdict read — and + // `deriveRenderedSourceStatus` already ranks them ahead of `running` + // for exactly that reason. Passing the verdict for those states would + // let a stale "Blocked" overwrite "Revoked". + verdictFallback: + running && !revoked && !paused && !pending ? deriveSourceVerdictStatus(connector.rendered_verdict) : null, + } + ), renderedStatus: deriveRenderedSourceStatus( connector.rendered_verdict, revoked, pending, terminalSetupDisposition, - running + running, + paused, + archived, + setupFailed ), revoked, routeId, diff --git a/apps/console/src/app/(console)/lib/source-recovery-state.test.ts b/apps/console/src/app/(console)/lib/source-recovery-state.test.ts index 418f58f05..4cb017da5 100644 --- a/apps/console/src/app/(console)/lib/source-recovery-state.test.ts +++ b/apps/console/src/app/(console)/lib/source-recovery-state.test.ts @@ -67,7 +67,7 @@ function verdict(overrides: Partial = {}): RefRenderedVerdic channel: "calm", detail: {}, forward_statement: "The next run is expected to fill the remaining data.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, progress: { gaps_drained_last_run: null, headline: "Collecting in the background.", @@ -169,7 +169,7 @@ test("recovery: a connector-defect code_fix verdict is a system issue with no re channel: "advisory", pill: { label: "Can't collect", tone: "red" }, required_actions: [ - action({ audience: "maintainer", cta: "Connector code needs a fix", kind: "code_fix", terminal: true }), + action({ audience: "maintainer", cta: "Some data from this source can't be collected", kind: "code_fix", terminal: true }), ], }), health({ detail_gap_backlog: backlog({ pending: 12 }), state: "degraded" }) diff --git a/apps/console/src/app/(console)/lib/source-setup-presentation.test.ts b/apps/console/src/app/(console)/lib/source-setup-presentation.test.ts new file mode 100644 index 000000000..c0892215d --- /dev/null +++ b/apps/console/src/app/(console)/lib/source-setup-presentation.test.ts @@ -0,0 +1,285 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Regression coverage for `isRunnableAddOffer`, the /sources/add gate that + * decides whether a catalog entry is offered as a runnable "add source" + * card (main list or Preview disclosure) versus filtered out entirely. + * + * Root-caused live bug: the Signal connector (publicTier "preview", + * disposition "local_collector_enroll") was registered and owner-actionable, + * but `isRunnableAddOffer`'s old two-arm condition only matched + * (supported && available_now) or (preview && experimental_opt_in). + * `sourceSetupAvailability` maps `local_collector_enroll` to + * `"available_now"`, so a preview-tier local-collector entry matched + * NEITHER arm and was silently dropped from both the main list and the + * Preview disclosure. This was latent until Signal shipped as the first + * preview-tier local-collector connector. + * + * These tests assert the OBSERVABLE contract (is this entry offered on + * /sources/add), not the implementation's internal branches, across the + * full publicTier x disposition matrix that matters for that gate. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import type { ConnectorCatalogEntry } from "./connection-catalog.ts"; +import { + isRunnableAddOffer, + sourceSetupAction, + sourceSetupGuidance, + sourceSetupStatus, +} from "./source-setup-presentation.ts"; + +/** + * A complete, valid catalog entry fixture. Every field a real + * `buildConnectorCatalog` entry would carry is present with an inert + * default; tests override only the fields relevant to the case under test. + */ +function makeEntry(overrides: Partial & Pick): ConnectorCatalogEntry { + return { + acquisitionPaths: [], + connectorKey: "stub-connector", + deploymentReadiness: { blockers: [], guidance: null, state: "ready" }, + displayName: "Stub connector", + externalDocs: [], + isKnownScaffold: false, + listingNote: null, + modality: "network", + nextStepKind: "unsupported", + ownerActionable: true, + ownerActionMethod: null, + ownerActionUrl: null, + proofGate: null, + refreshPolicyRationale: null, + runbookPath: null, + setupDescription: null, + setupHelpText: null, + setupModality: "unsupported", + supportState: "supported", + ...overrides, + } as ConnectorCatalogEntry; +} + +test("preview + local_collector_enroll is offered on /sources/add (the Signal bug)", () => { + const signal = makeEntry({ + connectorKey: "signal", + disposition: "local_collector_enroll", + displayName: "Signal", + modality: "local_collector", + publicTier: "preview", + setupModality: "local_collector", + }); + assert.equal( + isRunnableAddOffer(signal), + true, + "a registered, owner-actionable preview-tier local-collector entry must be offered on /sources/add" + ); +}); + +test("preview + static_secret_experimental is still offered (pre-existing experimental-opt-in path)", () => { + const entry = makeEntry({ + disposition: "static_secret_experimental", + publicTier: "preview", + setupModality: "static_secret", + supportState: "experimental", + }); + assert.equal(isRunnableAddOffer(entry), true); +}); + +test("supported + local_collector_enroll is offered (pre-existing)", () => { + const entry = makeEntry({ + disposition: "local_collector_enroll", + modality: "local_collector", + publicTier: "supported", + setupModality: "local_collector", + }); + assert.equal(isRunnableAddOffer(entry), true); +}); + +test("supported + every available_now disposition is offered", () => { + const availableNowDispositions: readonly ConnectorCatalogEntry["disposition"][] = [ + "local_collector_enroll", + "static_secret_connect", + "manual_upload_connect", + "browser_collector_manual", + "provider_auth_connect", + ]; + for (const disposition of availableNowDispositions) { + const entry = makeEntry({ disposition, publicTier: "supported" }); + assert.equal(isRunnableAddOffer(entry), true, `supported + ${disposition} should be offered`); + } +}); + +test("development tier is never offered, regardless of disposition", () => { + const dispositions: readonly ConnectorCatalogEntry["disposition"][] = [ + "local_collector_enroll", + "static_secret_connect", + "static_secret_experimental", + "browser_collector_manual", + "manual_upload_connect", + "provider_auth_connect", + "provider_auth_deployment_blocked", + ]; + for (const disposition of dispositions) { + const entry = makeEntry({ disposition, publicTier: "development" }); + assert.equal(isRunnableAddOffer(entry), false, `development + ${disposition} must never be offered`); + } +}); + +test("preview + a requires_server_setup disposition is not offered", () => { + const entry = makeEntry({ + deploymentReadiness: { + blockers: [{ key: "PROVIDER_APP_ID", label: "Provider app ID", secret: true }], + guidance: null, + state: "needs_config", + }, + disposition: "provider_auth_deployment_blocked", + publicTier: "preview", + setupModality: "provider_authorization", + supportState: "needs_deployment_config", + }); + assert.equal(isRunnableAddOffer(entry), false, "a server-setup-blocked entry must never render a dead add offer"); +}); + +test("preview + a not_available_here disposition is not offered", () => { + const entry = makeEntry({ + disposition: "api_network_unsupported", + ownerActionable: false, + publicTier: "preview", + supportState: "unsupported", + }); + assert.equal(isRunnableAddOffer(entry), false, "an unsupported disposition must never be offered as runnable"); +}); + +test("preview + browser_collector_manual (Venmo) is offered on /sources/add", () => { + // Root-caused live bug: the Venmo connector shipped with publicTier + // "development" (unproven-against-a-real-account, matching its manifest's + // own header comment), which unconditionally withholds the add offer + // regardless of disposition -- so it never appeared on /sources/add even + // though it is registered, owner-actionable, and browser_bound with static + // credential capture like reddit/amazon (disposition + // "browser_collector_manual", which resolves to availability + // "available_now"). Venmo has the same evidence profile that moved Signal + // to Preview: fixture-driven unit/integration tests pass, but no live run + // against a real account has been recorded. Promoted to "preview" so the + // owner can opt in to perform that first live run. + const venmo = makeEntry({ + connectorKey: "venmo", + disposition: "browser_collector_manual", + displayName: "Venmo", + modality: "browser_bound", + publicTier: "preview", + setupModality: "static_secret", + }); + assert.equal( + isRunnableAddOffer(venmo), + true, + "a registered, owner-actionable preview-tier browser-bound entry must be offered on /sources/add" + ); +}); + +/** + * The Development disclosure (below Preview on /sources/add) coverage. + * + * Root-caused live bug: every Development-tier connector -- both real, + * implemented-but-unproven connectors (imessage, spotify, oura, + * google_maps_data_portability, ...) and genuine SKIP_RESULT scaffolds + * (anthropic, linkedin, loom, ...) -- was invisible on /sources/add. The + * owner could not tell what existed, could not test an unproven connector + * himself, and could not distinguish a broken connector from a missing one. + * `buildOwnerConnectorCatalog` used to drop every publicTier "development" + * template before it became a catalog entry at all, so no presentation + * function ever saw one. These tests pin the fixed contract: a real + * Development entry is never a runnable ADD OFFER (isRunnableAddOffer stays + * false, unconditionally, for the whole tier -- see "development tier is + * never offered" above), but it DOES get a self-test action, distinct status + * label, and honest guidance in the Development disclosure. A KNOWN scaffold + * gets none of those: no action, "Not implemented" status, and guidance that + * says plainly there is nothing to test yet. + */ + +test("development + real (non-scaffold) self-testable disposition gets a self-test action, never a runnable offer", () => { + const imessage = makeEntry({ + connectorKey: "imessage", + disposition: "local_collector_enroll", + displayName: "iMessage", + isKnownScaffold: false, + modality: "local_collector", + ownerActionable: false, + publicTier: "development", + setupModality: "local_collector", + }); + assert.equal(isRunnableAddOffer(imessage), false, "development is never a runnable /sources/add offer"); + assert.ok( + sourceSetupAction(imessage), + "a real (non-scaffold) development entry with a self-testable disposition gets a self-test action" + ); + assert.equal(sourceSetupStatus(imessage).label, "Development"); +}); + +test("development + known scaffold gets no action and a distinct 'Not implemented' status", () => { + const anthropic = makeEntry({ + connectorKey: "anthropic", + disposition: "browser_bound_runbook", + displayName: "Anthropic", + isKnownScaffold: true, + modality: "browser_bound", + ownerActionable: false, + publicTier: "development", + setupModality: "browser_bound", + }); + assert.equal(isRunnableAddOffer(anthropic), false); + assert.equal(sourceSetupAction(anthropic), null, "a known scaffold must never render an add action"); + assert.equal(sourceSetupStatus(anthropic).label, "Not implemented"); + assert.match( + sourceSetupGuidance(anthropic), + /scaffold/i, + "scaffold guidance must say plainly there is nothing to test yet" + ); +}); + +test("development + known scaffold with a self-testable disposition still gets no action", () => { + // The structural safety property: isKnownScaffold overrides disposition. + // Even if a scaffold's disposition happens to land in the self-test + // allowlist (e.g. a future scaffold resolves to static_secret_connect), + // clicking an action for it can never collect anything. + const scaffoldWithRunnableDisposition = makeEntry({ + connectorKey: "hypothetical-scaffold", + disposition: "static_secret_connect", + isKnownScaffold: true, + publicTier: "development", + setupModality: "static_secret", + }); + assert.equal(sourceSetupAction(scaffoldWithRunnableDisposition), null); +}); + +test("development + a disposition with no safe action gets honest fallback guidance, not the generic dead-end", () => { + const localCollectorUnproven = makeEntry({ + connectorKey: "some-filesystem-connector", + disposition: "local_collector_unproven", + isKnownScaffold: false, + modality: "local_collector", + ownerActionable: false, + publicTier: "development", + setupModality: "local_collector", + }); + assert.equal(sourceSetupAction(localCollectorUnproven), null); + assert.match( + sourceSetupGuidance(localCollectorUnproven), + /Development/, + "a real development entry without a safe action still gets development-framed guidance, not the unclassified dead-end" + ); +}); + +test("development real entry surfaces its manifest listingNote in guidance when present", () => { + const spotify = makeEntry({ + connectorKey: "spotify", + disposition: "static_secret_connect", + isKnownScaffold: false, + listingNote: "Hidden from the reference dashboard catalog until a credentialed run proves useful records in the deployment.", + publicTier: "development", + setupModality: "static_secret", + }); + assert.equal(sourceSetupAction(spotify) !== null, true); +}); diff --git a/apps/console/src/app/(console)/lib/source-setup-presentation.ts b/apps/console/src/app/(console)/lib/source-setup-presentation.ts index 1d65875c8..4f99e7371 100644 --- a/apps/console/src/app/(console)/lib/source-setup-presentation.ts +++ b/apps/console/src/app/(console)/lib/source-setup-presentation.ts @@ -142,7 +142,15 @@ export function sourceSetupRank(entry: ConnectorCatalogEntry): number { /** The owner-facing status label + tone for first-account setup. */ export function sourceSetupStatus(entry: ConnectorCatalogEntry): SourceSetupStatus { if (entry.publicTier === "development") { - return { label: publicTierLabel(entry.publicTier), tone: "border-border bg-muted/30 text-muted-foreground" }; + // "Not implemented" and "Development" are deliberately different labels: + // a scaffold has no collection code at all (there is nothing to test), + // while a real Development entry has an implemented, self-testable setup + // path that simply has not been proven against a live account yet. + // Collapsing these into one badge would hide exactly the distinction the + // owner needs to decide whether clicking a card can do anything. + return entry.isKnownScaffold + ? { label: "Not implemented", tone: "border-border bg-muted/30 text-muted-foreground" } + : { label: publicTierLabel(entry.publicTier), tone: "border-border bg-muted/30 text-muted-foreground" }; } if (entry.publicTier === "preview") { return { label: publicTierLabel(entry.publicTier), tone: "border-[color:var(--warning)]/30 bg-status-warning-bg text-status-warning-fg" }; @@ -258,8 +266,75 @@ const CLASSIFIED_UNAVAILABLE_DISPOSITIONS = new Set([ "provider_auth_proof_gated", ]); +/** + * Dispositions `sourceSetupAction`'s switch resolves to a real link. A real + * (non-scaffold) Development entry is only ever offered a self-test action, + * and keeps its disposition's normal guidance copy (never the generic + * Development fallback below), when its disposition is one of these -- + * exactly the set a Preview or Supported entry with the same disposition + * would also get. Any disposition outside this set (proof-gated, + * deployment-blocked, unknown, etc.) has no safe action to offer regardless + * of tier. + */ +const DEVELOPMENT_SELF_TEST_DISPOSITIONS = new Set([ + "local_collector_enroll", + "static_secret_connect", + "static_secret_experimental", + "manual_upload_connect", + "browser_collector_manual", + "provider_auth_connect", +]); + +function isDevelopmentSelfTestDisposition(disposition: ConnectorCatalogEntry["disposition"]): boolean { + return DEVELOPMENT_SELF_TEST_DISPOSITIONS.has(disposition); +} + +/** + * Fallback guidance for a Development entry with no self-test action and no + * useful disposition-specific copy of its own (an unrecognised/proof-gated/ + * unproven disposition, the same set that would otherwise reach + * `unclassifiedSetupGuidance`'s generic "this dashboard does not recognise + * this source" text). A Development connector IS recognised -- it is just + * unproven or unimplemented -- so this names that honestly instead. + */ +function developmentFallbackGuidance(entry: ConnectorCatalogEntry): string { + if (entry.isKnownScaffold) { + const note = entry.listingNote; + return note + ? `This connector is scaffolded, not finished, and cannot collect data yet: ${note}` + : "This connector is scaffolded, not finished: it can reach the provider but does not collect data yet. There is nothing to test here yet."; + } + // A deployment-blocked entry needs the exact missing settings named, not a + // generic "unproven" note -- that fact is more actionable than anything + // else this branch could say, and it holds regardless of tier. + if (entry.disposition === "provider_auth_deployment_blocked") { + return `Development: also waiting on server settings: ${entry.deploymentReadiness.blockers + .map((blocker) => blocker.label || blocker.key) + .join(", ")}.`; + } + const note = entry.listingNote ?? entry.refreshPolicyRationale ?? entry.setupDescription; + const base = + "Development: this connector's setup path is implemented, but no live run against a real account has proven it yet."; + return note ? `${base} ${note}` : `${base} Test it with non-critical data.`; +} + /** One short owner-facing guidance line for first-account setup. */ export function sourceSetupGuidance(entry: ConnectorCatalogEntry): string { + // A self-testable Development entry (real, non-scaffold, disposition in + // the self-test allowlist) keeps the normal disposition-specific copy + // below -- the same text a Preview/Supported entry with that disposition + // gets -- because it correctly describes the self-test action this entry + // actually renders. Every OTHER Development entry gets the honest + // Development fallback here instead: a KNOWN scaffold always does (its own + // "nothing to test yet" fact must never be shadowed by a disposition case + // written for a non-development context, e.g. `browser_bound_runbook`'s + // "cannot start a new account from here yet", which sounds like a proven + // connector waiting on a feature rather than one with no collection code + // at all); a real entry with no safe action does too, so it never falls + // into the unclassified-disposition dead-end copy just below. + if (entry.publicTier === "development" && !isDevelopmentSelfTestDisposition(entry.disposition)) { + return developmentFallbackGuidance(entry); + } if (isUnavailableSetupEntry(entry) && !CLASSIFIED_UNAVAILABLE_DISPOSITIONS.has(entry.disposition)) { return unclassifiedSetupGuidance(entry); } @@ -305,7 +380,18 @@ export function sourceSetupGuidance(entry: ConnectorCatalogEntry): string { /** The primary next action for first-account setup, or null when none exists. */ export function sourceSetupAction(entry: ConnectorCatalogEntry): SourceSetupAction | null { - if (entry.publicTier === "development" || !(isOwnerActionableEntry(entry) || isExperimentalEntry(entry))) { + if (entry.publicTier === "development") { + // A KNOWN scaffold (unconditional SKIP_RESULT, no real collection) must + // never render an action: clicking it can never collect anything, and an + // add button on a stub is a worse experience than the stub staying + // invisible. A real-but-unproven Development entry falls through to the + // same disposition switch below, so it gets the exact action a Preview + // entry with the same disposition would get -- the only difference is the + // Development disclosure's own copy naming it unproven. + if (entry.isKnownScaffold || !isDevelopmentSelfTestDisposition(entry.disposition)) { + return null; + } + } else if (!(isOwnerActionableEntry(entry) || isExperimentalEntry(entry))) { return null; } // Browser-bound connectors that also declare credential capture still start @@ -385,9 +471,26 @@ export function sourceSetupAvailability(entry: ConnectorCatalogEntry): SourceSet /** Only runnable actions belong in /sources/add; server-setting and runbook cards do not. */ export function isRunnableAddOffer(entry: ConnectorCatalogEntry): boolean { + const availability = sourceSetupAvailability(entry); + if (entry.publicTier === "supported") { + return availability === "available_now"; + } + // A preview-tier entry is offered when its setup path is genuinely + // self-service, which is exactly what `available_now` already means: "the + // owner can add an account now from a shipped surface". The previous + // condition paired preview with ONLY `experimental_opt_in`, which held until + // the first preview-tier local-collector connector shipped -- enrollment + // issues a code the owner redeems on their own machine, so it resolves to + // `available_now` and matched neither arm. The connector was registered, + // owner-actionable, and invisible on /sources/add. + // + // Keyed on availability rather than on the one disposition that broke, so a + // future preview-tier disposition resolving to `available_now` does not + // reintroduce the same invisibility. The remaining availabilities + // (`requires_server_setup`, `not_available_here`) still correctly withhold + // the offer. return ( - (entry.publicTier === "supported" && sourceSetupAvailability(entry) === "available_now") || - (entry.publicTier === "preview" && sourceSetupAvailability(entry) === "experimental_opt_in") + entry.publicTier === "preview" && (availability === "experimental_opt_in" || availability === "available_now") ); } diff --git a/apps/console/src/app/(console)/lib/stream-evidence-state.ts b/apps/console/src/app/(console)/lib/stream-evidence-state.ts index 359ad6ef0..75d5fe922 100644 --- a/apps/console/src/app/(console)/lib/stream-evidence-state.ts +++ b/apps/console/src/app/(console)/lib/stream-evidence-state.ts @@ -18,7 +18,7 @@ * mirroring the `connection-evidence.ts` idiom. */ -import type { EvidenceTone } from "./connection-evidence.ts"; +import type { EvidenceTone } from "@pdpp/display"; import type { RefConnectorStreamRecord } from "./ref-client.ts"; export interface StreamCountLabel { diff --git a/apps/console/src/app/(console)/lib/stream-record-count-aggregate.test.ts b/apps/console/src/app/(console)/lib/stream-record-count-aggregate.test.ts new file mode 100644 index 000000000..fb4d9cce2 --- /dev/null +++ b/apps/console/src/app/(console)/lib/stream-record-count-aggregate.test.ts @@ -0,0 +1,101 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Console-side guard against the "Holding 0 records." defect class. + * + * `getConnectorOverview` used to aggregate per-stream counts with + * `streams.reduce((sum, s) => sum + (s.record_count ?? 0), 0)` and returned no + * `totalRecordsState`. Both halves were needed to make the lie: + * + * 1. `record_count` is `null` when the server could NOT measure a stream + * (its own contract: "rendered as unavailable, never fabricated as 0"), + * and `?? 0` turned every such stream into a confident zero. + * 2. `isTotalRecordsAuthoritative(undefined) === true`, so omitting the state + * told the renderer the resulting number was fully trustworthy. + * + * Together, a connection whose streams were never measured rendered + * "0 records ingested" with `reliable: true` — the same false statement the + * server-side fix removed from the rendered verdict. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { isTotalRecordsAuthoritative } from "@pdpp/operator-ui/lib/total-records-label"; +import { aggregateStreamRecordCounts, type StreamRecordCountEvidence } from "./stream-record-count-aggregate.ts"; + +function streamSummary(overrides: Partial): StreamRecordCountEvidence { + return { record_count: null, ...overrides }; +} + +test("aggregate: an UNMEASURED stream never contributes a fabricated zero to an authoritative total", () => { + // The live shape: the server could not measure this stream, so it sent null. + const out = aggregateStreamRecordCounts([streamSummary({ count_state: "unobserved", record_count: null })]); + assert.strictEqual( + isTotalRecordsAuthoritative(out.totalRecordsState), + false, + "an unmeasured connection must not report an authoritative count" + ); + assert.strictEqual(out.totalRecordsState, "unobserved"); +}); + +test("aggregate: a legacy reference (no count_state) still treats null record_count as unmeasured", () => { + // Older references omit count_state; `record_count === null` is the + // documented legacy "unavailable" signal and must be honored. + const out = aggregateStreamRecordCounts([streamSummary({ record_count: null })]); + assert.strictEqual(isTotalRecordsAuthoritative(out.totalRecordsState), false); + assert.strictEqual(out.totalRecordsState, "unobserved"); +}); + +test("aggregate: a PARTIALLY measured connection reports a non-authoritative total, not a confident one", () => { + // 500 counted, one stream never measured. The sum is real but incomplete, so + // it must never render as this connection's authoritative holdings. + const out = aggregateStreamRecordCounts([ + streamSummary({ count_state: "known", record_count: 500 }), + streamSummary({ count_state: "unobserved", record_count: null }), + ]); + assert.strictEqual(out.totalRecords, 500, "the measured part is still reported"); + assert.strictEqual( + isTotalRecordsAuthoritative(out.totalRecordsState), + false, + "a partial sum must not claim to be the total" + ); +}); + +test("aggregate: a fully measured connection reports its real count authoritatively", () => { + const out = aggregateStreamRecordCounts([ + streamSummary({ count_state: "known", record_count: 246_559 }), + streamSummary({ count_state: "known", record_count: 52_689 }), + ]); + assert.strictEqual(out.totalRecords, 299_248); + assert.strictEqual(out.totalRecordsState, "known"); + assert.strictEqual(isTotalRecordsAuthoritative(out.totalRecordsState), true); +}); + +test("aggregate: a PROVEN zero is authoritative (a measured zero is a real fact)", () => { + const out = aggregateStreamRecordCounts([streamSummary({ count_state: "known_zero", record_count: 0 })]); + assert.strictEqual(out.totalRecords, 0); + assert.strictEqual(out.totalRecordsState, "known_zero"); + assert.strictEqual( + isTotalRecordsAuthoritative(out.totalRecordsState), + true, + "a proven zero must stay authoritative; only UNMEASURED zeros are the lie" + ); +}); + +test("aggregate: a stale stream keeps the total non-authoritative", () => { + const out = aggregateStreamRecordCounts([streamSummary({ count_state: "stale", record_count: 42 })]); + assert.strictEqual(out.totalRecordsState, "stale"); + assert.strictEqual(isTotalRecordsAuthoritative(out.totalRecordsState), false); +}); + +test("aggregate: no streams at all claims nothing rather than a measured zero", () => { + const out = aggregateStreamRecordCounts([]); + assert.strictEqual(out.totalRecords, 0); + assert.strictEqual( + isTotalRecordsAuthoritative(out.totalRecordsState), + false, + "an empty stream list is an absence of evidence, not a measured zero" + ); +}); diff --git a/apps/console/src/app/(console)/lib/stream-record-count-aggregate.ts b/apps/console/src/app/(console)/lib/stream-record-count-aggregate.ts new file mode 100644 index 000000000..c8b5b8ccd --- /dev/null +++ b/apps/console/src/app/(console)/lib/stream-record-count-aggregate.ts @@ -0,0 +1,82 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Aggregation of per-stream retained-record counts into a connection total. + * + * Split out of `rs-client.ts` (which is server-only by transitive import) so + * this pure, display-critical rule can be tested directly. + */ + +import type { RefCountState } from "./ref-client.ts"; + +/** + * The per-stream count evidence this aggregation reads. Structurally a subset + * of `StreamSummary` in `rs-client.ts`, declared here so the rule stays free of + * that module's server-only dependencies. + */ +export interface StreamRecordCountEvidence { + readonly count_state?: RefCountState; + /** + * Retained-record count, or `null` when the count is unavailable. The server + * synthesizes exact zeros only when the retained-size projection is proven + * fresh and clean, so a declared stream with no row is an unreliable count. + */ + readonly record_count: number | null; +} + +/** + * Sum per-stream retained-record counts WITHOUT fabricating a count the + * streams do not support. + * + * `record_count` is `null` exactly when the server could not measure a stream + * ("rendered as unavailable, never fabricated as 0"). Coercing those nulls with + * `?? 0` and summing produces a confident total over unmeasured streams — the + * defect that told owners "Holding 0 records." about connections holding + * hundreds of thousands of records. + * + * The returned `totalRecordsState` is what keeps the sum honest downstream: + * `resolveRecordCountDisplay`/`formatTotalRecordsLabel` only render a bare + * number when the state is authoritative, and `isTotalRecordsAuthoritative` + * treats an OMITTED state as authoritative — so this always reports one rather + * than leaving it undefined. + */ +export function aggregateStreamRecordCounts(streams: readonly StreamRecordCountEvidence[]): { + totalRecords: number; + totalRecordsState: RefCountState; +} { + let total = 0; + let measuredAny = false; + let unmeasuredAny = false; + let staleAny = false; + for (const stream of streams) { + // `count_state` is the explicit evidence when the reference supplies it; an + // older reference omits it, leaving `record_count === null` as the + // documented legacy "unavailable" signal. + const state = stream.count_state; + const unmeasured = + state === undefined ? stream.record_count === null : state === "unobserved" || state === "unknown"; + if (unmeasured) { + unmeasuredAny = true; + continue; + } + if (state === "stale") { + staleAny = true; + } + total += stream.record_count ?? 0; + measuredAny = true; + } + if (unmeasuredAny) { + // A partial sum is not a total. Report the number we do have, but never as + // an authoritative count: some of this connection's data was never counted. + return { totalRecords: total, totalRecordsState: measuredAny ? "stale" : "unobserved" }; + } + if (staleAny) { + return { totalRecords: total, totalRecordsState: "stale" }; + } + if (!measuredAny) { + // No streams at all: nothing has been observed, so claim nothing. + return { totalRecords: 0, totalRecordsState: "unobserved" }; + } + return { totalRecords: total, totalRecordsState: total > 0 ? "known" : "known_zero" }; +} diff --git a/apps/console/src/app/(console)/read-resilience-root.invariants.test.ts b/apps/console/src/app/(console)/read-resilience-root.invariants.test.ts new file mode 100644 index 000000000..fc928693a --- /dev/null +++ b/apps/console/src/app/(console)/read-resilience-root.invariants.test.ts @@ -0,0 +1,91 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the console ROOT error boundary + * (`(console)/error.tsx`) — deliberately DIFFERENT from the leaf-segment + * boundaries (`sources/error.tsx`, `syncs/error.tsx`, etc.), which now all + * retry unbounded forever. + * + * The root boundary catches errors from ANYWHERE in the segment not already + * caught by a more specific nested boundary, including the dashboard + * overview `page.tsx` — which already fault-isolates every one of ITS OWN + * data reads via `safeRead()`, so an error reaching this root boundary is + * either (a) the same known RSC stream-teardown race the leaf boundaries + * handle, now unprovable-by-route, or (b) a genuine unhandled fault in + * render/layout code. There is no error-reporting integration in this + * codebase, so `console.error` here is the only diagnostic signal an + * operator has for (b); retrying that forever, silently, would delete the + * signal for a real crash. + * + * So the root boundary: + * - absorbs the SLVP-standard case the same way the leaf boundaries do — + * quiet skeleton, no failure copy, capped backoff — for a BOUNDED number + * of attempts; + * - falls back to the pre-existing "Something went wrong" panel only after + * that bound is exceeded, which is strictly no worse than its prior + * immediate-failure-panel behavior and materially better for the common + * transient case. + * + * These invariants pin: the quiet phase exists and matches the leaf + * boundaries' properties (skeleton reuse, module-scope counter, no failure + * copy during the quiet phase); the bound is finite and explicit (NOT + * unbounded, unlike every leaf boundary); and the terminal fallback panel is + * still reachable (this must not become an infinite silent retry loop that + * hides a genuine crash). + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const USES_LOADING_SKELETON_RE = / { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.match(src, CALLS_RESET_RE); + assert.match(src, SCHEDULES_RETRY_RE); +}); + +test("unlike every leaf-segment boundary, the root retry is explicitly BOUNDED, not unbounded", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match( + src, + BOUNDED_ATTEMPTS_RE, + "the root boundary must cap quiet retries so a genuine crash eventually surfaces" + ); + assert.match(src, NUMERIC_BOUND_RE); +}); + +test("the boundary still has a reachable terminal fallback panel after the quiet phase is exhausted", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, HAS_TERMINAL_FALLBACK_RE); + assert.match(src, HAS_TRY_AGAIN_BUTTON_RE); + assert.match(src, HAS_SIGN_IN_LINK_RE); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/schedules/error.tsx b/apps/console/src/app/(console)/schedules/error.tsx index bfbec8089..3f41bdb27 100644 --- a/apps/console/src/app/(console)/schedules/error.tsx +++ b/apps/console/src/app/(console)/schedules/error.tsx @@ -3,16 +3,62 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { SegmentError } from "../components/segment-error.tsx"; +import { useEffect } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts"; +import { ListLoadingSkeleton } from "../components/route-loading.tsx"; + +/** + * Schedules-segment error boundary (App Router convention) — SLVP bar: + * Stripe, Linear, Vercel, and Plaid never tell an owner "we hit a transient + * read interruption, retrying." The page renders, or it quietly shows + * last-known state. The owner never learns the backend hiccuped. + * + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`): the read itself is fine — React's Flight/RSC + * streaming writer reacting to the HTTP response closing before the stream + * finished flushing (e.g. `schedule-row.tsx`'s poller firing + * `router.refresh()` while a prior refresh's stream is still in flight). It + * is a client-transport race below the data layer, not a backend outage — + * see `sources/error.tsx` for the full original writeup. + * + * `/schedules` has no client-cached last-known-read marker, so this boundary + * shows the plain skeleton with no staleness caption rather than fabricate a + * timestamp. + * + * Self-contained on purpose: a `"use client"` boundary must not import + * server-only modules, since the dashboard shell transitively pulls in + * `lib/owner-token.ts` (`server-only`). + */ + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. + */ +const retryCounter = createRetryCounter(); + +export default function SchedulesError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. + console.error(error); + }, [error]); + + useEffect(() => { + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // counter. There is deliberately no ceiling on the counter itself — a + // persistent failure degrades to a slow quiet heartbeat, never a dead end. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + reset(); + }, delay); + return () => clearTimeout(id); + }, [reset]); -export default function SchedulesError(props: { error: Error & { digest?: string }; reset: () => void }) { return ( - +
+ +
); } diff --git a/apps/console/src/app/(console)/schedules/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/schedules/read-resilience.invariants.test.ts new file mode 100644 index 000000000..b4e62aee1 --- /dev/null +++ b/apps/console/src/app/(console)/schedules/read-resilience.invariants.test.ts @@ -0,0 +1,88 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the schedules segment, mirroring + * `sources/read-resilience.invariants.test.ts`. See that file and + * `syncs/read-resilience.invariants.test.ts` for the full standard this + * pattern enforces; this file pins the same properties for `/schedules`. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g; + +/** + * Strip `/* ... *‍/` block comments before checking for retired owner-facing + * copy. The boundary's doc comment legitimately QUOTES the retired phrases + * (to explain what this pattern replaces and why) — that is documentation, + * not rendered JSX text, so it must not trip the ban. + */ +function withoutBlockComments(src: string): string { + return src.replace(BLOCK_COMMENT_RE, ""); +} + +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_ERROR_HEADING_RE = /Read error/; +const RETIRED_TRY_AGAIN_RE = /Try again/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i; +const RETIRED_BACK_LINK_RE = /Back to schedules/; +const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/; + +const USES_LOADING_SKELETON_RE = / { + const rawSrc = await readFile(ERROR_FILE, "utf8"); + const src = withoutBlockComments(rawSrc); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE); + assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BACK_LINK_RE); + assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE); +}); + +test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + // loading.tsx uses ListLoadingSkeleton label="schedules" rows={6}. + assert.match(src, /ListLoadingSkeleton label="schedules" rows=\{6\}/); +}); + +test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, CALLS_RESET_RE); + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.doesNotMatch( + src, + NO_REACT_STATE_COUNTER_RE, + "the retry counter must live at module scope, not React state, or backoff never grows across remounts" + ); + assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded"); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/sources/[connector]/[stream]/[recordKey]/record-fields-display.test.ts b/apps/console/src/app/(console)/sources/[connector]/[stream]/[recordKey]/record-fields-display.test.ts index 47d3f3147..a9aa3897d 100644 --- a/apps/console/src/app/(console)/sources/[connector]/[stream]/[recordKey]/record-fields-display.test.ts +++ b/apps/console/src/app/(console)/sources/[connector]/[stream]/[recordKey]/record-fields-display.test.ts @@ -58,11 +58,11 @@ test("an object with no string-valued field falls back to compact JSON", () => { test("an array of {name, email} objects renders readable names, not raw JSON — the live Gmail `cc` bug", () => { const cc = [ - { email: "mmarco@law.harvard.edu", name: "Meg Marco" }, - { email: "anna@opendatalabs.xyz", name: "Anna Kazlauskas" }, + { email: "rowan.diaz@example.edu", name: "Rowan Diaz" }, + { email: "sasha.lindqvist@example.org", name: "Sasha Lindqvist" }, ]; const rendered = renderValue(cc, undefined); - assert.equal(rendered.text, "Meg Marco, Anna Kazlauskas"); + assert.equal(rendered.text, "Rowan Diaz, Sasha Lindqvist"); assert.doesNotMatch(rendered.text, JSON_SYNTAX_RE, "must never contain JSON syntax characters"); assert.equal(rendered.empty, false); }); diff --git a/apps/console/src/app/(console)/sources/[connector]/[stream]/record-list-money-wiring.test.ts b/apps/console/src/app/(console)/sources/[connector]/[stream]/record-list-money-wiring.test.ts index 26f2a7c23..6c4a7bcbc 100644 --- a/apps/console/src/app/(console)/sources/[connector]/[stream]/record-list-money-wiring.test.ts +++ b/apps/console/src/app/(console)/sources/[connector]/[stream]/record-list-money-wiring.test.ts @@ -116,11 +116,11 @@ test("BEHAVIOR null/absent cell values render empty, matching stringifyCell", () test("BEHAVIOR an array-of-objects cell (e.g. a Gmail `cc` field) renders readable names, never raw JSON", () => { const cc = [ - { email: "mmarco@law.harvard.edu", name: "Meg Marco" }, - { email: "anna@opendatalabs.xyz", name: "Anna Kazlauskas" }, + { email: "rowan.diaz@example.edu", name: "Rowan Diaz" }, + { email: "sasha.lindqvist@example.org", name: "Sasha Lindqvist" }, ]; const text = displayCell(cc, undefined); - assert.equal(text, "Meg Marco, Anna Kazlauskas"); + assert.equal(text, "Rowan Diaz, Sasha Lindqvist"); assert.doesNotMatch(text, JSON_SYNTAX_RE, "must never contain JSON syntax characters"); }); diff --git a/apps/console/src/app/(console)/sources/[connector]/actions.ts b/apps/console/src/app/(console)/sources/[connector]/actions.ts index 7012a9aca..ad20fddde 100644 --- a/apps/console/src/app/(console)/sources/[connector]/actions.ts +++ b/apps/console/src/app/(console)/sources/[connector]/actions.ts @@ -10,9 +10,11 @@ import { deleteConnection, deleteConnectionSchedule, deleteConnectorSchedule, + pauseConnection, pauseConnectionSchedule, pauseConnectorSchedule, reactivateConnection, + resumeConnection, resumeConnectionSchedule, resumeConnectorSchedule, revokeConnection, @@ -176,6 +178,89 @@ export async function deleteConnectorScheduleAction(formData: FormData) { redirect(connectorHref(routeId, message, error)); } +/** + * Owner-pause an active connection from the console. Re-verifies the owner + * session, then calls the shared owner-session + * `POST /_ref/connections/:id/pause` route. Pause stops future collection and + * gives up nothing: records, grants, schedule, and the stored sign-in are all + * retained, and `resumeConnectionAction` is the one-click way back. + * + * Deliberately NOT a danger-zone action and deliberately NOT confirmed: unlike + * revoke (ends the account relationship) and delete (erases records), pause is + * fully reversible from the same page, so a confirmation ceremony would be + * friction without a risk to guard. It redirects back to the operator-controls + * anchor rather than the connections list, because the owner stays on this + * connection to resume it later. + * + * The `not_active` typed outcome is messaged in place rather than thrown. + */ +export async function pauseConnectionAction(formData: FormData) { + const connectionId = asString(formData.get("connection_id")); + const routeId = connectionId; + await requireDashboardAccess(connectorHref(routeId)); + if (!connectionId) { + redirect(connectorHref(routeId, undefined, "This connection has no addressable id to pause.")); + } + + let message: string | undefined; + let error: string | undefined; + try { + const result = await pauseConnection(connectionId); + if (result.status === "paused") { + message = "Collection paused. Your records, schedule, and sign-in are kept — resume whenever you're ready."; + } else if (result.status === "not_active") { + message = "This connection wasn't collecting — no change was made."; + } else { + error = "Connection not found. It may have been deleted."; + } + } catch (err) { + error = errorMessage(err); + } + + revalidatePath("/sources"); + revalidatePath(`/sources/${encodeURIComponent(routeId)}`); + redirect(connectorHref(routeId, message, error)); +} + +/** + * Owner-resume a paused connection from the console. The inverse of + * `pauseConnectionAction`: re-verifies the owner session, then calls the + * shared owner-session `POST /_ref/connections/:id/resume` route. Flips the + * connection back to active so scheduled and manual runs land again; + * already-collected records, grants, and schedule are preserved. Credential + * freshness is handled on the next collection run. + * + * The `not_paused` typed outcome (connection was already active) is messaged + * in place rather than thrown. + */ +export async function resumeConnectionAction(formData: FormData) { + const connectionId = asString(formData.get("connection_id")); + const routeId = connectionId; + await requireDashboardAccess(connectorHref(routeId)); + if (!connectionId) { + redirect(connectorHref(routeId, undefined, "This connection has no addressable id to resume.")); + } + + let message: string | undefined; + let error: string | undefined; + try { + const result = await resumeConnection(connectionId); + if (result.status === "resumed") { + message = "Collection resumed. This source will collect again on its next run."; + } else if (result.status === "not_paused") { + message = "This connection was already collecting — no change was made."; + } else { + error = "Connection not found. It may have been deleted."; + } + } catch (err) { + error = errorMessage(err); + } + + revalidatePath("/sources"); + revalidatePath(`/sources/${encodeURIComponent(routeId)}`); + redirect(connectorHref(routeId, message, error)); +} + // Danger-zone anchor on the connection detail page. The revoke/delete forms // scroll here after a redirect so the operator lands on the result banner. function dangerZoneHref(routeId: string, message?: string, error?: string): string { diff --git a/apps/console/src/app/(console)/sources/[connector]/connection-diagnostics.tsx b/apps/console/src/app/(console)/sources/[connector]/connection-diagnostics.tsx index 1bb86bbeb..1e63676eb 100644 --- a/apps/console/src/app/(console)/sources/[connector]/connection-diagnostics.tsx +++ b/apps/console/src/app/(console)/sources/[connector]/connection-diagnostics.tsx @@ -131,7 +131,7 @@ export function ConnectionDiagnostics({ : null; return (
{renderedVerdict ? : null} @@ -861,19 +861,43 @@ function LocalCollectorGapDiagnostics({ source }: { source: DeviceSourceInstance 0 ? `Reasons: ${gaps.reasons.join(", ")}` : undefined} + title={ + gaps.reasons.length > 0 + ? `Reasons: ${gaps.reasons.map(localCollectorGapReasonLabel).join(", ")}` + : undefined + } > {formatLocalCollectorGaps(gaps)} ); } +/** + * Plain-English copy for the local-collector gap reasons. + * + * B9 (owner ledger 2026-08-22): these reason codes used to reach the owner + * verbatim — the owner saw `connector_child_failure` and called it "an odd + * stream". It is not a stream (that leak is fixed in `ref-control.ts` + * `pendingDetailGapCountsByStream`), but this IS the channel that legitimately + * carries it, so the wording has to mean something here. Both replacements stay + * as bad as the codes they replace — a crash still reads as a crash. + */ +const LOCAL_COLLECTOR_GAP_REASON_COPY: Readonly> = Object.freeze({ + connector_child_failure: "the collector crashed while gathering this data", + policy_budget: "the collector stopped at its scan limit", +}); + +function localCollectorGapReasonLabel(reason: string): string { + return LOCAL_COLLECTOR_GAP_REASON_COPY[reason] ?? reason.replace(/[_-]+/g, " "); +} + function formatLocalCollectorGaps(gaps: NonNullable): string { if (gaps.unreliable) { return "Local gap diagnostics unreliable."; } if (gaps.pending_count > 0) { - const reason = gaps.reasons.length > 0 ? ` · ${gaps.reasons.join(", ")}` : ""; + const reason = + gaps.reasons.length > 0 ? ` · ${gaps.reasons.map(localCollectorGapReasonLabel).join(", ")}` : ""; return `${gaps.pending_count.toLocaleString()} local detail gap${ gaps.pending_count === 1 ? "" : "s" } pending${reason}.`; diff --git a/apps/console/src/app/(console)/sources/[connector]/page.tsx b/apps/console/src/app/(console)/sources/[connector]/page.tsx index 0d48e7366..05db01419 100644 --- a/apps/console/src/app/(console)/sources/[connector]/page.tsx +++ b/apps/console/src/app/(console)/sources/[connector]/page.tsx @@ -72,7 +72,7 @@ import { isUnexpectedStreamDeclaration, streamCountLabel } from "../../lib/strea import { connectorInstanceIdForConnection, resolveConnectionForRecordsRoute } from "../connection-route.ts"; import { findManifestForConnectorId } from "../lib/relationships.ts"; import { formatConnectorHeaderCount } from "../sources-view-model.ts"; -import { resumeConnectorScheduleAction } from "./actions.ts"; +import { pauseConnectionAction, resumeConnectionAction, resumeConnectorScheduleAction } from "./actions.ts"; import { ConnectionDangerZone } from "./connection-danger-zone.tsx"; import { ConnectionDiagnostics } from "./connection-diagnostics.tsx"; import { RenameConnection } from "./rename-connection.tsx"; @@ -648,6 +648,22 @@ function ConnectorPageView({ scheduleActiveRunId, }); const revoked = isRevokedConnection(overview); + const paused = !revoked && overview.connectionStatus === "paused"; + // A recovered historical-archive row: paused (never revoked), landed by the + // archive-recovery operation rather than an owner revoke. It gets the + // credential-repair notice INSTEAD of a plain Resume action — see + // `PausedHistoricalArchiveSection`'s doc comment for why resuming such a row + // without first repairing its credential would just fail on the next run. + const pausedHistoricalArchive = paused && sourceBindingKind === "historical_archive"; + // Every OTHER paused row — including one the owner paused deliberately — gets + // a real Resume action. Without this, a `paused` connection outside the + // archive-recovery journey would be visible but unrecoverable from the + // console, which is the dead end pause/resume exists to remove. + const pausedResumable = paused && !pausedHistoricalArchive; + // Pause is offered only on a connection that is actually collecting. A + // draft/revoked/already-paused row has nothing to pause, and the route would + // answer `connector_instance_not_active` anyway. + const pausable = !(revoked || paused) && overview.connectionStatus === "active"; // Stable rename selector: prefer the explicit instance id, fall back to the // connection id. Both address the same connection on the backend route. const renameSelector = connectorInstanceId ?? connectionId; @@ -760,6 +776,10 @@ function ConnectorPageView({ {revoked ? : null} + {pausedHistoricalArchive ? : null} + + {pausedResumable ? : null} + 0 - ? "Record counts show what this source currently retains. Coverage and next-run disposition come from the latest collection report; an unknown denominator reads unknown, never complete." + ? "Record counts show what this source currently retains. Coverage and next-run disposition come from the latest collection report; when the total is unmeasured, coverage reads \"not measured\", never complete." : undefined } title={`Streams (${streams.length})`} @@ -841,6 +861,8 @@ function ConnectorPageView({ sourceInstancesError={sourceInstancesError} /> + {pausable ? : null} + +
+ {connectionId ? : null} + + Resume collecting + +
+
+ ); +} + +/** + * Pause control for a connection that is actively collecting. Deliberately + * NOT in the danger zone with revoke/delete, and deliberately unconfirmed: + * pause changes no data and is undone by one click on the same page, so the + * confirmation ceremony those destructive actions require would be friction + * with nothing to protect. The copy names the reversibility and contrasts + * pause with revoke, so the owner can tell the two apart before acting. + */ +function PauseConnectionSection({ connectionId }: { connectionId: string | null }) { + return ( +
+
+ {connectionId ? : null} + + Pause collecting + +
+
+ ); +} + +function PausedHistoricalArchiveSection({ credentialUpdateHref }: { credentialUpdateHref: string | null }) { + if (!credentialUpdateHref) { + return null; + } + return ( +
+ + Reconnect + +
+ ); +} + function RevokedConnectionSection({ connectorId, revokedAt }: { connectorId: string; revokedAt: string | null }) { return (
/; const REATTACH_SCHEDULE_CONNECTOR_ID_INPUT = //; diff --git a/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts b/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts index af698fb1f..4f952ee68 100644 --- a/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts +++ b/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts @@ -17,6 +17,8 @@ export function buildAddSourceDemoCatalog(): { displayName: "ChatGPT", disposition: "static_secret_connect", externalDocs: [], + isKnownScaffold: false, + listingNote: null, modality: "api_network", nextStepKind: "capture_static_secret", publicTier: "supported", @@ -43,6 +45,8 @@ export function buildAddSourceDemoCatalog(): { displayName: "Amazon", disposition: "manual_upload_connect", externalDocs: [], + isKnownScaffold: false, + listingNote: null, modality: "api_network", nextStepKind: "provide_import_file", publicTier: "supported", @@ -62,6 +66,8 @@ export function buildAddSourceDemoCatalog(): { disposition: "local_collector_enroll", externalDocs: [], enrollmentKey: "claude_code", + isKnownScaffold: false, + listingNote: null, modality: "local_collector", nextStepKind: "enroll_local_collector", publicTier: "supported", @@ -84,6 +90,8 @@ export function buildAddSourceDemoCatalog(): { displayName: "Calendar Demo", disposition: "provider_auth_deployment_blocked", externalDocs: [], + isKnownScaffold: false, + listingNote: null, modality: "api_network", nextStepKind: "needs_deployment_config", publicTier: "development", @@ -102,6 +110,8 @@ export function buildAddSourceDemoCatalog(): { displayName: "Browser Archive Demo", disposition: "browser_bound_runbook", externalDocs: [], + isKnownScaffold: false, + listingNote: null, modality: "browser_bound", nextStepKind: "manual_runbook", publicTier: "development", @@ -113,6 +123,52 @@ export function buildAddSourceDemoCatalog(): { setupModality: "browser_bound", supportState: "proof_gated", }, + { + // Illustrates the Development disclosure's real/unproven bucket: a + // registered, implemented connector whose setup path resolves to a + // real action, but no live run has proven it yet. + acquisitionPaths: [], + connectorKey: "podcast_history_demo", + deploymentReadiness: { blockers: [], guidance: null, state: "ready" }, + displayName: "Podcast History Demo", + disposition: "static_secret_connect", + externalDocs: [], + isKnownScaffold: false, + listingNote: "Hidden from the reference dashboard catalog until a credentialed run proves useful records in the deployment.", + modality: "api_network", + nextStepKind: "capture_static_secret", + publicTier: "development", + proofGate: null, + refreshPolicyRationale: null, + runbookPath: null, + setupDescription: null, + setupHelpText: null, + setupModality: "static_secret", + supportState: "supported", + }, + { + // Illustrates the Development disclosure's scaffold bucket: a + // registered manifest whose collector is an unconditional SKIP_RESULT + // placeholder. Never gets an add action, in the disclosure or anywhere else. + acquisitionPaths: [], + connectorKey: "receipts_scaffold_demo", + deploymentReadiness: { blockers: [], guidance: null, state: "ready" }, + displayName: "Receipts Scaffold Demo", + disposition: "browser_bound_runbook", + externalDocs: [], + isKnownScaffold: true, + listingNote: null, + modality: "browser_bound", + nextStepKind: "manual_runbook", + publicTier: "development", + proofGate: "browser_collector_live_proof_missing", + refreshPolicyRationale: null, + runbookPath: null, + setupDescription: null, + setupHelpText: null, + setupModality: "browser_bound", + supportState: "proof_gated", + }, ], existingSourcesByConnector: { amazon: [ diff --git a/apps/console/src/app/(console)/sources/connector-detail-credential-routing.invariants.test.ts b/apps/console/src/app/(console)/sources/connector-detail-credential-routing.invariants.test.ts index fa4b252f5..269fe49b7 100644 --- a/apps/console/src/app/(console)/sources/connector-detail-credential-routing.invariants.test.ts +++ b/apps/console/src/app/(console)/sources/connector-detail-credential-routing.invariants.test.ts @@ -59,6 +59,34 @@ test("detail-page repair routing is connection-binding-first (session repair bef assert.match(src, STATIC_SECRET_UPDATE_LINK_VISIBLE); }); +// A recovered historical-archive row (paused, never revoked) surfaces ONE +// reconnect notice, reusing the SAME `credentialUpdateHref` repair routes +// resolve above — no separate resume button/action (a recovered row typically +// has no surviving credential; resuming without one would just fail on the +// next run). +// The gate is now expressed in two steps — a general `paused` lifecycle flag +// (shared with the plain Resume action, which every OTHER paused row gets) +// narrowed by the archive binding kind. The invariant is unchanged: an +// archive row is gated on being paused, non-revoked, AND historical_archive. +const PAUSED_GATE = /const paused = !revoked && overview\.connectionStatus === "paused"/; +const PAUSED_HISTORICAL_ARCHIVE_GATE = + /const pausedHistoricalArchive = paused && sourceBindingKind === "historical_archive"/; +const PAUSED_HISTORICAL_ARCHIVE_SECTION_RENDERED = + /pausedHistoricalArchive \? : null/; +const PAUSED_HISTORICAL_ARCHIVE_SECTION_HAS_NO_RESUME_ACTION = + /function PausedHistoricalArchiveSection\(\{ credentialUpdateHref \}: \{ credentialUpdateHref: string \| null \}\) \{[\s\S]{0,400}href=\{credentialUpdateHref\}/; +// A paused row that is NOT a recovered archive gets the plain Resume action; +// an actively-collecting row gets Pause. Both post to their owner-session +// server action, so the console's pause/resume cycle is closed. +const PAUSED_RESUMABLE_GATE = /const pausedResumable = paused && !pausedHistoricalArchive/; +const PAUSED_CONNECTION_SECTION_RENDERED = + /pausedResumable \? : null/; +const PAUSED_CONNECTION_SECTION_POSTS_RESUME = + /function PausedConnectionSection\([\s\S]{0,600}
/; +const PAUSABLE_GATE = /const pausable = !\(revoked \|\| paused\) && overview\.connectionStatus === "active"/; +const PAUSE_CONNECTION_SECTION_POSTS_PAUSE = + /function PauseConnectionSection\([\s\S]{0,600}/; + test("detail-page rendered reauth routes and labels by the server-owned action surface", async () => { const src = await readFile(DETAIL_PAGE, "utf8"); assert.match(src, PRIMARY_ACTION_SURFACE_READ); @@ -68,3 +96,26 @@ test("detail-page rendered reauth routes and labels by the server-owned action s assert.match(src, STORED_CREDENTIAL_COPY_IS_UPDATE); assert.match(src, REAUTH_FALLBACK_FOR_OLD_PAYLOADS); }); + +test("a recovered historical-archive row surfaces one reconnect notice, no separate resume action", async () => { + const src = await readFile(DETAIL_PAGE, "utf8"); + assert.match(src, PAUSED_GATE); + assert.match(src, PAUSED_HISTORICAL_ARCHIVE_GATE); + assert.match(src, PAUSED_HISTORICAL_ARCHIVE_SECTION_RENDERED); + // Still the archive journey's ONLY action: repair the credential. A bare + // resume would flip the row to active and then fail on the next run, + // because a recovered archive typically carries no surviving credential. + assert.match(src, PAUSED_HISTORICAL_ARCHIVE_SECTION_HAS_NO_RESUME_ACTION); +}); + +// The complement of the archive case: every OTHER paused row — notably one +// the owner paused deliberately — must get a real Resume action, or pausing +// from the console would be a one-way door. +test("a non-archive paused row surfaces a real resume action, and an active row a pause action", async () => { + const src = await readFile(DETAIL_PAGE, "utf8"); + assert.match(src, PAUSED_RESUMABLE_GATE); + assert.match(src, PAUSED_CONNECTION_SECTION_RENDERED); + assert.match(src, PAUSED_CONNECTION_SECTION_POSTS_RESUME); + assert.match(src, PAUSABLE_GATE); + assert.match(src, PAUSE_CONNECTION_SECTION_POSTS_PAUSE); +}); diff --git a/apps/console/src/app/(console)/sources/error.tsx b/apps/console/src/app/(console)/sources/error.tsx index bd456b0e1..139d81962 100644 --- a/apps/console/src/app/(console)/sources/error.tsx +++ b/apps/console/src/app/(console)/sources/error.tsx @@ -3,126 +3,137 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { buttonVariants } from "@pdpp/brand-react"; import { useEffect, useState } from "react"; +import { ListLoadingSkeleton } from "../components/route-loading.tsx"; import { readLastRecordsReadAt } from "./last-known-read.ts"; /** - * Records-segment error boundary (App Router convention) — partial-aware. + * Sources-segment error boundary (App Router convention) — SLVP bar: Stripe, + * Linear, Vercel, and Plaid never tell an owner "we hit a transient read + * interruption, retrying." The page renders, or it quietly shows last-known + * state. The owner never learns the backend hiccuped. * - * The owner reported hitting "Couldn't load your connections" — every card - * gone — during a reference rebuild, when a transient read failed mid - * `router.refresh()` (a poll tick or a post-Sync revalidation; - * `records-page-poller.tsx` / `connector-row.tsx`). A read blip at the very - * moment the owner most wants the page (ChatGPT consuming deployment resources - * mid-run) should never blank all 19 cards. + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`, digest-stamped by Next): the read itself is fine. The + * `/_ref/connectors` read this page depends on returns 200 in ~1.2s — the read + * already succeeded. The throw is React's Flight/RSC streaming writer + * (`react-server-dom-webpack-server`) reacting to the destination (the HTTP + * response) closing before the stream finished flushing — e.g. a poll tick + * from `records-page-poller.tsx` firing `router.refresh()` while a prior + * refresh's stream is still in flight, or the tab backgrounding/throttling the + * connection mid-render. It is a client-transport race below the data layer, + * not a backend outage, so there is no "fix the read" available here — the + * fetch already succeeded by the time this fires. * - * So this boundary is NOT a full-viewport takeover. It renders a compact, - * top-anchored banner that: - * - frames the failure honestly as a *read* failure, not a data change; - * - names *when* the data was last confirmed live (last-successful-load - * timestamp, read from the client-side `sessionStorage` marker the poller - * stamps — see `last-known-read.ts`), without claiming cached rows exist; - * - offers an explicit Retry; and - * - quietly auto-retries once after a short delay, so a transient blip - * self-heals back to the live list without the owner lifting a finger. + * Given that, this boundary NEVER renders owner-facing failure copy, at any + * stage. It: + * - retries immediately and then on a capped exponential backoff, + * UNBOUNDED — there is no terminal "give up and show a Retry button" + * state, because a manual-retry dead end is itself the thing the owner + * complained about (parked on an error for an hour); + * - while retrying, renders the exact same skeleton `loading.tsx` uses, so + * a transient stream-teardown is visually indistinguishable from a normal + * page load — never a warning-colored card, never the words "error", + * "interruption", "retrying", "couldn't", or "failed"; + * - once a last-good render has happened at least once, adds ONLY a quiet, + * dimmed "Updated Xs/Xm ago" caption under the skeleton — honest staleness + * signal, framed exactly like a normal freshness note elsewhere in the + * product, never framed as a failure. * * Self-contained on purpose (mirrors `dashboard/error.tsx`): a `"use client"` * boundary must not import server-only modules, since the dashboard shell * transitively pulls in `lib/owner-token.ts` (`server-only`). The last-known - * snapshot therefore comes from a client-cached marker, never a server read - * inside the boundary. See https://nextjs.org/docs/app/getting-started/error-handling. + * timestamp therefore comes from a client-cached marker (`last-known-read.ts`), + * never a server read inside the boundary. See + * https://nextjs.org/docs/app/getting-started/error-handling. */ -// How long to wait before the single automatic recovery attempt. Long enough to -// let a transient reference rebuild / 500 clear, short enough to feel like a -// self-healing page rather than a stuck one. -const AUTO_RETRY_DELAY_MS = 4000; +/** First retry is near-immediate — long enough to dodge a tight synchronous loop. */ +const RETRY_BASE_DELAY_MS = 300; +/** Backoff ceiling: keep retrying at a calm, bounded cadence forever rather than escalating without limit. */ +const RETRY_MAX_DELAY_MS = 15_000; -function formatLastKnown(at: number | null): string | null { +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state. React remounts this component fresh every time `reset()` triggers + * another catch (a new error instance re-enters the boundary), so a + * `useState` counter would silently reset to 0 on every failure and the + * backoff would never actually grow past its base delay. A page navigation / + * hard reload naturally resets this module's state, which is the right + * lifetime: "how many times has this boundary caught in a row since the page + * was last freshly loaded." + */ +let consecutiveAttempts = 0; + +/** Capped exponential backoff. Never returns a delay the owner would perceive as "given up". */ +function nextRetryDelayMs(attempt: number): number { + const scaled = RETRY_BASE_DELAY_MS * 2 ** attempt; + return Math.min(scaled, RETRY_MAX_DELAY_MS); +} + +/** Quiet, second/minute-granularity "how long ago" — no day-scale rounding, this page polls every few seconds. */ +function formatUpdatedAgo(at: number | null, nowMs: number): string | null { if (at === null) { return null; } - try { - return new Date(at).toLocaleString(); - } catch { + const deltaMs = nowMs - at; + if (!Number.isFinite(deltaMs) || deltaMs < 0) { return null; } + if (deltaMs < 5000) { + return "Updated just now"; + } + if (deltaMs < 60_000) { + return `Updated ${Math.round(deltaMs / 1000)}s ago`; + } + const minutes = Math.round(deltaMs / 60_000); + return `Updated ${minutes}m ago`; } -export default function RecordsError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { - const [lastKnown, setLastKnown] = useState(null); - const [autoRetried, setAutoRetried] = useState(false); +export default function SourcesError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + const [lastKnownAt, setLastKnownAt] = useState(null); + const [updatedAgoLabel, setUpdatedAgoLabel] = useState(null); useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. console.error(error); - // Read the client-cached last-good timestamp on mount (sessionStorage is - // unavailable during SSR, so this stays in an effect). - setLastKnown(formatLastKnown(readLastRecordsReadAt())); + setLastKnownAt(readLastRecordsReadAt()); }, [error]); useEffect(() => { - // One automatic recovery attempt: re-run the segment render after a short - // delay so a transient read failure clears itself. If the read still fails - // the boundary re-mounts and the owner is left with the manual Retry — we - // never loop, so a persistent failure does not thrash the deployment. - if (autoRetried) { - return; - } + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // `consecutiveAttempts` counter. There is deliberately no ceiling on the + // counter itself — a persistent failure degrades to a slow quiet + // heartbeat, never to a dead end. + const delay = nextRetryDelayMs(consecutiveAttempts); const id = setTimeout(() => { - setAutoRetried(true); + consecutiveAttempts += 1; reset(); - }, AUTO_RETRY_DELAY_MS); + }, delay); return () => clearTimeout(id); - }, [autoRetried, reset]); - - const lastKnownLine = lastKnown ? `Last successful load: ${lastKnown}.` : "The last successful load time is unknown."; + }, [reset]); - if (!autoRetried) { - return ( -
-

Refreshing source status

-

- The Sources view hit a transient read interruption. Retrying automatically before showing an error.{" "} - {lastKnownLine} -

-
- ); - } + useEffect(() => { + // Recompute the relative-time caption independently of the retry timer so + // it stays live (e.g. "Updated 3s ago" ticking up) even between retries. + if (lastKnownAt === null) { + return; + } + const tick = () => setUpdatedAgoLabel(formatUpdatedAgo(lastKnownAt, Date.now())); + tick(); + const id = setInterval(tick, 1000); + return () => clearInterval(id); + }, [lastKnownAt]); return ( -
-

Couldn't refresh your connections

-

- The Sources view hit an error reading from your reference deployment. Your data and connections are unaffected — - this is a read failure, not a change. {lastKnownLine} -

-
-
+
+ + {updatedAgoLabel ? ( +

+ {updatedAgoLabel} +

+ ) : null} +
); } diff --git a/apps/console/src/app/(console)/sources/page.tsx b/apps/console/src/app/(console)/sources/page.tsx index d24f4aa74..737c55f9e 100644 --- a/apps/console/src/app/(console)/sources/page.tsx +++ b/apps/console/src/app/(console)/sources/page.tsx @@ -70,7 +70,13 @@ async function resolveHost(): Promise { } function fetchSourcesPage(pageState: ReturnType) { - return loadConnectorSummaryPage(pageState, (opts) => liveDashboardDataSource.listConnectorSummaries(opts)); + // `sourcesVisibility: true` asks the reference to exclude a pure recovered + // historical fragment BEFORE its own LIMIT, so `hasMore`/the next cursor + // stay correct for the rows this page actually renders — never a + // post-LIMIT filter. Explore and every other surface omit this flag. + return loadConnectorSummaryPage(pageState, (opts) => + liveDashboardDataSource.listConnectorSummaries({ ...opts, sourcesVisibility: true }) + ); } export default async function RecordsIndexPage({ diff --git a/apps/console/src/app/(console)/sources/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/sources/read-resilience.invariants.test.ts index dceb631e2..618ed0157 100644 --- a/apps/console/src/app/(console)/sources/read-resilience.invariants.test.ts +++ b/apps/console/src/app/(console)/sources/read-resilience.invariants.test.ts @@ -2,26 +2,57 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Read-resilience acceptance invariants for the records segment [Defect 3]. + * Read-resilience acceptance invariants for the sources segment [Defect 3, + * revised]. The prior version of this file pinned a banner that told the + * owner "Refreshing source status… The Sources view hit a transient read + * interruption. Retrying automatically before showing an error." — and, on a + * second failure, "Couldn't refresh your connections" behind a manual Retry + * button. The owner reported that failure-copy state sitting on screen for an + * hour and said, verbatim: "I should never see this" / "/sources should + * always work." * - * The owner hit "Couldn't load your connections" — all 19 cards gone — during a - * reference rebuild, when a transient read failed mid `router.refresh()`. These - * pin the fix so it cannot regress to a full-viewport blank: + * THE STANDARD (stated explicitly by the owner): Stripe, Linear, Vercel, and + * Plaid never show a user "we hit a transient read interruption, retrying." + * The page renders, or it shows last-known state with a quiet staleness + * indicator. The user never learns the backend hiccuped. That is the MINIMUM + * bar, not the target. * - * 1. The records error boundary first renders quiet retrying copy, not an - * explicit failure headline, and the eventual failure state is still a - * partial banner rather than a full-viewport takeover. - * 2. The boundary reads a CLIENT-cached last-known marker (it must not import - * a server-only module) and surfaces last-known status + a retry. - * 3. The boundary auto-retries once so a transient blip self-heals. - * 4. The poller stamps the last-good read time and guards the soft - * revalidation so a throw never escapes the timer. + * Root cause the boundary now assumes: `Error: The destination stream closed + * early` is React's RSC streaming writer reacting to the HTTP response socket + * closing before the flight stream finished flushing (a poll-driven + * `router.refresh()` superseding an in-flight one, or the tab + * backgrounding/throttling the connection) — NOT a failed data read. The + * upstream `/_ref/connectors` call already returned 200. So this boundary + * treats every activation as recoverable-by-construction: it never renders + * error-shaped copy, retries unbounded on a capped backoff, and — while + * retrying — is visually identical to the ordinary `loading.tsx` skeleton, + * with at most a quiet "Updated Xs/Xm ago" caption once a last-known + * timestamp exists. + * + * These invariants pin the STRONGER property (no error copy ever reaches the + * owner, no terminal manual-retry dead end) rather than merely deleting the + * old, weaker pin: + * + * 1. The boundary source contains NONE of the retired failure-ish copy + * ("error", "interruption", "retrying", "couldn't", "failed", "wrong") + * in any owner-facing string. + * 2. The boundary renders the SAME `ListLoadingSkeleton` component + * `loading.tsx` uses, not a bespoke banner/card — so the transient state + * is indistinguishable from an ordinary page load. + * 3. The boundary retries on every mount (no `autoRetried`/"give up" flag) + * with a capped, growing backoff, and never renders a manual "Retry" + * button or link as a terminal state. + * 4. The boundary reads the CLIENT-cached last-known marker (it must not + * import a server-only module) and, when present, renders only a quiet, + * dimmed relative-time caption — never a claim that failed data changed. + * 5. The poller still stamps the last-good read time and guards the soft + * revalidation so a throw never escapes the timer (unchanged contract). * * Source-regex over the shipped client components, mirroring the existing * records-list-view / sources-ia invariant style: these are `"use client"` * React components that the behavioral marker logic (last-known-read.test.ts) - * already covers as a pure unit; here we pin the boundary's structural copy and - * the load-bearing wiring from source. + * already covers as a pure unit; here we pin the boundary's structural copy + * and the load-bearing wiring from source. */ import assert from "node:assert/strict"; @@ -35,67 +66,86 @@ const POLLER_FILE = `${HERE}records-page-poller.tsx`; const MARKER_FILE = `${HERE}last-known-read.ts`; // Regexes hoisted to module scope (project lint: useTopLevelRegex). -const BANNER_TESTID_RE = /data-testid="records-read-failure-banner"/; -const PENDING_TESTID_RE = /data-testid="records-read-retry-pending"/; -const PENDING_COPY_RE = /Refreshing source status/; -const FAILURE_HEADLINE_RE = /Couldn't refresh your connections/; -const FAILURE_GATED_AFTER_RETRY_RE = /if\s*\(!autoRetried\)[\s\S]*records-read-retry-pending[\s\S]*return\s*\(/; -const FULL_VIEWPORT_TAKEOVER_RE = /min-h-\[60vh\]/; -const READ_FAILURE_FRAMING_RE = /read failure, not a change/; +// +// The owner-facing-string bans below intentionally allow the word "error" to +// appear as a JS identifier (the boundary prop is literally named `error`, +// per the Next.js `error.tsx` contract) but forbid it inside rendered JSX +// text content. We assert on the specific retired phrases rather than +// banning "error" as a bare substring, since the prop name and the +// `console.error(error)` diagnostics call are legitimate and must remain. +const RETIRED_PENDING_COPY_RE = /Refreshing source status/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_RETRYING_COPY_RE = /Retrying automatically/i; +const RETIRED_FAILURE_HEADLINE_RE = /Couldn't refresh your connections/; +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i; +const RETIRED_BANNER_TESTID_RE = /data-testid="records-read-failure-banner"/; +const RETIRED_PENDING_TESTID_RE = /data-testid="records-read-retry-pending"/; +const RETIRED_RETRY_TESTID_RE = /data-testid="records-read-failure-retry"/; +const RETIRED_MANUAL_RETRY_LABEL_RE = /Retry now|Reload Sources/; +const RETIRED_AUTO_RETRIED_GUARD_RE = /autoRetried/; + +const USES_LOADING_SKELETON_RE = / { +test("the boundary source contains none of the retired owner-facing failure copy", async () => { const src = await readFile(ERROR_FILE, "utf8"); - assert.match(src, PENDING_TESTID_RE); - assert.match(src, PENDING_COPY_RE); - assert.match(src, FAILURE_HEADLINE_RE); - assert.match(src, FAILURE_GATED_AFTER_RETRY_RE); + assert.doesNotMatch(src, RETIRED_PENDING_COPY_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_RETRYING_COPY_RE); + assert.doesNotMatch(src, RETIRED_FAILURE_HEADLINE_RE); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BANNER_TESTID_RE); + assert.doesNotMatch(src, RETIRED_PENDING_TESTID_RE); + assert.doesNotMatch(src, RETIRED_RETRY_TESTID_RE); + assert.doesNotMatch(src, RETIRED_MANUAL_RETRY_LABEL_RE); + assert.doesNotMatch( + src, + RETIRED_AUTO_RETRIED_GUARD_RE, + "no gated one-shot auto-retry flag — retry must be unbounded" + ); }); -test("the records persistent-failure state is a partial banner, not a full-viewport blank", async () => { +test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => { const src = await readFile(ERROR_FILE, "utf8"); - // A banner (section/role=status), not the full-height centered takeover the - // generic segment-error shell uses. - assert.match(src, BANNER_TESTID_RE); - assert.doesNotMatch(src, FULL_VIEWPORT_TAKEOVER_RE, "the records boundary must not be a full-viewport takeover"); - // Honest framing: a read failure, not a data change. - assert.match(src, READ_FAILURE_FRAMING_RE); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); }); -test("the boundary surfaces last-known status from a client-cached marker, never a server read", async () => { +test("the boundary surfaces only a quiet relative-time caption from a client-cached marker, never a server read", async () => { const src = await readFile(ERROR_FILE, "utf8"); // Reads the client-side marker… assert.match(src, READS_MARKER_HELPER_RE); assert.match(src, IMPORTS_MARKER_RE); - // …and reports the last successful load without claiming to render cached - // source rows. - assert.match(src, LAST_SUCCESSFUL_LOAD_COPY_RE); - assert.doesNotMatch(src, OVERCLAIMED_LAST_KNOWN_COPY_RE); + // …and, when present, renders it as a quiet "Updated …" caption, never + // failure-framed copy. + assert.match(src, QUIET_UPDATED_CAPTION_RE); // Self-contained: no server-only module is *imported* into the boundary. // (The doc comment may name `server-only` to explain why it is avoided; we // scan import statements, not prose.) assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); }); -test("the boundary offers a retry and auto-recovers once", async () => { +test("the boundary retries unbounded on a capped backoff, with no manual-retry terminal state", async () => { const src = await readFile(ERROR_FILE, "utf8"); - assert.match(src, RETRY_TESTID_RE); assert.match(src, CALLS_RESET_RE); - // A single automatic recovery attempt (guarded so it never loops). - assert.match(src, AUTO_RETRY_DELAY_RE); - assert.match(src, AUTO_RETRIED_GUARD_RE); + // Every mount schedules the next retry — no gate that stops after N tries. + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, BACKOFF_CAP_RE); }); test("the poller stamps the last-good read time and guards the soft revalidation", async () => { diff --git a/apps/console/src/app/(console)/sources/sources-cockpit.invariants.test.ts b/apps/console/src/app/(console)/sources/sources-cockpit.invariants.test.ts index 8995eced0..972fb79fd 100644 --- a/apps/console/src/app/(console)/sources/sources-cockpit.invariants.test.ts +++ b/apps/console/src/app/(console)/sources/sources-cockpit.invariants.test.ts @@ -33,8 +33,16 @@ const OLD_TOUCHING_PADDING_RE = /\.rr-s-item\s*\{[\s\S]*?padding:\s*10px\s+0\s+1 const STATE_GEOMETRY_RE = /\.rr-s-item(?:\.|[^\n{])*(?:degraded|attention|warning)[^{]*\{[\s\S]*?(?:margin|width|border-radius)\s*:/i; const COLLECTION_REPORT_INDEX_RE = /indexCollectionReportByStream\(summary\.collection_report\)/; -const DUPLICATE_COLLAPSE_RE = /collapseDuplicateFallbackSources\(instances\)/; +// Collapse runs over the LIVE rows, not every row: an archived source must +// not be pulled into a duplicate group, whose ordinal relabelling +// ("account 2") would imply it belongs to a live sibling set it has left. +const DUPLICATE_COLLAPSE_RE = /collapseDuplicateFallbackSources\(liveInstances\)/; const DUPLICATE_GROUP_TESTID_RE = /data-testid="sources-duplicate-group"/; +const ARCHIVED_GROUP_TESTID_RE = /data-testid="sources-archived-group"/; +// The group heading must say outright that these are not collecting, so a +// collapsed group can never be mistaken for healthy live sources. +const ARCHIVED_GROUP_LABEL_RE = /Archived — not collecting \(\{archivedInstances\.length\}\)/; +const ARCHIVED_PARTITION_RE = /instances\.filter\(\(i\) => i\.archived\)/; const FACTS_UNAVAILABLE_COPY_RE = /Collection facts not available yet/; const RECORDS_HEADER_RE = /records<\/TableHeader>/; const STREAM_RECORDS_RE = /summary\.stream_records/; @@ -88,6 +96,13 @@ test("repeated unnamed same-type sources are collapsed into a review group", asy assert.match(view, DUPLICATE_GROUP_TESTID_RE); }); +test("archived sources render in their own group, labelled as not collecting", async () => { + const view = await readFile(VIEW_FILE, "utf8"); + assert.match(view, ARCHIVED_PARTITION_RE); + assert.match(view, ARCHIVED_GROUP_TESTID_RE); + assert.match(view, ARCHIVED_GROUP_LABEL_RE); +}); + test("source passport suppresses generic sync for non-owner verdict actions", async () => { const view = await readFile(VIEW_FILE, "utf8"); assert.match(view, PRIMARY_VERDICT_ACTION_RE); @@ -104,3 +119,31 @@ test("source list shows advisory owner-action cues as non-mutating review copy", assert.doesNotMatch(cueElement, BUTTON_TAG_RE); assert.doesNotMatch(cueElement, MUTATING_ACTION_RE); }); + +// The fused status line must actually reach the owner's eyes. Before it landed, +// the row's only status signal was a colored glyph whose label was `sr-only`, +// so a sighted owner could not tell fresh from stale, or syncing from stuck, +// without opening the detail panel. See +// design-notes/fused-source-status-2026-08-22.md. +const FUSED_STATUS_TESTID_RE = /data-testid="sources-fused-status"/; +const FUSED_STATUS_RENDERS_LINE_RE = /\{instance\.fusedStatus\.line\}/; +const FUSED_STATUS_TONE_RE = /data-tone=\{instance\.fusedStatus\.tone\}/; +// The old sr-only duplicate: with the status visible, keeping it would make +// screen readers announce every row's status twice. +const SR_ONLY_STATUS_DUPLICATE_RE = /\{instance\.status\.label\}<\/span>/; + +test("the source row renders the fused status line visibly", async () => { + const view = await readFile(VIEW_FILE, "utf8"); + assert.match(view, FUSED_STATUS_TESTID_RE); + assert.match(view, FUSED_STATUS_RENDERS_LINE_RE); + assert.match(view, FUSED_STATUS_TONE_RE); +}); + +test("the visible fused status replaces the sr-only status rather than doubling it", async () => { + const view = await readFile(VIEW_FILE, "utf8"); + assert.doesNotMatch( + view, + SR_ONLY_STATUS_DUPLICATE_RE, + "the status is now visible text, so the sr-only copy would be announced twice" + ); +}); diff --git a/apps/console/src/app/(console)/sources/sources-view-model.test.ts b/apps/console/src/app/(console)/sources/sources-view-model.test.ts index 3a2c166a7..28a8a1e4d 100644 --- a/apps/console/src/app/(console)/sources/sources-view-model.test.ts +++ b/apps/console/src/app/(console)/sources/sources-view-model.test.ts @@ -25,6 +25,7 @@ import { buildSourcesChurnAdvisory, buildSourcesRuntimeAdvisory, collapseDuplicateFallbackSources, + collapseSetupFailedSources, exploreHrefFor, formatSchedule, manualUploadHrefForSource, @@ -143,10 +144,10 @@ function passportField(view: ReturnType, key: strin } test("deriveRenderedSourceStatus prefers the server-owned verdict over raw health state", () => { - const flag = deriveRenderedSourceStatus(renderedVerdict({ pill: { label: "Degraded", tone: "amber" } }), false); + const flag = deriveRenderedSourceStatus(renderedVerdict({ pill: { label: "Missing data", tone: "amber" } }), false); assert.equal(flag.kind, "degraded"); assert.equal(flag.tone, "warning"); - assert.equal(flag.label, "Degraded"); + assert.equal(flag.label, "Missing data"); }); test("deriveRenderedSourceStatus carries freshness annotations from rendered verdict", () => { @@ -333,7 +334,7 @@ test("toSourceInstanceView does not render maintainer or wait actions as owner C { affects: [], audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -420,7 +421,7 @@ test("toSourceInstanceView renders calibrated live-journey verdict copy without display_name: "Chase", rendered_verdict: renderedVerdict({ annotations: [{ kind: "freshness", text: "Transactions stuck since Apr 22." }], - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ { affects: ["transactions"], @@ -446,7 +447,7 @@ test("toSourceInstanceView renders calibrated live-journey verdict copy without }), }) ); - assert.equal(chase.status.label, "Degraded · Transactions stuck since Apr 22."); + assert.equal(chase.status.label, "Missing data · Transactions stuck since Apr 22."); assert.equal(chase.nextAction, null); assert.equal(chase.primaryVerdictAction?.cta, "Retry now"); }); @@ -859,6 +860,335 @@ test("toSourcesView disambiguates duplicate unnamed connections without exposing assert.equal(views[2]?.listKind, null); }); +test("toSourcesView renders a pure recovered historical fragment as an archived row", () => { + const views = toSourcesView([ + summary({ + connection_id: "cin_fragment", + connector_id: "chase", + display_name: "Chase", + source_visibility: "archived", + }), + ]); + + assert.equal(views.length, 1, "an archived source must still appear — its records exist and must be reachable"); + assert.equal(views[0]?.archived, true, "it must be classified archived, not rendered as a live source"); +}); + +test("toSourcesView accepts the retired hidden_from_sources spelling as archived", () => { + // A console deployed ahead of its reference still receives the old value. + // It must classify as archived, never fall through to a live-looking row. + const views = toSourcesView([ + summary({ + connection_id: "cin_fragment", + connector_id: "chase", + display_name: "Chase", + source_visibility: "hidden_from_sources", + }), + ]); + + assert.equal(views.length, 1); + assert.equal(views[0]?.archived, true, "the retired spelling must not read as a live source"); +}); + +test("an archived source never renders a healthy status, even with a green stored verdict", () => { + // The fabricated-green guard: an archived source's LAST run may have + // succeeded, so its stored verdict can still be green. Rendering that tone + // would tell the owner a source that will never collect again is healthy. + const views = toSourcesView([ + summary({ + connection_id: "cin_fragment", + connector_id: "chase", + display_name: "Chase", + rendered_verdict: renderedVerdict({ pill: { label: "Healthy", tone: "green" } }), + source_visibility: "archived", + }), + ]); + + assert.equal(views[0]?.status.kind, "archived", "a green verdict must not survive archival"); + assert.notEqual(views[0]?.status.tone, "success", "an archived source must never render a success tone"); +}); + +test("an archived source offers no Reconnect action — the prompt that leads nowhere", () => { + // The verdict carries a real owner-satisfiable Reconnect action, exactly as + // a fragment's stored verdict does. Reconnecting mints a NEW connection and + // resumes nothing here, so no surface may offer it (the intent dfbbb8843 + // established). Without a genuinely actionable verdict this assertion would + // pass vacuously, so the fixture must carry one. + const reconnectVerdict = renderedVerdict({ + channel: "attention", + pill: { label: "Can't collect", tone: "red" }, + required_actions: [ + { + affects: [], + audience: "owner", + cta: "Reconnect this account", + kind: "reauth", + satisfied_when: { kind: "credential_present_and_unrejected" }, + terminal: false, + urgency: "soon", + }, + ], + }); + + const live = toSourcesView([ + summary({ connection_id: "cin_live", rendered_verdict: reconnectVerdict, source_visibility: "active" }), + ]); + assert.equal( + live[0]?.primaryVerdictAction?.cta, + "Reconnect this account", + "control: a LIVE source with this verdict does surface the action" + ); + + const archived = toSourcesView([ + summary({ connection_id: "cin_archived", rendered_verdict: reconnectVerdict, source_visibility: "archived" }), + ]); + assert.equal(archived[0]?.primaryVerdictAction, null, "an archived source must not surface a Reconnect action"); + assert.equal(archived[0]?.nextAction, null, "nor as a body CTA"); + assert.equal(archived[0]?.ownerActionCue, null, "nor as a list-row cue"); +}); + +test("toSourcesView classifies a never-succeeded revoked setup shell as setupFailed, not archived or a live revoked row", () => { + const views = toSourcesView([ + summary({ + connection_id: "cin_venmo_shell", + connector_id: "venmo", + display_name: "Venmo", + revoked_at: "2026-08-21T15:42:36.412Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + ]); + + assert.equal(views.length, 1, "a setup-failed source must still appear — it is the owner's only record of trying"); + assert.equal(views[0]?.setupFailed, true); + assert.equal(views[0]?.archived, false, "setup-failed is distinct from archived — nothing was ever collected"); +}); + +test("a setup-failed source carries the server's specific forward_statement (quiet-expiry defect fix, owner ruling 2026-08-22)", () => { + const views = toSourcesView([ + summary({ + connector_id: "venmo", + rendered_verdict: renderedVerdict({ + forward_statement: + "This setup attempt expired while waiting for you to finish signing in. No records were collected. Start a fresh attempt when you're ready.", + }), + revoked_at: "2026-08-21T15:42:36.412Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + ]); + + assert.equal( + views[0]?.setupFailedForwardStatement, + "This setup attempt expired while waiting for you to finish signing in. No records were collected. Start a fresh attempt when you're ready.", + "the console must render the server's TTL-specific sentence, not a generic fallback" + ); +}); + +test("setupFailedForwardStatement is null for a non-setup-failed source, even if rendered_verdict carries a forward_statement", () => { + const views = toSourcesView([ + summary({ + connector_id: "venmo", + rendered_verdict: renderedVerdict({ forward_statement: "Collection is current." }), + status: "active", + total_records: 10, + }), + ]); + + assert.equal( + views[0]?.setupFailedForwardStatement, + null, + "the field must only ever surface for a setup-failed row, never leak an unrelated verdict's prose" + ); +}); + +test("a setup-failed source never renders a healthy status, even with a green stored verdict", () => { + const views = toSourcesView([ + summary({ + connector_id: "venmo", + rendered_verdict: renderedVerdict({ pill: { label: "Healthy", tone: "green" } }), + revoked_at: "2026-08-21T15:42:36.412Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + ]); + + assert.equal(views[0]?.status.kind, "setup_failed", "a green verdict must not survive setup-failure classification"); + assert.notEqual(views[0]?.status.tone, "success", "a setup-failed source must never render a success tone"); +}); + +test("a setup-failed source offers no action — no Reconnect, no Try-again CTA on the row, no list cue", () => { + const reconnectVerdict = renderedVerdict({ + channel: "attention", + pill: { label: "Can't collect", tone: "red" }, + required_actions: [ + { + affects: [], + audience: "owner", + cta: "Reconnect this account", + kind: "reauth", + satisfied_when: { kind: "credential_present_and_unrejected" }, + terminal: false, + urgency: "soon", + }, + ], + }); + + const setupFailed = toSourcesView([ + summary({ + connector_id: "venmo", + rendered_verdict: reconnectVerdict, + revoked_at: "2026-08-21T15:42:36.412Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + ]); + assert.equal( + setupFailed[0]?.primaryVerdictAction, + null, + "a setup-failed source must not surface a Reconnect action — there is no connection to reconnect" + ); + assert.equal(setupFailed[0]?.nextAction, null, "nor as a body CTA — the row's own detail page has nothing to offer"); + assert.equal(setupFailed[0]?.ownerActionCue, null, "nor as a list-row cue"); +}); + +test("collapseSetupFailedSources coalesces every setup-failed attempt for a connector into one representative row", () => { + const views = toSourcesView([ + summary({ + connection_id: "cin_venmo_1", + connector_id: "venmo", + display_name: "Venmo", + revoked_at: "2026-08-21T15:42:36.412Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + summary({ + connection_id: "cin_venmo_2", + connector_id: "venmo", + display_name: "Venmo", + revoked_at: "2026-08-21T22:22:07.510Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + summary({ + connection_id: "cin_venmo_3", + connector_id: "venmo", + display_name: "Venmo", + revoked_at: "2026-08-22T02:40:11.560Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + ]); + + assert.equal(views.length, 3, "toSourcesView itself does not merge rows — every attempt is still classified"); + + const { setupFailedGroups } = collapseSetupFailedSources(views); + assert.equal(setupFailedGroups.length, 1, "three attempts against the same connector collapse to one UI row"); + assert.equal(setupFailedGroups[0]?.attemptCount, 3); + assert.equal( + setupFailedGroups[0]?.representative.connectionId, + "cin_venmo_3", + "the most recent attempt (last in created_at-ordered input) is the representative" + ); +}); + +test("collapseSetupFailedSources keeps a single attempt as its own one-item group — coalescing does not require a minimum count", () => { + const views = toSourcesView([ + summary({ + connection_id: "cin_amazon_1", + connector_id: "amazon", + revoked_at: "2026-08-18T05:28:03.111Z", + source_visibility: "setup_failed", + status: "revoked", + total_records: 0, + }), + ]); + + const { setupFailedGroups } = collapseSetupFailedSources(views); + assert.equal(setupFailedGroups.length, 1); + assert.equal(setupFailedGroups[0]?.attemptCount, 1); +}); + +test("toSourcesView keeps a UAT-transferred historical_archive row visible", () => { + // Google Maps / WhatsApp UAT imports carry a historical_archive binding but + // a UAT-transfer marker — server-side `source_visibility` reads "active" + // for these, distinguishing them from a bare recovered fragment. + const views = toSourcesView([ + summary({ + connection_id: "cin_uat", + connector_id: "google_maps", + display_name: "Google Maps", + source_visibility: "active", + }), + ]); + + assert.equal(views.length, 1, "a UAT-transferred source must remain visible on Sources"); + assert.equal(views[0]?.connectionId, "cin_uat"); +}); + +test("toSourcesView keeps an active promoted connection visible", () => { + const views = toSourcesView([ + summary({ + connection_id: "cin_active", + connector_id: "gmail", + display_name: "Gmail", + source_visibility: "active", + }), + ]); + + assert.equal(views.length, 1); + assert.equal(views[0]?.connectionId, "cin_active"); +}); + +test("toSourcesView keeps a summary visible when source_visibility is absent (older reference)", () => { + const views = toSourcesView([summary({ connection_id: "cin_legacy" })]); + + assert.equal(views.length, 1, "an older reference omitting source_visibility must fail open to visible"); +}); + +test("toSourcesView classifies fragments in a mixed page while preserving order for every row", () => { + const views = toSourcesView([ + summary({ connection_id: "cin_1", connector_id: "gmail", display_name: "Gmail", source_visibility: "active" }), + summary({ + connection_id: "cin_fragment_1", + connector_id: "amazon", + display_name: "Amazon", + source_visibility: "hidden_from_sources", + }), + summary({ + connection_id: "cin_uat", + connector_id: "whatsapp", + display_name: "WhatsApp", + source_visibility: "active", + }), + summary({ + connection_id: "cin_fragment_2", + connector_id: "reddit", + display_name: "Reddit", + source_visibility: "hidden_from_sources", + }), + summary({ connection_id: "cin_2", connector_id: "chase", display_name: "Chase", source_visibility: "active" }), + ]); + + assert.deepEqual( + views.map((view) => view.connectionId), + ["cin_1", "cin_fragment_1", "cin_uat", "cin_fragment_2", "cin_2"], + "every row is kept, in input order — archived rows are classified, not dropped" + ); + assert.deepEqual( + views.filter((view) => view.archived).map((view) => view.connectionId), + ["cin_fragment_1", "cin_fragment_2"], + "exactly the fragments are archived; live rows are untouched" + ); +}); + test("duplicate source review flags same-type unnamed active sources without hiding them", () => { const views = toSourcesView([ summary({ diff --git a/apps/console/src/app/(console)/sources/sources-view-model.ts b/apps/console/src/app/(console)/sources/sources-view-model.ts index 7059a8333..7db2c5720 100644 --- a/apps/console/src/app/(console)/sources/sources-view-model.ts +++ b/apps/console/src/app/(console)/sources/sources-view-model.ts @@ -41,6 +41,7 @@ import { } from "pdpp-reference-implementation/connection-setup-plan"; import { formatStreamCollectionFacts, indexCollectionReportByStream } from "../lib/collection-report.ts"; import { isActiveConnectorRunSummaryStatus } from "../lib/connector-run-summary-status.ts"; +import type { FusedSourceStatus } from "../lib/fused-source-status.ts"; import type { FormattedNextAction } from "../lib/next-action.ts"; import type { RefConnectorRunSummary, @@ -52,7 +53,9 @@ import type { } from "../lib/ref-client.ts"; import { scheduleEnabled, scheduleIntervalSeconds } from "../lib/schedule-evidence.ts"; import { + isArchivedSource, isRevokedConnector, + isSetupFailedSource, isSetupInProgressConnector, projectSourceActionability, type SourceOwnerActionCue, @@ -130,6 +133,14 @@ export interface SourcePassportField { export interface SourceInstanceView { /** Human account/identity line for the list (display name vs. type). */ accountLine: string; + /** + * Preserved records, no collection, never resuming. Derived from the + * server's `source_visibility`. An archived source renders in its own + * Sources group and must never read as healthy or current: the list row + * shows no live status dot and offers no action implying collection could + * resume. + */ + archived: boolean; /** Stable connection selector for routing + revoke (connection_id). */ connectionId: string | null; /** Connector type id (e.g. "gmail"), used for sync + add-source. */ @@ -140,6 +151,11 @@ export interface SourceInstanceView { detailHref: string; /** Owner-facing display name (passport + list title). */ displayName: string; + /** + * The fused owner-facing status line (state · last updated · syncing now). + * Never cheerier than its worst axis; see `lib/fused-source-status.ts`. + */ + fusedStatus: FusedSourceStatus; /** Optional manifest-declared brand glyph; absent renders the Monogram fallback (see ConnectorIcon). */ icon?: SourceManifestLike["icon"]; /** Stable React key + route id. */ @@ -170,6 +186,26 @@ export interface SourceInstanceView { */ primaryVerdictAction: SourcePrimaryVerdictAction | null; revoked: boolean; + /** + * A revoked retired-setup-shell row that never had a successful run — + * repeated failed setup, zero records. Derived from the server's + * `source_visibility: "setup_failed"`. Renders in the Sources list's own + * "Setup never completed" group and, like `archived`, must never read as + * healthy or current, and offers no action implying a connection exists to + * resume — the honest next step is a fresh attempt via Add Source. + */ + setupFailed: boolean; + /** + * The server's own honest sentence for WHY this setup failed — e.g. "This + * setup attempt expired while waiting for you to finish signing in" for a + * TTL-expired shell, vs. the generic "Setup never finished" for every other + * cause. Server-derived (`archiveRenderedVerdict` in `ref-control.ts`) so + * every consumer of `rendered_verdict.forward_statement` — this passport + * pane, the `sources-report` CLI, anything else — renders the SAME + * specific reason, never a console-local guess. `null` only when + * `setupFailed` is false or the mirror predates `rendered_verdict`. + */ + setupFailedForwardStatement: string | null; /** Status flag (dot + Endorse) derived from rendered verdict, with legacy fallback. */ status: SourceStatusFlag; /** Stream manifest rows for the passport table. */ @@ -200,6 +236,19 @@ export interface DuplicateSourceGroup { total: number; } +/** + * One row per connector standing in for every `setupFailed` attempt against + * it. `representative` is the most recent attempt (rows arrive in + * `created_at ASC` order from the identity page, so the last item for a + * connector is the newest) — its status/label are what renders; the others + * exist only to be counted in `attemptCount`. + */ +export interface SetupFailedSourceGroup { + attemptCount: number; + connectorId: string; + representative: SourceInstanceView; +} + export interface SourcesRuntimeAdvisory { headline: string; note: string; @@ -503,6 +552,8 @@ export function toSourceInstanceView( // biome-ignore lint/suspicious/noUnnecessaryConditions: see comment above. const routeId = connectionId ?? connectorInstanceId ?? actionability.routeId; const revoked = isRevokedConnector(summary); + const archived = isArchivedSource(summary); + const setupFailed = isSetupFailedSource(summary); // Modality is persisted server authority. A missing heartbeat must not // resurrect remote Sync controls for a local-device connection. const isLocalDevicePush = summary.source_kind === "local_device"; @@ -552,6 +603,7 @@ export function toSourceInstanceView( const nextAction = primaryVerdictAction?.ownerRunnable ? null : actionability.nextAction; const { ownerActionCue } = actionability; const status = actionability.renderedStatus; + const { fusedStatus } = actionability; const manifest = options.manifests ? options.manifests.find((candidate) => manifestMatchesConnectorId(candidate, connectorId)) @@ -630,9 +682,13 @@ export function toSourceInstanceView( needsOwnerLabel: hasFallbackLabel, nextAction, ownerActionCue, + archived, passportFields, primaryVerdictAction, revoked, + fusedStatus, + setupFailed, + setupFailedForwardStatement: setupFailed ? (summary.rendered_verdict?.forward_statement ?? null) : null, status, streams, totalRecords: summary.total_records, @@ -712,13 +768,61 @@ export function collapseDuplicateFallbackSources(instances: readonly SourceInsta }; } -/** Map a list of summaries into the Sources view, preserving input order. */ +/** + * Coalesces every `setupFailed` row into ONE row per connector. Unlike + * {@link collapseDuplicateFallbackSources} (which only groups active + * duplicates past a minimum count), this ALWAYS collapses — even a single + * failed attempt renders through this path, so a connector's setup-failure + * history reads as one row with an attempt count rather than depending on + * how many times the owner happened to retry. Coalescing UI rows only: the + * underlying `connector_instances` rows are untouched, and `attemptCount` + * is a display aggregate, not a merge. + */ +export function collapseSetupFailedSources(instances: readonly SourceInstanceView[]): { + setupFailedGroups: readonly SetupFailedSourceGroup[]; +} { + const byConnector = new Map(); + for (const instance of instances) { + if (!instance.setupFailed) { + continue; + } + const bucket = byConnector.get(instance.connectorId); + if (bucket) { + bucket.push(instance); + } else { + byConnector.set(instance.connectorId, [instance]); + } + } + + const setupFailedGroups: SetupFailedSourceGroup[] = []; + for (const [connectorId, items] of byConnector) { + const representative = items.at(-1); + if (!representative) { + continue; + } + setupFailedGroups.push({ attemptCount: items.length, connectorId, representative }); + } + + return { setupFailedGroups: setupFailedGroups.sort((a, b) => a.connectorId.localeCompare(b.connectorId)) }; +} + +/** + * Map a list of summaries into the Sources view, preserving input order. + * + * Every summary maps to a row, including archived ones. The list previously + * dropped pure recovered historical fragments here; that made their records + * — 163,966 of them on the owner's instance — visible on no summary surface + * at all, violating the standing principle that no data known to the system + * may be invisible in the UI. They are now classified (`archived`) and + * rendered in their own group rather than filtered away. + */ export function toSourcesView( summaries: RefConnectorSummary[], options: { manifests?: readonly SourceManifestLike[] } = {} ): SourceInstanceView[] { + const visibleSummaries = summaries; const fallbackCountByConnector = new Map(); - for (const summary of summaries) { + for (const summary of visibleSummaries) { if ( isFallbackConnectionLabel({ connectorId: summary.connector_id, @@ -730,7 +834,7 @@ export function toSourcesView( } } const fallbackOrdinalByConnector = new Map(); - return summaries.map((summary) => { + return visibleSummaries.map((summary) => { const isAmbiguousFallback = (fallbackCountByConnector.get(summary.connector_id) ?? 0) > 1 && isFallbackConnectionLabel({ diff --git a/apps/console/src/app/(console)/sources/sources-view.css b/apps/console/src/app/(console)/sources/sources-view.css index e1338d76c..7fd98d1b4 100644 --- a/apps/console/src/app/(console)/sources/sources-view.css +++ b/apps/console/src/app/(console)/sources/sources-view.css @@ -110,9 +110,80 @@ a.rr-s-item { white-space: nowrap; } -.rr-s-item__cue { +/* The fused status line (state · last updated · syncing now). Sits directly + under the account line and above the owner-action cue, so the row reads + identity → facts → status → action. Tone is carried by the text color, the + same vocabulary the status dot uses, so the two can never disagree. */ +.rr-s-item__status { grid-row: 3; grid-column: 1 / -1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + font-size: 11px; + line-height: 1.35; + color: var(--muted-foreground); + white-space: nowrap; +} + +/* Syncing is additive context, so it gets a quiet pulse rather than a color of + its own — the color slot stays owned by the worst honest axis. + The dot's box is declared unconditionally below so that toggling the syncing + state changes only opacity/animation, never geometry (see the + "status state does not create separate row geometry" invariant). */ +.rr-s-status::after { + display: inline-block; + width: 5px; + height: 5px; + margin-left: 6px; + vertical-align: middle; + content: ""; + background: currentcolor; + border-radius: 50%; + opacity: 0; +} + +.rr-s-status[data-syncing="true"]::after { + opacity: 0.55; + animation: rr-s-status-pulse 1.6s ease-in-out infinite; +} + +/* State changes COLOR ONLY — never box metrics. Keeping these rules to `color` + is what upholds the "status state does not create separate row geometry" + invariant: every row occupies identical space whatever its state, so the list + never reflows as sources change health. */ +.rr-s-status[data-tone="success"] { + color: var(--muted-foreground); +} + +.rr-s-status[data-tone="warning"] { + color: var(--warning-foreground, var(--foreground)); +} + +.rr-s-status[data-tone="destructive"] { + color: var(--destructive-foreground, var(--destructive)); +} + +@keyframes rr-s-status-pulse { + 0%, + 100% { + opacity: 0.25; + } + 50% { + opacity: 0.9; + } +} + +@media (prefers-reduced-motion: reduce) { + .rr-s-status[data-syncing="true"]::after { + opacity: 0.7; + animation: none; + } +} + +.rr-s-item__cue { + grid-row: 4; + grid-column: 1 / -1; width: fit-content; padding: 2px 7px; margin-top: 2px; diff --git a/apps/console/src/app/(console)/sources/sources-view.test.ts b/apps/console/src/app/(console)/sources/sources-view.test.ts index 42cfe53cb..e0c84e0c7 100644 --- a/apps/console/src/app/(console)/sources/sources-view.test.ts +++ b/apps/console/src/app/(console)/sources/sources-view.test.ts @@ -117,3 +117,25 @@ test("stream manifest columns respond to the panel's own width, so the stream na "the stream-column breakpoint must not key on viewport width — the panel is far narrower than the viewport" ); }); + +// Quiet-expiry defect fix (owner ruling 2026-08-22): the setup-failed +// passport note must render the server's specific `forward_statement` +// (`setupFailedForwardStatement`) rather than always showing the same +// generic sentence regardless of WHY setup failed — that generic-only +// behavior is exactly the pre-fix defect (a TTL-expired attempt reading +// identically to an owner-abandoned one). +const SETUP_FAILED_NOTE_RE = + /instance\.setupFailedForwardStatement\s*\?\?\s*\n?\s*"This connection attempt never finished setup\. No records were collected/; + +test("SourcesView's setup-failed passport note prefers the server's specific forward_statement over the generic fallback", async () => { + const src = await readFile(SOURCES_VIEW_FILE, "utf8"); + const block = src.slice( + src.indexOf("{instance.setupFailed ? ("), + src.indexOf("{instance.revoked && !instance.setupFailed ? (") + ); + assert.match( + block, + SETUP_FAILED_NOTE_RE, + "the passport note must read setupFailedForwardStatement first, falling back to the generic sentence only when absent" + ); +}); diff --git a/apps/console/src/app/(console)/sources/sources-view.tsx b/apps/console/src/app/(console)/sources/sources-view.tsx index 037078b68..e2ff2fa69 100644 --- a/apps/console/src/app/(console)/sources/sources-view.tsx +++ b/apps/console/src/app/(console)/sources/sources-view.tsx @@ -85,6 +85,7 @@ import { SOURCE_ACCESS_NOTE } from "./sources-copy.ts"; import { buildDuplicateSourceReview, collapseDuplicateFallbackSources, + collapseSetupFailedSources, type DuplicateSourceGroup, type DuplicateSourceReview, reactivateRecordCopy, @@ -128,15 +129,38 @@ export function SourcesView({ revokeAction, runtimeAdvisory, }: SourcesViewProps) { - const activeInstances = instances.filter((i) => !i.revoked); - const revokedInstances = instances.filter((i) => i.revoked); - const duplicateReviews = buildDuplicateSourceReview(instances); - const { duplicateGroups, visibleActiveInstances } = collapseDuplicateFallbackSources(instances); + // Archived and setup-failed sources are partitioned out FIRST: neither is + // active nor an ordinary revoked row, and both must not reach the + // duplicate-collapse pass, whose ordinal relabelling ("account 2") implies + // a live sibling set they are not part of. + const archivedInstances = instances.filter((i) => i.archived); + const setupFailedInstances = instances.filter((i) => i.setupFailed); + const liveInstances = instances.filter((i) => !(i.archived || i.setupFailed)); + const activeInstances = liveInstances.filter((i) => !i.revoked); + const revokedInstances = liveInstances.filter((i) => i.revoked); + const duplicateReviews = buildDuplicateSourceReview(liveInstances); + const { duplicateGroups, visibleActiveInstances } = collapseDuplicateFallbackSources(liveInstances); + const { setupFailedGroups } = collapseSetupFailedSources(setupFailedInstances); - // Default selection: first active source, or first revoked if all are revoked. - const defaultId = (visibleActiveInstances[0] ?? duplicateGroups[0]?.items[0] ?? revokedInstances[0])?.id ?? null; + // Default selection: first active source, then revoked, then archived, then + // setup-failed — a setup-failed row is only ever the default when the + // owner has nothing else, in which case showing it beats an empty pane. + const defaultId = + ( + visibleActiveInstances[0] ?? + duplicateGroups[0]?.items[0] ?? + revokedInstances[0] ?? + archivedInstances[0] ?? + setupFailedGroups[0]?.representative + )?.id ?? null; const [selectedId, setSelectedId] = useState(defaultId); - const selected = instances.find((i) => i.id === selectedId) ?? activeInstances[0] ?? revokedInstances[0] ?? null; + const selected = + instances.find((i) => i.id === selectedId) ?? + activeInstances[0] ?? + revokedInstances[0] ?? + archivedInstances[0] ?? + setupFailedGroups[0]?.representative ?? + null; if (instances.length === 0) { return ( @@ -189,6 +213,53 @@ export function SourcesView({ ) : null} + {/* Archived sources: records preserved, collection finished. Shown + because data the system holds must be reachable in the UI, and + grouped separately because they are NOT live — the summary states + that outright so a collapsed group can never read as healthy. */} + {archivedInstances.length > 0 ? ( +
+ + Archived — not collecting ({archivedInstances.length}) + +

+ These sources are no longer collecting. Their records are kept and stay searchable. +

+ {archivedInstances.map((instance) => ( + setSelectedId(instance.id)} + selected={selected?.id === instance.id} + /> + ))} +
+ ) : null} + + {/* Setup-failed sources: repeated failed setup, zero records — the + owner's only evidence a connector was ever attempted. Coalesced to + one row per connector (attemptCount) rather than one row per + retried shell, and grouped separately so a collapsed group can + never be mistaken for a live or ever-successful source. */} + {setupFailedGroups.length > 0 ? ( +
+ + Setup never completed ({setupFailedGroups.length}) + +

+ These connections never finished setup. No records were collected — try again from Add a source. +

+ {setupFailedGroups.map((group) => ( + setSelectedId(group.representative.id)} + selected={selected?.id === group.representative.id} + /> + ))} +
+ ) : null} +
add a source → @@ -329,6 +400,22 @@ function InstanceListItem({ panel, while the list shows only the owner label, retained facts, and health. */} {instance.accountLine} + {/* + * The fused status line: what it is, when it last updated, and whether + * it is syncing right now — one string, worst honest axis first. Before + * this, the row showed only a colored glyph (label sr-only), so an owner + * could not tell a fresh source from a stale one, or a syncing source + * from a stuck one, without opening the detail panel. See + * design-notes/fused-source-status-2026-08-22.md. + */} + + {instance.fusedStatus.line} + {instance.ownerActionCue ? ( ) : null} - {/* The dot is a decorative reinforcement of the status; the textual - label is announced via the sr-only span so color is never the sole - signal and the glyph itself carries no a11y burden. */} + {/* The dot is a decorative reinforcement of the status. The textual + status now renders visibly in `.rr-s-item__status`, so color is + never the sole signal and this glyph carries no a11y burden — a + second sr-only copy here would make screen readers announce the + status twice per row. */} - {instance.status.label} ); @@ -440,7 +528,22 @@ function InstancePassport({ {instance.nextAction ? ( ) : null} - {instance.revoked ? ( + {instance.setupFailed ? ( +

+ {/* Server-owned sentence so a TTL-expired attempt says so plainly + ("expired while waiting for you to finish signing in") + instead of collapsing into the same generic sentence an + owner-abandoned or connector-failed attempt gets — see + `archiveRenderedVerdict` in `server/ref-control.ts`. The + fallback (pre-existing generic copy, own CTA appended) applies + only when the mirror predates `rendered_verdict` — never for a + live reference, whose forward_statement always names the next + step itself. */} + {instance.setupFailedForwardStatement ?? + "This connection attempt never finished setup. No records were collected — start a new attempt from Add a source."} +

+ ) : null} + {instance.revoked && !instance.setupFailed ? (

Future collection is stopped. Already-collected records stay visible and searchable; revoke does not erase anything. @@ -607,7 +710,11 @@ function PassportActions({ ) : null} - {interactive && reactivateAction && instance.connectionId && instance.revoked ? ( + {/* A setup-failed row has no credential, schedule, or prior collection + to reactivate — it is a spent enrollment shell, not a paused live + connection. Offering "Reactivate" here would be the same dishonest + promise archived sources already refuse to make. */} + {interactive && reactivateAction && instance.connectionId && instance.revoked && !instance.setupFailed ? ( setConfirmingReactivate((v) => !v)} @@ -944,9 +1051,9 @@ function StreamManifestRow({ - Unknown + Not measured )} {collection?.dispositionLabel ? ( diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts index 0828019ec..de4ba79ab 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts @@ -42,6 +42,29 @@ test("410 with companion_unavailable code classifies as companion_unavailable", assert.equal(reason, "companion_unavailable"); }); +test("503 with streaming_companion_unavailable code classifies as companion_unavailable", () => { + // Regression: the reference server raises StreamingCompanionUnavailableError + // as HTTP 503 with body code `streaming_companion_unavailable` + // (reference-implementation/server/streaming/routes.ts). That fell through + // to `unknown`, whose copy ("Couldn't reach the browser stream after + // several tries.") reads as a network/reachability problem when the server + // in fact answered with a specific, actionable reason. + const { reason, troubleMessage } = classifyStreamReachFailure({ + probeCode: "streaming_companion_unavailable", + probeStatus: 503, + }); + assert.equal(reason, "companion_unavailable"); + assert.equal(troubleMessage, "The browser session is no longer running on the server. Start the browser step again."); +}); + +test("503 with an unrelated or absent code still classifies as unknown", () => { + // Other 503s (e.g. the managed n.eko window-settle probe's + // managed_surface_window_settle_unavailable) are a distinct condition; + // only the companion-unavailable body code should map to companion_unavailable. + const { reason } = classifyStreamReachFailure({ probeCode: null, probeStatus: 503 }); + assert.equal(reason, "unknown"); +}); + test("a thrown probe (no HTTP response) classifies as unreachable_origin", () => { const { reason } = classifyStreamReachFailure({ probeError: true, probeStatus: null }); assert.equal(reason, "unreachable_origin"); @@ -59,9 +82,12 @@ test("an answered-but-unrecognized status (5xx) classifies as unknown", () => { assert.equal(reason, "unknown"); }); -test("unknown preserves the prior generic give-up message verbatim (no regression)", () => { +test("unknown with a known status appends the status code to the generic give-up message", () => { + // A future unmapped status must not hide behind identical generic copy the + // way 503/streaming_companion_unavailable did before this fix — the status + // is now visible in the message even though the reason stays `unknown`. const { troubleMessage } = classifyStreamReachFailure({ probeStatus: 500 }); - assert.equal(troubleMessage, "Couldn't reach the browser stream after several tries."); + assert.equal(troubleMessage, "Couldn't reach the browser stream after several tries. (server responded 500)"); }); test("every reason yields a non-empty operator message that never claims success", () => { diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts index 527da137b..8901ac32e 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts @@ -39,8 +39,12 @@ const STREAM_REACH_REASON_SET = new Set(STREAM_REACH_REASONS); /** * Operator-facing copy per reason. Operator-console voice: address the operator * running their own instance, name the failure class, point at the next action. - * Never claim the stream connected or recovered. `unknown` preserves the prior - * generic give-up message verbatim so no occurrence regresses. + * Never claim the stream connected or recovered. `unknown`'s entry here is the + * fallback used only when no status was available at all (see + * {@link unknownTroubleMessage}); when a status IS known it is appended so an + * unmapped status is visible rather than hidden behind identical generic copy + * — that silence is exactly what let the 503/streaming_companion_unavailable + * case go unnoticed before this reason was mapped. */ const STREAM_REACH_MESSAGE: Record = { companion_unavailable: "The browser session is no longer running on the server. Start the browser step again.", @@ -52,6 +56,11 @@ const STREAM_REACH_MESSAGE: Record = { "Couldn't reach the browser stream. Check that the reference server is reachable, then try again.", }; +function unknownTroubleMessage(probeStatus: number | null): string { + const base = STREAM_REACH_MESSAGE.unknown; + return probeStatus === null ? base : `${base} (server responded ${probeStatus})`; +} + export interface StreamReachProbeResult { /** * `error.code` parsed from the probe response body, when present. Used to @@ -93,7 +102,8 @@ export function sanitizeStreamReachReason(value: unknown): StreamReachReason { */ export function classifyStreamReachFailure(probe: StreamReachProbeResult): StreamReachClassification { const reason = classifyReason(probe); - return { reason, troubleMessage: STREAM_REACH_MESSAGE[reason] }; + const troubleMessage = reason === "unknown" ? unknownTroubleMessage(probe.probeStatus) : STREAM_REACH_MESSAGE[reason]; + return { reason, troubleMessage }; } function classifyReason(probe: StreamReachProbeResult): StreamReachReason { @@ -113,9 +123,27 @@ function classifyReason(probe: StreamReachProbeResult): StreamReachReason { // default to the more common expiry case rather than fabricating // companion loss. return probe.probeCode === "companion_unavailable" ? "companion_unavailable" : "session_expired"; + case 503: + // The reference server raises StreamingCompanionUnavailableError as 503 + // ("a browser-control interaction is current, but no ready browser + // surface is registered for this run") with body code + // `streaming_companion_unavailable`. That fell through to `unknown`, + // whose copy read as a network/proxy fault and sent the owner looking + // in the wrong place while the server had in fact answered with a + // specific, actionable reason. Verified 2026-08-18: the single + // STREAMING_COMPANION_UNAVAILABLE occurrence in the production log is + // recorded with statusCode 503. Other 503 codes (e.g. the managed n.eko + // window-settle probe's `managed_surface_window_settle_unavailable`) + // are a different, currently-unclassified condition — only match the + // companion-unavailable body code here rather than collapsing every + // 503 into this one reason. + return probe.probeCode === "streaming_companion_unavailable" ? "companion_unavailable" : "unknown"; default: - // 5xx, proxy errors, or any other answered status: real but unclassified. - // Preserve the prior generic give-up rather than guessing. + // 5xx, proxy errors, or any other answered status: real but + // unclassified. Keep the reason `unknown` (the spine vocabulary stays + // closed), but the message includes the status code so the next + // unmapped status is at least visible instead of hiding behind + // identical generic copy the way this one did. return "unknown"; } } diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.test.ts new file mode 100644 index 000000000..bbad26cb6 --- /dev/null +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.test.ts @@ -0,0 +1,89 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { normalizedPointerButton, readablePointerInput } from "./stream-viewer-pointer-input.ts"; + +test("a primary-contact touch tap is forwarded, matching real touch pointerdown/pointerup semantics", () => { + // Per the Pointer Events spec, `button` is 0 for the primary contact on a + // touch pointerdown/pointerup. A regression that gates touch on `button` + // drops every real-world tap while leaving synthetic-event tests (which + // often omit `button` or set it to a mouse-like value) green. + assert.deepEqual(readablePointerInput({ buttons: 1, pointerType: "touch", type: "pointerdown" }), { + pointerType: "touch", + type: "pointerdown", + }); + assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointerup" }), { + pointerType: "touch", + type: "pointerup", + }); +}); + +test("touch pointermove is always forwarded regardless of the mouse-only hover-move gate", () => { + assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointermove" }), { + pointerType: "touch", + type: "pointermove", + }); +}); + +test("a mouse hover move with no button held is dropped to avoid flooding the wire", () => { + assert.equal(readablePointerInput({ buttons: 0, pointerType: "mouse", type: "pointermove" }), null); +}); + +test("a mouse drag move with a button held is forwarded", () => { + assert.deepEqual(readablePointerInput({ buttons: 1, pointerType: "mouse", type: "pointermove" }), { + pointerType: "mouse", + type: "pointermove", + }); +}); + +test("pointercancel is always forwarded for every pointer type", () => { + assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointercancel" }), { + pointerType: "touch", + type: "pointercancel", + }); + assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "mouse", type: "pointercancel" }), { + pointerType: "mouse", + type: "pointercancel", + }); +}); + +test("an unrecognized DOM event type is dropped", () => { + assert.equal(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointerenter" }), null); +}); + +// `normalizedPointerButton` exists because the forwarded `button` is not a +// filter downstream — it is arithmetic. neko computes the X11 button as +// `(button ?? 0) + 1`, and X11 button 1 is primary, so a touch pointerdown +// reporting the non-spec `button === -1` becomes X11 button 0 (no button at +// all) and the remote page never sees a press. This is the residual half of +// the owner's "can't tap the captcha on mobile" report: an earlier fix stopped +// DROPPING such events but still forwarded the raw value into that arithmetic. + +test("a touch pointerdown reporting button -1 is normalized to primary contact", () => { + // The exact value the sibling gate's own doc comment documents as real on + // touch input paths. Left raw, it silently disarms the tap. + assert.equal(normalizedPointerButton(-1, "touch"), 0); +}); + +test("a pen pointerdown reporting button -1 is normalized to primary contact", () => { + assert.equal(normalizedPointerButton(-1, "pen"), 0); +}); + +test("an ordinary primary-contact touch button is left alone", () => { + assert.equal(normalizedPointerButton(0, "touch"), 0); +}); + +test("mouse buttons are passed through so middle/right/back/forward survive", () => { + // Touch and pen have no secondary button, but a mouse does — normalizing it + // would turn every right-click into a left-click. + assert.equal(normalizedPointerButton(0, "mouse"), 0); + assert.equal(normalizedPointerButton(1, "mouse"), 1); + assert.equal(normalizedPointerButton(2, "mouse"), 2); + assert.equal(normalizedPointerButton(3, "mouse"), 3); + assert.equal(normalizedPointerButton(4, "mouse"), 4); + // A mouse never legitimately reports -1 on a press, but if it does the value + // is preserved rather than invented — mouse `button` is meaningful. + assert.equal(normalizedPointerButton(-1, "mouse"), -1); +}); diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts new file mode 100644 index 000000000..2f38543af --- /dev/null +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts @@ -0,0 +1,102 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Extracted out of stream-viewer.tsx (a "use client" React component file) + * into a plain module so this gate can be unit-tested under plain + * `node --test` without a DOM/PointerEvent runtime. This function decides + * whether a captured DOM pointer event becomes a forwarded remote-surface + * intent at all; a wrong decision here silently drops input with no visible + * symptom other than "taps don't do anything." + */ + +export type RemotePointerActionType = "pointercancel" | "pointerdown" | "pointermove" | "pointerup"; +export type RemotePointerType = "mouse" | "pen" | "touch"; + +export interface ReadablePointerEventLike { + buttons: number; + pointerType: string; + type: string; +} + +export interface ReadablePointerInput { + pointerType: RemotePointerType; + type: RemotePointerActionType; +} + +function remoteTypeFor(type: string): RemotePointerActionType | null { + switch (type) { + case "pointerdown": + return "pointerdown"; + case "pointermove": + return "pointermove"; + case "pointerup": + return "pointerup"; + case "pointercancel": + return "pointercancel"; + default: + return null; + } +} + +/** + * Gates a raw DOM pointer event down to a forwardable remote-surface intent, + * or `null` if it should be dropped. + * + * Per the W3C Pointer Events spec, `button`/`buttons` describe *mouse-style* + * button state and are largely meaningless for touch: `button` is 0 (primary) + * for a touch pointerdown/pointerup, but some touch input paths (stylus + * palm-rejection proxies, certain WebViews) report `button === -1` on a + * touch pointerup even though the contact itself is legitimate. Gating touch + * on `event.button` at all is the wrong signal — touch has no secondary + * button, so every primary-contact touch event must pass through regardless + * of `button`. Only `buttons` (a bitmask) is meaningful across pointer types, + * and it is only used here to suppress hover-only mouse moves. + */ +export function readablePointerInput(event: ReadablePointerEventLike): ReadablePointerInput | null { + const type = remoteTypeFor(event.type); + if (!type) { + return null; + } + const pointerType: RemotePointerType = + event.pointerType === "touch" || event.pointerType === "pen" ? event.pointerType : "mouse"; + // Hover-move gate: suppress mouse moves with no button held to prevent + // hover floods. Touch/pen have no hover-move concept at this layer. + if (type === "pointermove" && pointerType === "mouse" && event.buttons === 0) { + return null; + } + return { pointerType, type }; +} + +/** + * Normalizes `PointerEvent.button` into a primary-contact button index that is + * safe to forward, for touch and pen only. + * + * The gate above stopped *dropping* touch events with a non-zero `button`, but + * the raw value is still forwarded in the wire payload — and downstream it is + * arithmetic, not a filter. `NekoPointerController.handle` (remote-surface + * 1.5.2, `controllers/neko-pointer-controller.js`) computes the X11 button as + * `(event.button ?? 0) + 1`, where X11 button 1 is primary. A touch + * `pointerdown` reporting `button === -1` — the same non-spec value this + * module's own comment above documents as real on touch input paths — + * therefore becomes X11 button **0**, which is not a button at all, so neko + * presses nothing and the tap never clicks. + * + * Verified against the installed controller by replaying the exact client + * payload shape: `button: 0` on down yields `buttonDown(1)`, while + * `button: -1` on down yields `buttonDown(0)`. + * + * `pointerup` was already safe by luck — the controller prefers the remembered + * press button over the event's own — but `pointerdown` has nothing to fall + * back to, so it must be normalized here at the payload boundary. + * + * Touch and pen have no secondary button, so pinning them to primary loses no + * information. Mouse is passed through untouched: its `button` is meaningful + * (middle/right/back/forward) and must survive. + */ +export function normalizedPointerButton(button: number, pointerType: RemotePointerType): number { + if (pointerType === "mouse") { + return button; + } + return button < 0 ? 0 : button; +} diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts index 65f1c6126..19aa1a8d0 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts @@ -51,7 +51,7 @@ function readViewerSource(): Promise { test("trusted-touch keyboard focus reaches the viewer session synchronously", async () => { const src = await readViewerSource(); const tapStart = src.indexOf("const handleMobileKeyboardPointer ="); - const tapEnd = src.indexOf("const remoteTypeFor =", tapStart); + const tapEnd = src.indexOf("const dispatchPointerIntent =", tapStart); const trustedTap = src.slice(tapStart, tapEnd); assert.notEqual(tapStart, -1, "the production trusted-touch handler is present"); diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-touch-tap-oracle.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-touch-tap-oracle.test.ts new file mode 100644 index 000000000..09b75e018 --- /dev/null +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-touch-tap-oracle.test.ts @@ -0,0 +1,133 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * End-to-end oracle for the owner's "can't tap the captcha on mobile" report. + * + * The unit tests beside this file assert `normalizedPointerButton` in + * isolation, which proves only that it does what it says. This file replays + * the client's real wire payload through the REAL installed + * `NekoPointerController` from `@opendatalabs/remote-surface` and asserts on + * the X11 button the remote actually receives. That is the property the owner + * cares about — "the tap presses a button on the remote page" — and it stays + * honest if the dependency changes its mapping, because the oracle is the + * dependency itself rather than a restatement of our own arithmetic. + * + * A captcha checkbox is not special here: the remote input path is + * coordinate-based (neko X11 / CDP `Input.dispatchMouseEvent`), so it crosses + * cross-origin iframe boundaries like reCAPTCHA's by construction. Nothing in + * the path hit-tests the top document. What actually broke the tap was the + * button index, which fails identically inside or outside an iframe — it just + * gets noticed on a captcha because that is where a tap is unavoidable. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { normalizedPointerButton } from "./stream-viewer-pointer-input.ts"; + +// X11 button codes, as used by neko's `control.buttonDown`/`buttonUp`. +const X11_PRIMARY = 1; +const X11_NO_BUTTON = 0; + +interface ControlCall { + button: number; + kind: "buttonDown" | "buttonUp"; +} + +type NekoPointerControllerCtor = new (deps: { + control: Record; + mapToRemote: (x: number, y: number) => { x: number; y: number }; +}) => { handle: (event: Record) => void }; + +/** Loads the real controller, or `null` when the dependency isn't installed. */ +async function loadNekoPointerController(): Promise { + try { + const mod = await import("@opendatalabs/remote-surface"); + return (mod.NekoPointerController as unknown as NekoPointerControllerCtor | undefined) ?? null; + } catch { + return null; + } +} + +/** + * Replays a touch tap the way the client sends it, returning the X11 button + * calls neko would make. `rawButton` is what the browser put on the + * PointerEvent; `normalize` mirrors whether the client sanitizes it. + */ +function tapThroughController( + Controller: NonNullable>>, + rawButton: number, + normalize: boolean +): ControlCall[] { + const calls: ControlCall[] = []; + const controller = new Controller({ + control: { + buttonDown: (pressed: number) => calls.push({ button: pressed, kind: "buttonDown" }), + buttonUp: (released: number) => calls.push({ button: released, kind: "buttonUp" }), + move: () => undefined, + scroll: () => undefined, + }, + mapToRemote: (x: number, y: number) => ({ x, y }), + }); + const button = normalize ? normalizedPointerButton(rawButton, "touch") : rawButton; + // The payload shape built in stream-viewer.tsx's `dispatchPointerIntent`. + controller.handle({ button, pointerId: 1, pointerType: "touch", type: "pointerdown", x: 100, y: 200 }); + controller.handle({ button, pointerId: 1, pointerType: "touch", type: "pointerup", x: 100, y: 200 }); + return calls; +} + +test("a touch tap reporting button -1 presses a real button on the remote page", async (t) => { + const Controller = await loadNekoPointerController(); + if (!Controller) { + t.skip("@opendatalabs/remote-surface is not installed in this workspace"); + return; + } + + const calls = tapThroughController(Controller, -1, true); + + assert.deepEqual( + calls, + [ + { button: X11_PRIMARY, kind: "buttonDown" }, + { button: X11_PRIMARY, kind: "buttonUp" }, + ], + "a touch tap must press and release X11 primary, or the remote page sees no click at all" + ); +}); + +test("the unnormalized payload is what silently disarmed the tap", async (t) => { + const Controller = await loadNekoPointerController(); + if (!Controller) { + t.skip("@opendatalabs/remote-surface is not installed in this workspace"); + return; + } + + // Characterizes the defect rather than asserting the fix: forwarding the raw + // -1 makes neko press X11 button 0, which is not a button. If a future + // dependency bump makes the raw value safe on its own, this test fails and + // says so, instead of leaving the normalization as unexplained cargo. + const calls = tapThroughController(Controller, -1, false); + + assert.equal( + calls[0]?.button, + X11_NO_BUTTON, + "raw button -1 maps to X11 button 0 (no button) — this is why the captcha tap did nothing" + ); +}); + +test("an ordinary touch tap reporting button 0 was never broken", async (t) => { + const Controller = await loadNekoPointerController(); + if (!Controller) { + t.skip("@opendatalabs/remote-surface is not installed in this workspace"); + return; + } + + // Bounds the blast radius of the report: taps on engines that report a spec + // -compliant 0 always worked, which is why this reproduced only on some + // devices and why "taps are broken" was never reproducible on desktop. + const normalized = tapThroughController(Controller, 0, true); + const raw = tapThroughController(Controller, 0, false); + + assert.deepEqual(normalized, raw, "normalization must be a no-op for a spec-compliant primary contact"); + assert.equal(normalized[0]?.button, X11_PRIMARY); +}); diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx index 72de60fbc..adb47cba4 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx @@ -141,6 +141,7 @@ import { readViewerViewport, viewportLayoutFromInfo, } from "./stream-viewer-geometry.ts"; +import { normalizedPointerButton, readablePointerInput } from "./stream-viewer-pointer-input.ts"; import { parseAttachedMessage } from "./stream-viewer-protocol.ts"; import { createPdppRemoteSurfaceTransport, @@ -4180,42 +4181,6 @@ function NekoSurface({ userActivationActive: navigator.userActivation?.isActive ?? null, }); }; - const remoteTypeFor = (type: string): "pointerdown" | "pointermove" | "pointerup" | "pointercancel" | null => { - switch (type) { - case "pointerdown": - return "pointerdown"; - case "pointermove": - return "pointermove"; - case "pointerup": - return "pointerup"; - case "pointercancel": - return "pointercancel"; - default: - return null; - } - }; - const readablePointerInput = ( - event: PointerEvent - ): { - pointerType: "mouse" | "touch" | "pen"; - type: "pointercancel" | "pointerdown" | "pointermove" | "pointerup"; - } | null => { - const type = remoteTypeFor(event.type); - if (!type) { - return null; - } - const pointerType: "mouse" | "touch" | "pen" = - event.pointerType === "touch" || event.pointerType === "pen" ? event.pointerType : "mouse"; - if ((type === "pointerdown" || type === "pointerup") && pointerType === "touch" && event.button !== 0) { - return null; - } - // Hover-move gate (step 2 open question #2): suppress mouse moves - // with no button held to prevent hover floods. - if (type === "pointermove" && event.pointerType === "mouse" && event.buttons === 0) { - return null; - } - return { pointerType, type }; - }; const dispatchPointerIntent = ( event: PointerEvent, type: "pointercancel" | "pointerdown" | "pointermove" | "pointerup", @@ -4227,7 +4192,11 @@ function NekoSurface({ } const pointerIntent: Extract = { action: type, - button: event.button, + // Touch/pen `button` is normalized to primary before it leaves the + // client: neko turns this into an X11 button by adding 1, so a raw + // `-1` from a touch pointerdown would press X11 button 0 (no button) + // and the tap would never click. See `normalizedPointerButton`. + button: normalizedPointerButton(event.button, pointerType), buttons: event.buttons, clickCount: event.detail, pointerId: event.pointerId, diff --git a/apps/console/src/app/(console)/syncs/error.tsx b/apps/console/src/app/(console)/syncs/error.tsx index 5bf1d76cb..cd8fc17e1 100644 --- a/apps/console/src/app/(console)/syncs/error.tsx +++ b/apps/console/src/app/(console)/syncs/error.tsx @@ -3,38 +3,63 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { buttonVariants } from "@pdpp/brand-react"; import { useEffect } from "react"; +import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts"; +import { ListLoadingSkeleton } from "../components/route-loading.tsx"; /** - * Runs-segment error boundary (App Router convention). + * Syncs-segment error boundary (App Router convention) — SLVP bar: Stripe, + * Linear, Vercel, and Plaid never tell an owner "we hit a transient read + * interruption, retrying." The page renders, or it quietly shows last-known + * state. The owner never learns the backend hiccuped. * - * Scopes a runs-area failure to the runs area instead of the dashboard-wide - * `Something went wrong`. A run that failed unexpectedly should not also crash - * the surrounding page to a contextless boundary. Self-contained on purpose - * (mirrors `dashboard/error.tsx`): no server-only imports. + * Root cause of the throw this boundary catches (`Error: The destination + * stream closed early`): the read itself is fine — React's Flight/RSC + * streaming writer reacting to the HTTP response closing before the stream + * finished flushing (a poll tick from `run-detail-poller.tsx`/`LivePoller` + * firing `router.refresh()` while a prior refresh's stream is still in + * flight, or the tab backgrounding mid-render). It is a client-transport race + * below the data layer, not a backend outage — see `sources/error.tsx` for + * the full original writeup of this pattern. + * + * `/syncs` has no client-cached last-known-read marker (unlike + * `sources/last-known-read.ts`), so this boundary shows the plain skeleton + * with no staleness caption rather than fabricate a timestamp. + * + * Self-contained on purpose: a `"use client"` boundary must not import + * server-only modules, since the dashboard shell transitively pulls in + * `lib/owner-token.ts` (`server-only`). + */ + +/** + * Consecutive-failure counter, held at MODULE scope rather than component + * state — see `read-resilient-retry.ts` for why a `useState` counter would + * silently reset every catch and never actually back off. */ +const retryCounter = createRetryCounter(); + export default function RunsError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { useEffect(() => { + // Logged for operator diagnostics only — never surfaced to the owner. console.error(error); }, [error]); + useEffect(() => { + // Unbounded, capped backoff: every mount (i.e. every failed attempt) + // schedules the next retry at a delay that grows with the module-scoped + // counter. There is deliberately no ceiling on the counter itself — a + // persistent failure degrades to a slow quiet heartbeat, never a dead end. + const delay = nextRetryDelayMs(retryCounter.attempts); + const id = setTimeout(() => { + retryCounter.attempts += 1; + reset(); + }, delay); + return () => clearTimeout(id); + }, [reset]); + return ( -

-

Read error

-

Couldn't load syncs

-

- The Syncs view ran into an error while reading from your reference deployment. Your syncs are unaffected — this - is a read failure, not a change. Try again, or check your reference deployment status. -

-
- - - Back to Syncs - -
-
+
+ +
); } diff --git a/apps/console/src/app/(console)/syncs/page.invariants.test.ts b/apps/console/src/app/(console)/syncs/page.invariants.test.ts index 74a76d4d9..142715b11 100644 --- a/apps/console/src/app/(console)/syncs/page.invariants.test.ts +++ b/apps/console/src/app/(console)/syncs/page.invariants.test.ts @@ -222,7 +222,11 @@ test("recent syncs pager links to the next cursor the server returned, never a f assert.match(src, RECENT_PAGER_NEXT_HREF_RE, "the older-syncs link must carry the real next_cursor from the server"); }); -const COVERAGE_HUMANIZER_IMPORT_RE = /formatCoverageAxis[\s\S]{0,40}from "\.\.\/lib\/connection-evidence\.ts"/; +// The shared humanizer moved from the console-local `connection-evidence.ts` +// into `@pdpp/display` so the headless `sources-report` CLI renders the SAME +// coverage words this view does. The invariant is unchanged — coverage copy +// must go through the shared humanizer, never interpolate the raw axis key. +const COVERAGE_HUMANIZER_IMPORT_RE = /formatCoverageAxis[\s\S]{0,40}from "@pdpp\/display"/; const COVERAGE_HUMANIZER_CALL_RE = /formatCoverageAxis\(condition\)\.value/; const RAW_COVERAGE_INTERPOLATION_RE = /`\s*·\s*\$\{condition\}`/; diff --git a/apps/console/src/app/(console)/syncs/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/syncs/read-resilience.invariants.test.ts new file mode 100644 index 000000000..320c6e988 --- /dev/null +++ b/apps/console/src/app/(console)/syncs/read-resilience.invariants.test.ts @@ -0,0 +1,104 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Read-resilience acceptance invariants for the syncs segment, mirroring + * `sources/read-resilience.invariants.test.ts`. The prior version of this + * boundary showed the owner "Couldn't load syncs" behind a manual "Try + * again" button on the FIRST catch, with no auto-retry at all — strictly + * worse than the original `sources/error.tsx` this pattern replaces. + * + * THE STANDARD (stated explicitly by the owner): Stripe, Linear, Vercel, and + * Plaid never show a user "we hit a transient read interruption, retrying." + * The page renders, or it shows last-known state. The user never learns the + * backend hiccuped. + * + * `/syncs` has no client-cached last-known-read marker (unlike + * `sources/last-known-read.ts`), so this boundary shows the plain skeleton + * with no staleness caption — see the file's own doc comment for why that is + * the correct choice rather than fabricating a timestamp. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const HERE = fileURLToPath(new URL(".", import.meta.url)); +const ERROR_FILE = `${HERE}error.tsx`; + +const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g; + +/** + * Strip `/* ... *‍/` block comments before checking for retired owner-facing + * copy. The boundary's doc comment legitimately QUOTES the retired phrases + * (to explain what this pattern replaces and why) — that is documentation, + * not rendered JSX text, so it must not trip the ban. + */ +function withoutBlockComments(src: string): string { + return src.replace(BLOCK_COMMENT_RE, ""); +} + +// Regexes hoisted to module scope (project lint: useTopLevelRegex). The +// owner-facing-string bans intentionally allow "error" as a JS identifier +// (the boundary prop is literally named `error`) but forbid it in rendered +// JSX text content, so we assert on specific retired phrases. +const RETIRED_COULDNT_RE = /Couldn't/; +const RETIRED_ERROR_HEADING_RE = /Read error/; +const RETIRED_TRY_AGAIN_RE = /Try again/; +const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i; +const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i; +const RETIRED_BACK_LINK_RE = /Back to Syncs/; +const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/; + +const USES_LOADING_SKELETON_RE = / { + const rawSrc = await readFile(ERROR_FILE, "utf8"); + const src = withoutBlockComments(rawSrc); + assert.doesNotMatch(src, RETIRED_COULDNT_RE); + assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE); + assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE); + assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE); + assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE); + assert.doesNotMatch(src, RETIRED_BACK_LINK_RE); + assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE); +}); + +test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, IMPORTS_LOADING_SKELETON_RE); + assert.match(src, USES_LOADING_SKELETON_RE); + assert.match(src, RECOVERING_TESTID_RE); + // loading.tsx uses ListLoadingSkeleton label="Syncs" rows={8}; the boundary + // must match so a caught teardown is visually identical to a normal load. + assert.match(src, /ListLoadingSkeleton label="Syncs" rows=\{8\}/); +}); + +test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.match(src, CALLS_RESET_RE); + assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE); + assert.match(src, IMPORTS_SHARED_RETRY_RE); + assert.match(src, MODULE_SCOPE_COUNTER_RE); + assert.doesNotMatch( + src, + NO_REACT_STATE_COUNTER_RE, + "the retry counter must live at module scope, not React state, or backoff never grows across remounts" + ); + assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded"); +}); + +test("the boundary is self-contained: no server-only import", async () => { + const src = await readFile(ERROR_FILE, "utf8"); + assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE); +}); diff --git a/apps/console/src/app/(console)/syncs/syncs-demo.ts b/apps/console/src/app/(console)/syncs/syncs-demo.ts index 3c0850840..4bde1fdc0 100644 --- a/apps/console/src/app/(console)/syncs/syncs-demo.ts +++ b/apps/console/src/app/(console)/syncs/syncs-demo.ts @@ -105,7 +105,7 @@ const COOLING_VERDICT = { channel: "advisory", detail: {}, forward_statement: "The source is throttling this connection; it will retry automatically.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, progress: { gaps_drained_last_run: null, headline: "Waiting for the next attempt.", diff --git a/apps/console/src/app/(console)/syncs/syncs-model.test.ts b/apps/console/src/app/(console)/syncs/syncs-model.test.ts index 396c22b2f..1b632b0fe 100644 --- a/apps/console/src/app/(console)/syncs/syncs-model.test.ts +++ b/apps/console/src/app/(console)/syncs/syncs-model.test.ts @@ -203,7 +203,7 @@ test("source-pressure cooldown produces a WAIT card, never a reconnect prompt", channel: "advisory", forward_statement: "The source is throttling this connection, so the scheduler is spacing out automatic attempts.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ action({ audience: "none", @@ -258,7 +258,7 @@ test("a blocked connection with a source-pressure backlog still gets the WAIT ca channel: "advisory", forward_statement: "The source is throttling this connection, so the scheduler is spacing out automatic attempts.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ action({ audience: "none", @@ -749,7 +749,7 @@ test("a broken connector does not rewrite a successful last run into sync failed required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -794,7 +794,7 @@ test("failure cards bind terminal gaps to rendered verdict copy, never retryable required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -811,7 +811,7 @@ test("failure cards bind terminal gaps to rendered verdict copy, never retryable // biome-ignore lint/suspicious/noUnnecessaryConditions: array/Map-lookup access under noUncheckedIndexedAccess is genuinely T | undefined; tsc rejects removing this guard (Biome does not honor that tsconfig flag here). assert.equal(card?.summary.prose, "This connector needs a code fix before it can collect again."); assert.equal(card.summary.cta, "wait"); - assert.equal(card.summary.actionLabel, "Connector code needs a fix"); + assert.equal(card.summary.actionLabel, "Some data from this source can't be collected"); assert.equal(card.summary.ownerActionRequired, false); // biome-ignore lint/suspicious/noUnnecessaryConditions: array/Map-lookup access under noUncheckedIndexedAccess is genuinely T | undefined; tsc rejects removing this guard (Biome does not honor that tsconfig flag here). assert.doesNotMatch(card?.summary.prose, RESUME_FALSE_REASSURANCE_RE); @@ -834,7 +834,7 @@ test("failure cards bind retryable gaps to the rendered Retry now action", () => rendered_verdict: renderedVerdict({ channel: "advisory", forward_statement: "Retry now to give the recoverable gap another run.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [action()], }), }), @@ -908,7 +908,7 @@ test("failure cards bind dead-letter backlog to collector action, not resume-nor rendered_verdict: renderedVerdict({ channel: "attention", forward_statement: "Check the collector before this source can make progress.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ action({ cta: "Check the collector", @@ -949,7 +949,7 @@ test("device-local recovery counts as need-your-hand while navigating to recover rendered_verdict: renderedVerdict({ channel: "attention", forward_statement: "The local collector has saved records on its host that did not upload to this server.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ action({ cta: "Run local recovery", @@ -1028,7 +1028,7 @@ test("failure cards carry shared source-work groups for Runs presentation", () = rendered_verdict: renderedVerdict({ channel: "advisory", forward_statement: "Latest collection completed with known coverage gaps.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [ action({ audience: "maintainer", @@ -1062,12 +1062,12 @@ test("syncs ranking only treats attention plus primary owner action as need-your source_work: "system_issue", rendered_verdict: renderedVerdict({ channel: "attention", - forward_statement: "Connector code needs a fix before this can collect again.", + forward_statement: "Some data from this source can't be collected.", pill: { label: "Can't collect", tone: "red" }, required_actions: [ action({ audience: "maintainer", - cta: "Connector code needs a fix", + cta: "Some data from this source can't be collected", kind: "code_fix", satisfied_when: { kind: "none" }, terminal: true, @@ -1139,7 +1139,7 @@ test("syncs overview collapses repeated unnamed fallback sources", () => { const amazonAdvisory = renderedVerdict({ channel: "advisory", forward_statement: "Retry now to give the recoverable gap another run.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [action()], }); const model = buildSyncsViewModel({ @@ -1279,7 +1279,7 @@ test("syncs overview shows ALL review cards (no cap) and the band counts the ful const advisoryVerdict = renderedVerdict({ channel: "advisory", forward_statement: "Run a refresh to bring this up to date.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [action({ cta: "Refresh now", kind: "retry_gap" })], }); const connectors = Array.from({ length: 8 }, (_, index) => @@ -1326,7 +1326,7 @@ test("syncs cross-surface: rendered review-card count equals the failure cards b const advisoryVerdict = renderedVerdict({ channel: "advisory", forward_statement: "Run a refresh to bring this up to date.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, required_actions: [action({ cta: "Refresh now", kind: "retry_gap" })], }); const model = buildSyncsViewModel({ @@ -1361,7 +1361,7 @@ test("syncs cross-surface: an inactive queued recovery card is passive progress, const deferredRecoveryVerdict = renderedVerdict({ channel: "calm", forward_statement: "The next run is expected to fill the remaining data.", - pill: { label: "Degraded", tone: "amber" }, + pill: { label: "Missing data", tone: "amber" }, progress: { gaps_drained_last_run: null, headline: "Collecting in the background.", diff --git a/apps/console/src/app/(console)/syncs/syncs-view.tsx b/apps/console/src/app/(console)/syncs/syncs-view.tsx index 1cbe05428..1805372f5 100644 --- a/apps/console/src/app/(console)/syncs/syncs-view.tsx +++ b/apps/console/src/app/(console)/syncs/syncs-view.tsx @@ -28,10 +28,10 @@ import { TableHeader, TableHeaderRow, } from "@pdpp/brand-react"; -import { humanizeFieldLabel } from "@pdpp/display"; +import { formatCoverageAxis, humanizeFieldLabel } from "@pdpp/display"; import { dashboardRoutes } from "@pdpp/operator-ui/components/views/routes"; import Link from "next/link"; -import { formatCoverageAxis } from "../lib/connection-evidence.ts"; + import { SOURCE_WORK_GROUP_COPY } from "../lib/source-actionability.ts"; import { type DuplicateSyncGroup, @@ -533,8 +533,10 @@ function formatCollectedThisRun(row: SyncRow): string | null { } /** - * The coverage condition is shown only when it adds information: "complete" is - * the expected baseline and "unknown" is noise, so both are suppressed. + * The coverage condition is shown only when it adds information: `complete` is + * the expected baseline and `unknown` (rendered "not measured") is noise, so + * both are suppressed. The comparison is against the raw AXIS KEY, not the + * owner-facing label, so renaming the label leaves this correct. * Otherwise this renders the SAME owner-facing wording as the source detail * page and connection diagnostics (`formatCoverageAxis`'s humanized `value`, * e.g. "won't backfill" / "retryable gap") — never the raw internal axis key diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index a578c8ee7..89d7694ef 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -89,6 +89,30 @@ services: POSTGRES_USER: pdpp POSTGRES_PASSWORD: ${PDPP_POSTGRES_PASSWORD:-pdpp} POSTGRES_DB: pdpp + # Postgres ships defaults sized for a small dev database. A PDPP node is a + # write-heavy record store: a real instance reached 5.4M records / 76 GB, + # where the stock max_wal_size of 1GB forced a checkpoint every 16-22 + # seconds. Postgres logged "checkpoints are occurring too frequently" + # 8,209 times, ingest returned intermittent 503 + # (ingest_batch_storage_error), and two backend processes were killed + # under the write pressure. Raising the WAL ceiling and checkpoint window + # removed the warnings outright. + # + # These are deliberately modest -- safe on a 4 GB VPS, and far better than + # the defaults on anything larger. An operator with more RAM should raise + # shared_buffers (~25% of RAM) and effective_cache_size (~50-75%). + command: + - postgres + - -c + - max_wal_size=${PDPP_POSTGRES_MAX_WAL_SIZE:-8GB} + - -c + - min_wal_size=${PDPP_POSTGRES_MIN_WAL_SIZE:-1GB} + - -c + - checkpoint_timeout=${PDPP_POSTGRES_CHECKPOINT_TIMEOUT:-15min} + - -c + - shared_buffers=${PDPP_POSTGRES_SHARED_BUFFERS:-512MB} + - -c + - effective_cache_size=${PDPP_POSTGRES_EFFECTIVE_CACHE_SIZE:-1536MB} # No published port: Postgres is reachable only from the compose network. # If you publish one, change PDPP_POSTGRES_PASSWORD first. volumes: diff --git a/design-notes/browser-stream-status-honesty-2026-08-22.md b/design-notes/browser-stream-status-honesty-2026-08-22.md new file mode 100644 index 000000000..4db61756d --- /dev/null +++ b/design-notes/browser-stream-status-honesty-2026-08-22.md @@ -0,0 +1,183 @@ +# Browser-stream: the captcha tap, and the false "logged in" claim + +2026-08-22 + +Two owner-reported defects on the interactive browser-stream path, both +repeated across several days. They are unrelated in mechanism but share a +theme: the console asserted things it could not prove. + +## 1. "Can't tap the captcha on mobile" — the residual half + +Reported at least three times from 2026-08-18. Scroll and rotate worked; taps +did not. + +### The iframe theory is wrong, and worth retiring explicitly + +A reCAPTCHA checkbox lives in a cross-origin iframe, so the natural suspicion is +that synthetic pointer events do not cross the iframe boundary, or that +coordinates fail to resolve inside the iframe's document. + +**That is not what happens here.** The remote input path is coordinate-based all +the way down — neko dispatches X11 pointer events, and the CDP backend calls +`Input.dispatchMouseEvent` with raw `x`/`y`. Both operate at the browser +compositor level, *below* the DOM, so they cross cross-origin iframe boundaries +by construction. Nothing in the path hit-tests the top document: there is no +`elementFromPoint`, no `querySelector` against remote DOM, no frame targeting. +The only `elementFromPoint` in the streaming server is a calibration-beacon +diagnostic, not the input path. + +The captcha was never special. It is simply the one control an owner *cannot* +work around, so it is where a general tap defect gets noticed and reported. + +### What two prior commits fixed, and what they left + +- `a21a9a1be` stopped *dropping* touch taps that reported a non-zero + `event.button`. Correct: `button` is mouse-state and must not gate touch. +- `5274dbd4c` rerouted touch press/release onto the CDP mouse path, because + `Input.dispatchTouchEvent` does not reliably synthesize a `click`. + +Both are real fixes. Neither covers the residual case, because the first +commit stopped *filtering* on `button` but still **forwards the raw value**, and +downstream that value is not a filter — it is arithmetic: + +```js +// remote-surface 1.5.2, controllers/neko-pointer-controller.js +const x11Button = (event.button ?? 0) + 1; +``` + +X11 button 1 is primary. A touch `pointerdown` reporting `button === -1` — the +same non-spec value `stream-viewer-pointer-input.ts`'s own doc comment documents +as real on touch input paths — therefore becomes **X11 button 0**, which is not +a button. neko presses nothing, and the tap never clicks. + +Verified by replaying the exact client payload through the real installed +controller: + +| `button` on pointerdown | X11 call emitted | +|---|---| +| `0` (spec-compliant) | `buttonDown(1)` — correct | +| `-1` | `buttonDown(0)` — **no button pressed** | + +`pointerup` was already safe by luck: the controller prefers the *remembered* +press button over the event's own. `pointerdown` has nothing to fall back to. + +This also explains the reporting pattern. Engines that report a spec-compliant +`0` always worked, which is why the bug never reproduced on desktop and why it +looked intermittent rather than universal. + +### The fix + +`normalizedPointerButton` in +`apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts` +pins touch and pen to primary contact at the payload boundary. Touch and pen +have no secondary button, so this loses no information. Mouse is passed through +untouched — its `button` is meaningful, and normalizing it would turn every +right-click into a left-click. + +Guarded by `stream-viewer-touch-tap-oracle.test.ts`, which asserts against the +**real dependency** rather than restating our own arithmetic, so a future +dependency bump that changes the mapping fails loudly instead of silently. + +### Honest limits + +**This is not confirmed against a real phone.** I do not have the owner's +device, and I did not attempt a Reddit login (OTP/bot-detection sensitive, and +the owner has been explicit about not burning accounts). What is proven is +mechanical: the payload the client sends, replayed through the real controller, +pressed no button before this change and presses primary after it. Whether that +was the *only* remaining cause of the owner's symptom is unverified — it is a +genuine defect on the exact reported path, not a confirmed end-to-end repro. + +## 2. The false "logged in" state — the more dangerous half + +The UI told the owner he was already logged in to Reddit when he was not. + +### The evidence the UI used was a proxy + +`deriveSetupState` in `reference-implementation/runtime/static-secret-setup-status.ts` +reaches `first_sync_running` / `first_sync_pending` for a browser session from +`hasDraftSetupProgress`, which is satisfied by run evidence alone: + +```ts +return hasSetupMaterial || (setupKind === "browser_session" && hasRunEvidence(input)); +``` + +The console then rendered, in `connect/status/[connectionId]/page.tsx`: + +> "Login is complete and the first sync is running." + +**For a `browser_session` connection, the run IS the login attempt.** It starts +precisely so the owner can sign in inside the streamed browser. So a run row +proves a sign-in was *attempted*, never that it completed. The projection's own +comment stated the flawed inference out loud — "(the owner completed login and a +first sync started)" — which nothing in the projection verifies. `lastRun` also +satisfies it, so a *previous* failed run made the UI claim login was complete on +the current attempt. + +The asymmetry makes the defect obvious once seen. For the other two setup kinds +the same states carry claims that **are** backed by evidence: + +| Setup kind | Claim | Backed by | +|---|---|---| +| `static_secret` | "The provider credential is captured" | `setup_material.present === true` | +| `manual_upload` | "The import file is captured" | `setup_material.present === true` | +| `browser_session` | ~~"Login is complete"~~ | **nothing** — `defaultSetupMaterial` pins `present: false` | + +### The fix + +The browser-session copy now describes only what is observed — a sync is +running — and points the owner back at the browser, admitting the page cannot +confirm the login itself. The static-secret and manual-upload claims are +deliberately left intact, and a test pins them, so this does not over-correct +into scrubbing information that is genuinely proven. + +Guarded by `browser-session-login-honesty.invariants.test.ts`, which fails on +any phrasing that asserts a completed sign-in. + +### What was NOT fixed, and should be + +The connector layer already holds **real proof** of Reddit login state. +`isSessionLive` in `packages/polyfill-connectors/src/auto-login/reddit.ts` +fetches an owner-only JSON endpoint and requires HTTP 200 — ground truth, not a +heuristic, and deliberately so per its own doc comment. + +**That verdict is never plumbed into the console.** Wiring it through would let +the UI make a positive, evidence-backed claim instead of merely declining to +make a false one. That is a larger change across the RI projection boundary and +is left for a follow-up. + +Two related weaknesses found and not fixed here: + +- `isSessionLive` degrades to counting a logout-link selector when + `REDDIT_USERNAME` is unset — which is exactly the credential-less manual + handoff path. The repo's own tests document that this selector is unreliable + in both directions. +- An instance flips to `status = 'active'` on first successful ingest and is + **never re-validated**. A session that worked in June and has since been + logged out server-side still projects `active` → `healthy`. There is no + `last_verified_at` on the credential row, and for browser sessions the + credential row is bypassed entirely. + +Both mean "connection active" is a durable-row claim, not a live one. That is a +real instance of the same defect class and deserves its own change. + +## 3. "Couldn't reach the browser stream" — assessed, largely already sound + +Root-caused but not re-engineered, because the existing design is mostly honest. +The give-up message fires after 10 attempts with backoff, and is followed by a +diagnostic probe (`stream-reach-diagnostics.ts`) that recovers the real HTTP +status and error code — necessary because `EventSource` collapses every +pre-attach failure into a payload-less error. Four of six reasons already get +specific, actionable copy. + +Two genuine residual gaps, left for a follow-up rather than fixed blind: + +1. `managed_surface_window_settle_unavailable` (a 503 the server describes as a + transient restart worth retrying) is not in the classifier's match list, so + it falls through to the vague generic copy. +2. The generic message is shown *before* the probe resolves, so there is a + window where the owner reads network-flavored copy even when the cause is + precisely known moments later. + +Neither is a fabricated-green defect — the classification rests on a real HTTP +status — so they are accuracy gaps, not dishonesty. diff --git a/design-notes/connector-coverage-findings-2026-08-22.md b/design-notes/connector-coverage-findings-2026-08-22.md new file mode 100644 index 000000000..77b0cd2f6 --- /dev/null +++ b/design-notes/connector-coverage-findings-2026-08-22.md @@ -0,0 +1,214 @@ +# Connector coverage findings — USAA and Gmail (2026-08-22) + +Four owner-reported coverage gaps (ledger items D1–D4). Two of the four had a +wrong premise in the report; the evidence is recorded here because each finding +is the kind that gets rediscovered as something else entirely six months later. + +## D1 — USAA `transactions` stuck at 2 of 4: a marketing interstitial, NOT a UI change + +**The finding that will otherwise be rediscovered as "USAA changed their UI".** + +USAA serves a promotional interstitial in FRONT of a checking account page: + +``` +https://www.usaa.com/my/banking-offer/atm-deposit + ?accountId=…&accountType=checking&goto=https://www.usaa.com/inet/ent_home/CpHome +title: "Find an ATM | USAA" ("Depositing cash just got more convenient!") +``` + +`locateExportPage` navigated to the account URL, was redirected here, found no +Export button, and reported `source_structure_changed` — whose owner-facing +meaning is "the source's UI changed; retrying is pointless until the connector's +selectors are revisited". **The selectors were fine.** The connector was looking +at a page that never claimed to have an Export button. + +Only `LOGON_REDIRECT_RE` was checked after navigation. `/my/banking-offer` +matched neither that nor `USAA_ACCOUNT_DETAIL_ROUTE_RE`, so it classified as +`unknown` and fell through to the no-affordance path. + +### The correlation that proves it + +The denominator for `transactions` is **accounts, not statements** +(`emitTransactionsDetailCoverage`, `state_stream: "accounts"`): 5 accounts, 4 +transaction-eligible (Chase is `external-account`). + +| account | type | offered ATM deposit? | outcome | +|---|---|---|---| +| Checking | checking | yes → interstitial | **gap** | +| Family Checking | checking | yes → interstitial | **gap** | +| Signature Visa | credit-card | no | covered | +| American Express | credit-card | no | covered | + +**The 2 covered accounts were precisely the 2 credit cards** — the accounts that +get no ATM-deposit offer. Both checking accounts hit the interstitial; both +credit cards exported normally. That is the whole of the 2-of-4 gap. Captured +live 2026-07-14 in the connector's own page artifacts (URL + title). + +Note also: `pdf_template_unknown` is a *separate*, much smaller skip on the +statements→PDF path. It is NOT the cause of the 2-of-4 transactions gap, and +chasing it would have fixed nothing. + +**Fix:** navigate through the offer once, using the interstitial's own `goto` +param when it names a USAA account-detail route (off-host or wrong-section +`goto` values are refused, so a redirect chain cannot steer the connector). +Session death is re-checked *after* the hop, because an offer page can itself +bounce to logon — that must surface as session-dead (which triggers re-auth), +never as a missing affordance. + +**Watch for:** other banks doing the same thing. A promotional interstitial in +front of an account page is a generic pattern, and the failure mode it produces +— "the source changed its UI" — is maximally misleading. + +## D2 — a gap-opening path with no gap-closing path (general defect class) + +USAA `statements` simultaneously reported: + +- stream fact: `covered 10 / considered 10, checkpoint: committed` +- gap table: **4 rows still `pending`**, three with `attempt_count = 0` + +Both readings were current. They disagreed because the four "pending" +statements were not missing at all — every one has a durable `pdf_sha256` on its +record. They had been downloaded. + +### The class + +A pending detail gap leaves `pending` **only** on an explicit +`DETAIL_GAP_RECOVERED` (`connector-detail-gap-store.ts`). Nothing closes one +implicitly: not a later success, not a full-coverage `DETAIL_COVERAGE`, not a +committed checkpoint. + +USAA emitted that message for `transactions` +(`recoverServedAccountTransactionGaps`) and for both credit-card streams — +**but never for `statements`**. So the first statement PDF that ever failed to +download opened a gap that no subsequent run could close, however many times it +succeeded afterwards. Three of the four were never even re-attempted. + +> **The general shape: any stream with a gap-OPENING path but no matching +> gap-CLOSING path accumulates permanent false gaps.** It is silent — the stream +> looks fully collected on every axis except the gap table — and it is +> self-inflicted, needing no source misbehavior at all. + +**Worth auditing across every connector**: for each stream that can emit +`DETAIL_GAP`, confirm a `DETAIL_GAP_RECOVERED` path exists and is reachable. +The asymmetry within a single connector (USAA had closers for 3 streams and not +the 4th) suggests these are added per-stream as each is built, so the omission +is easy to repeat. + +**Fix:** `emitStatementCoverage` now emits recovery for each served gap whose +statement is hydrated this run, driven by `coverage.hydratedKeys` — the same set +that feeds the coverage numerator, so recovery and coverage cannot drift apart. + +## D3 — Gmail `message_bodies` has no denominator: PROHIBITED, not unimplemented + +Definitive answer to the owner's direct question ("are you sure it's not +possible?"). + +`manifests/gmail.json` declares `message_bodies` with `state_stream: "messages"`, +making it a **static single-parent detail stream** whose checkpoint status is +projected from the parent's own commit outcome. `runtime/index.ts:1514` +(`validateDetailCoverageAgainstManifest`) **throws and fails the entire run** if +such a stream emits `DETAIL_COVERAGE`. This is not a missing feature; emitting a +denominator is an error. + +The owner's reasoning is sound but lands one level off. The message count *is* +knowable — but it is the **parent's** number. Reporting it under +`message_bodies` would assert `covered == considered` for bodies that were +skipped or whose fetch failed. A real denominator would require a per-key +hydration tally, which this stream does not produce. + +`attachments` is the instructive contrast: it has no `state_stream`, and it +earns coverage from a genuine attempt-per-key tally — which is exactly why it +is permitted to emit at all. + +Already corrected in-tree (`cfe738071`); live evidence shows `collected: 4, +checkpoint: committed` with no fabricated coverage. + +## D4 — Gmail `attachments` "won't backfill": a forged impossibility proof + +The 32 terminal `too_large` gaps are **all collectible**. None is genuinely +oversized. + +Gmail's hydrator briefly sized attachments from imapflow's `meta.expectedSize`, +populated from the FETCH `RFC822.SIZE` item — the size of the **entire message**, +identical for every part of a multipart message. Against a per-part cap this +condemned every attachment of a message whenever their SUM crossed it. + +The durable signature, straight from the rows: + +| claimed "observed" | sum of that message's parts | largest single part | +|---|---|---| +| 32,229,094 | 32,218,046 | 7,711,218 | +| 30,062,404 | 30,051,218 | 8,709,138 | +| 29,830,196 | 29,800,496 | 10,947,340 | + +32 gaps, only **7 distinct claimed sizes** (real per-part sizes are never +byte-for-byte identical across distinct attachments); every claim ≈ the sum of +its message's parts; the smallest condemned item is **3,080 bytes** against a +26,214,400-byte cap. + +The connector was fixed (`1bf3f6cfa`, per-part BODYSTRUCTURE size), but the rows +already written stayed terminal, and `isProvenUnfillableGap` reads +`observed > limit` as durable per-item proof — so the false verdict outlived the +defect that produced it. + +> **Transferable lesson: a durable "proof" is only as good as the measurement +> behind it.** `too_large` was treated as unfalsifiable because a size beats a +> retry count as evidence. But the number itself came from a buggy source, and +> nothing re-checked it against the item's own recorded size. Any terminal state +> justified by a recorded measurement should be falsifiable by independent +> evidence. + +**Fix:** `classifyTooLargeProof` adjudicates a row against the item's OWN size — +requeue only on positive contradiction (`fabricated_proof`); keep terminal for +`proof_holds`, `no_corroborating_record`, and `not_a_size_proof`. Absence of +contradiction is not proof of fabrication. + +Repair tool: `scripts/repair/requeue-fabricated-too-large-detail-gaps.ts` +(gmail/attachments-locked, dry-run by default). + +**APPLIED 2026-08-22** to `cin_12407c1afb78d56848fe0b20`, with backup table +`gmail_gaps_backup_20260822034236` (32 rows): + +| | before | after | +|---|---|---| +| `recovered / temporary_unavailable` | 10,236 | 10,240 | +| `terminal / too_large` | **32** | **0** | +| `terminal / quarantined` | 3 | 0 | +| `pending / temporary_unavailable` | 0 | 31 | + +All 32 adjudicated `fabricated_proof` (0 `proof_holds`, 0 +`no_corroborating_record`). The 3 `quarantined` rows went through the existing +allowlisted tool in the same pass. **Zero terminal gaps remain.** + +Recovery is confirmed real, not hollow: within a minute, 4 of the 32 had already +moved to `recovered` with `hydration_status: hydrated` and content-addressed +blobs (2,248 / 13,162 / 220,273 / 1,970,646 bytes) — every one far under the cap +it had been condemned against. So the requeued gaps ARE picked up by the ordinary +recovery path; this is not the D2 defect class inverted. + +Each requeued row carries an audit trail +(`class: "too_large_proof_contradicted"`) recording the claimed size, the cap, +and the item's real size, so the repair is legible in the row's own history. + + +## Operational: a repair CLI that silently misread its own arguments + +Found while applying the D4 repair. `--reason too_large` (space-separated) +parsed as `reason = "true"`: the parser read a value only from `--key=value` and +substituted the boolean `true` otherwise. The operator saw + + --reason='true' is not requeueable (allowed: quarantined, ...) + +naming a value they never typed. The refusal was correct — `too_large` IS +refused by that tool by design — but the message pointed at the wrong thing, so +the tool looked broken in a different way than it was, and the real routing +(use the adjudicating tool) stayed hidden. + +Both repair CLIs now accept `--flag value` and `--flag=value`, and REFUSE a +value-taking flag given no value rather than defaulting. Defaulting is what +produced the defect; failing closed is the only safe reading of an ambiguous +argument list for a tool that writes to production. + +> **Worth checking in other operator tooling:** the `--key=value`-only parser +> with a `true` fallback is a common hand-rolled shape, and it fails silently +> and specifically on the argument form most operators type first. diff --git a/design-notes/connector-sidecar-packaging-2026-08-17.md b/design-notes/connector-sidecar-packaging-2026-08-17.md new file mode 100644 index 000000000..eee9b9de9 --- /dev/null +++ b/design-notes/connector-sidecar-packaging-2026-08-17.md @@ -0,0 +1,199 @@ +# Who owns a connector's native sidecar once connectors leave the server repo? + +**Status:** intake. No requirement proposed. Written from evidence produced while +shipping the Signal connector on 2026-08-17. +**Date:** 2026-08-17 + +## The question + +Several connectors shell out to a native binary they do not own: + +| connector | sidecar | license | how it ships today | +|---|---|---|---| +| slack | `slackdump` v4.4.2 | AGPL-3.0 | pinned tarball, SHA256-verified, builder stage in the RI `Dockerfile` | +| google_messages | `gmcli` | — | same arms-length-subprocess pattern | +| signal | `sigtop` v0.24.0 | ISC | built from pinned source in a Go builder stage (added today) | + +All three live in the **reference implementation's** `Dockerfile`. If connectors move +into their own distribution — the data-connectors reorg — that stops working: a +connector shipped separately cannot edit the server's image build. + +So: **how does an independently-distributed connector declare and obtain a native +dependency, and who verifies it works on the runtime that will actually execute it?** + +## Evidence from shipping sigtop today + +Seven build failures, in order. Every one was caught before shipping, but the pattern +matters more than the count: + +1. `golang:1.23` too old — sigtop needs Go ≥ 1.25 +2. missing `libsecret-1-dev` at build time +3. license file is `LICENSE.md`, not `LICENSE` or `COPYING` +4. **`libsecret-1.so.0` missing at runtime** — binary compiled cleanly, could not load +5. **GLIBC 2.38 vs 2.36** — `golang:latest` is Debian trixie, the runtime image is + bookworm; the binary ran in the builder and died in the final image +6. `sigtop -v` is not a valid subcommand +7. `sigtop version` is not either + +**4 and 5 are the load-bearing ones.** Both produced a binary that built successfully +and would have failed on the owner's first real sync. Neither is discoverable from the +connector's own source; both are properties of the *runtime image* the connector will +be executed in. + +That is the crux. A connector author can pin a version and a checksum. A connector +author cannot know the runtime's glibc, its installed shared libraries, or its +architecture — and today's evidence says those are exactly what break. + +## What the current pattern gets right + +Worth preserving whatever the packaging answer is: + +- **Pinned version + SHA256** on the downloaded artifact (`slackdump`), or a pinned + source tag with a commit-exact `SOURCE_URL` recorded (`sigtop`). +- **Isolated builder stage** — Go and build dependencies never reach the final image. +- **License and corresponding-source URL copied into the image**, which AGPL §6(d) + requires for `slackdump` and is good practice for ISC. +- **Build-time smoke test.** `slackdump version` and (now) an execute-and-check for + `sigtop`. This is what caught failures 4 and 5. A verification step written as + `... || true` would have shipped both. + +## Options, none yet chosen + +**A. Connector declares, runtime resolves.** The manifest names a sidecar (source, pinned +version, checksum, license) and the runtime image build reads those declarations and +produces the binaries. Keeps one place that knows the runtime's glibc and libraries. +Cost: the runtime build must enumerate every connector, which partially re-couples what +the reorg is trying to separate. + +**B. Connector ships prebuilt per platform.** Each connector distributes its own +binaries for supported platform triples; the runtime verifies checksum and executability +on load. Fully decoupled. Cost: connector authors take on cross-compilation and a +platform matrix, and today's evidence says that is precisely where the failures live. + +**C. Sidecar declared as a runtime prerequisite.** The connector declares "requires +`sigtop` ≥ 0.24 on PATH" and refuses to register when absent, with a clear message. Zero +packaging burden, but it breaks the property that makes the current product good — a +self-hoster following the docker/railway/fly.io steps gets working connectors with no +extra install. Slack works today because `slackdump` is *in the image*. + +## The constraint any answer must satisfy + +**Whatever ships must be verified against the runtime it will execute on, at build time, +by executing it.** Not "the artifact downloaded," not "the checksum matched" — those both +passed today while the binary was unrunnable. The only check that caught it was running +the thing. + +## Open questions + +- Does the reorg keep a single runtime image, or do connectors get their own containers? + Option B is much more attractive in the latter case. +- Is there an existing prior-art answer here? Language package managers with native + extensions solve a similar problem (Python wheels' manylinux, Node prebuilds), and + manylinux exists specifically because of the glibc problem hit today. Worth a sweep + before designing. +- How does a self-hoster on a non-Debian base fare today? Untested — the current + `slackdump`/`sigtop` stages both assume Debian. + +## Related + +Same shape as the collector/server contract gap +(`upstream-disclosure-window-2026-08-17.md` and the collector-contract findings): a +component whose correctness depends on a peer's version, with nothing verifying the pair +is compatible. Here the failure is loud at build time if a smoke test exists, and silent +until first use if it does not. + +--- + +## Proposed requirement (appended 2026-08-17 after prior-art research) + +Research: `~/.tmp/reorg-0814/sidecar-abi-prior-art.md` (corpus entry filed). Key finding: +manylinux, Node prebuilds, and N-API all declare compatibility as data and verify by static +analysis or eliminate the variable by construction — **no surveyed ecosystem executes the +artifact on the real target before accepting it**. The constraint this note demanded is a +genuine gap in prior art; adopting it puts this registry ahead of, not behind, the state of +the practice. + +Direction (option B, shaped by the registry design; proposed, not owner-ratified): + +1. **Static by default** — `CGO_ENABLED=0`/musl-static for any sidecar without a real + `dlopen` dependency; erases the glibc class by construction (verify per-tool, don't assume). +2. **ABI tags where dynamic is unavoidable** — per-artifact `{os, arch, libc, libc_floor, + linkage}` (manylinux/prebuildify model), built inside a pinned deliberately-old shared + build image (the registry's manylinux-image equivalent), so the floor is infrastructure, + not per-author judgment. +3. **`smoke_cmd` becomes a manifest/artifact field** — the trusted installer executes it on + the actual runtime at install time and refuses on failure; loader errors already + distinguish "missing library" from "symbol too new" with no parsing. +4. **Graceful fallback** — on smoke failure, try the static/alternate build before failing + the connector. + +Why not options A/C: A cannot survive in-app connector install (no image rebuild available +at user install time) — transitional-only by construction; C breaks the self-hoster +works-out-of-the-box property this note already names. + +Transitional: today's Dockerfile builder stages are server-repo property, untouched by the +connector-content move; recorded as a known coupling whose removal trigger is registry +artifacts carrying ABI-tagged (or per-connector-container) sidecars. For server deployments +the container sandbox tier ultimately makes the sidecar ABI self-contained inside the +connector's own image; the tag machinery chiefly serves bare-metal desktop. + +--- + +## The packaging rule (settled 2026-08-17, window 20 disposition) + +**Sidecar packaging keys off the connector's placement bindings, not one uniform +mechanism.** + +- **Network-authenticated sidecars** (`slackdump`) belong in the server's runtime image. + The tool reaches the provider over the network, so the server is a legitimate place to + run it, and the builder-stage pattern above is the right answer. +- **Session-bound sidecars** (`sigtop`) can only be acquired to the *user's* machine. + No server-side image stage can help, because the constraint is not where the file is — + it is where the key can be unwrapped. + +That sentence is what makes the rest of this note cohere, and it is why "put the binary in +the image" was the wrong instinct for Signal. + +### Evidence: Signal cannot run server-side, by construction + +Tested against real data on this host, four successive configurations: + +| attempt | result | +|---|---| +| container, no mounts | `open /root/.config/Signal/sql/db.sqlite: no such file` | +| + Signal data mounted read-only | `cannot decrypt database key: cannot connect to D-Bus session bus` | +| + host `/run/user/1000/bus` mounted | `EOF` (uid mismatch) | +| + `--user 1000:1000` | `An AppArmor policy prevents this sender from sending this message` | + +`~/.config/Signal/config.json` holds `encryptedKey` with `safeStorageBackend: kwallet6` +and no plaintext key. Mounting the database is insufficient because **the key is not in +the file** — it unwraps only through a session-bound keyring daemon. + +### Consequences adopted + +1. **Signal ships local-collector-only**, with a PATH/`SIGTOP_BIN` resolution and a clear + install error as the interim acquisition story. Connector code must not fetch + executables at runtime; a downloader in the npm package today would be the insecure + version of the signed, ABI-tagged registry artifacts already designed above. +2. **The constraint is now declared, not discovered.** `desktop_session` is a + first-class binding in `runtime_requirements.bindings`, and + `sourceKindFromManifestBindings` resolves it to `local_device` — the same placement + mechanism that already keeps browser connectors off the collector profile. The engine + refuses server-side placement up front rather than failing four D-Bus layers deep. +3. **The `sigtop` builder stage is removed from the Core image.** Shipping a binary that + cannot work there implies support that does not exist. The builder-stage pattern + remains proven via `slackdump`. + +### Edge case worth documenting, not shipping for + +Signal Desktop configured with `safeStorageBackend: basic_text` stores the key +**unwrapped**, so a server-side path does exist for users who have disabled their keyring. +That is a documentation note, not a reason to carry an image stage — and a connector that +declares `desktop_session` should keep declaring it, since the common configuration is the +session-bound one. + +### Carry-through + +The connector fleet was copied to `data-connectors` around this change. The manifest and +engine edits above were made in pdpp's canonical copy and **must be carried through the +cutover rather than silently diverging.** diff --git a/design-notes/cursor-provenance-audit-2026-08-20.md b/design-notes/cursor-provenance-audit-2026-08-20.md new file mode 100644 index 000000000..59678fca0 --- /dev/null +++ b/design-notes/cursor-provenance-audit-2026-08-20.md @@ -0,0 +1,114 @@ +# Detecting a cursor a connection never earned + +**Status:** deferred. Code removed from the branch; no requirement is proposed yet. +**Date:** 2026-08-20 + +## What was built, and why it is not in the tree + +`reference-implementation/runtime/cursor-provenance.ts` and its test were removed from +`fix/sweep-fairness-and-transformer-bounds` before that PR landed. The module was never wired into +anything: its only reference anywhere in the repo was its own test file, and it does not exist on +`main`. Its own commit (`6af425709`, "detect the three ways a cursor silently excludes data") says +so deliberately — it "stays unwired: it needs the full connection set, so it belongs in the +maintenance sweep rather than the per-run commit." + +Unwired, it still failed the zero-connector-knowledge conformance gate, because its watermark table +names eleven connectors in RI production source. That is a real gate finding, not a false positive. +Shipping the module unwired bought nothing and blocked a PR carrying real production fixes, so the +code came out and the analysis stays here. + +This note exists so the next person does not rediscover the defect, the ruling, or the blocker. + +## The defect: `duplicate_of_sibling` + +A watermark cursor ("fetch everything newer than T") is only sound when T was reached by *this* +connection walking its own history. Seed a new connection with another connection's high-water mark +and every record older than T becomes permanently unreachable — the connection will only ever ask +for newer items — while `covered == considered` keeps reporting complete coverage, because the run +genuinely did process everything it fetched. + +This is live, not hypothetical. ChatGPT connection `cin_484604984db7c091bd08b259` (created +2026-08-17) held a `conversations` cursor of `2026-06-19T20:30:04.127Z`, byte-identical to the +millisecond to the cursor of a different, paused connection (`cin_e4ab231c7d49b8f59e4c80ed`) that +reached that value on its own final run. Two separate accounts do not independently walk to the same +millisecond. The value was copied, and everything older than it is unreachable for the newer +connection. + +The signal is exact equality across sibling connections for the same connector and stream, never +proximity and never staleness. Staleness is explicitly not evidence: an account nobody has posted to +in six months has a legitimately frozen watermark, and firing on it would cry wolf on exactly the +quiet connections an owner is least able to check by hand. + +A finding is `suspected`, never a quantified loss. The evidence proves the cursor's *provenance* is +unsound, which makes any completeness claim built on it unfounded; it does not measure how much data +is unreachable. The right outcome is to withhold the healthy claim and tell the owner to re-seed. + +One rule was tried and disproved by the live fleet: flagging a cursor that predates its own +connection's `created_at` produced seven false positives across reddit, github, and notion. Those +watermarks store the newest *content item's* timestamp, not an observation time, so a connection +created today that fully walks an account whose newest post is from 2024 correctly stores 2024. The +rule was removed rather than tuned. + +## The ruling: this audit stays RI-owned + +The architecture owner's decision, which this note records so it is not relitigated: the watermark +specs must NOT move into connector manifests, unlike the sibling `cursor-band-contiguity` check +whose `cursor_shape` enum did move. + +Letting a connector declare which of its fields are checkable lets the audited party define the +audit's scope. A connector could omit or mis-path its watermark and silently exempt itself, or +mis-declare `valueKind` (`iso8601` as `epoch_seconds`), making comparisons meaningless with no +error raised. This is an integrity check *about* the connector evaluated *against* the connector, +so the connector cannot be the one who says what counts. + +The contrast with `cursor-band-contiguity` is the useful part. There, the manifest declares only a +closed enum (`cursor_shape: "imap_uid_band"`) and the RI owns every semantic — the paths, the +`UIDVALIDITY` epoch guard, the arithmetic. Declaring can only opt a stream *in*; omission yields +silence, never a healthy verdict. That asymmetry is what makes manifest declaration safe there and +unsafe here. + +## The blocker: the sibling-JSON route fails both ways + +The obvious fix — move `WATERMARK_SPECS` into an RI-owned JSON data file (e.g. +`reference-implementation/data/watermark-specs.json`) and load it at runtime, keeping the data +RI-owned rather than connector-declared — does not work. It was tried and probed empirically against +the real scanner, not just reasoned about. Both halves of rule 5 reject it: + +1. **The path.** `SANCTIONED_POLICY_RESOURCES` in + `reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts` is + deliberately an empty `Map`. Its own doc records that the two production files which used to load + a sibling RI-owned registry through it (`compact-record-history.ts`, `version-disposition.ts`) + were migrated *away* on purpose. A probe file loading such a JSON returns + `unsanctioned-policy-resource-path`. + +2. **The content.** Even with an allowlist entry, `classifySanctionedPolicyResource` parses the + file and walks it for any string — object key, value, or array element, at any depth — equal to a + manifest-derived connector key, reporting `hardcoded-connector-literal-in-ri-owned-json`. A + watermark-spec file is nothing but connector keys, so it fails here even if the path were + sanctioned. + +The scanner states the principle directly, and it is correct: moving a connector-identity fact out +of `.ts` source into a sibling RI-owned JSON file "is exactly as much self-attested connector +knowledge as the literal it replaced, reached via a different seam." + +Adding an allowlist entry to get past this would be defeating a gate that is working as designed, +not satisfying it. + +## The viable path + +Wire the check into the **maintenance sweep**, with the watermark specs supplied by the call site +rather than held as a module-level table in RI production source. + +This fits the shape of the check anyway. `evaluateCursorProvenance` needs the full set of +connections for a connector/stream to compare siblings, which is a sweep-time input, not a per-run +one — the reason the original commit left it unwired. Making the specs a parameter means the pure +comparison logic stays RI-owned and connector-agnostic, and the identity table lives wherever the +sweep legitimately assembles per-connector context. + +That is a real design task with its own review surface: where the sweep gets the specs, whether a +finding withholds a healthy verdict or only annotates, and how `suspected` is surfaced to an owner +without implying a measured loss. It deserves its own PR rather than being rushed to unblock CI. + +Recovering the code: `git show 6af425709 -- reference-implementation/runtime/cursor-provenance.ts` +(and the sibling test path). The module and its 9 tests were passing when removed; only the +conformance gate objected, and only to the identity table. diff --git a/design-notes/failure-diagnosability-2026-08-18.md b/design-notes/failure-diagnosability-2026-08-18.md new file mode 100644 index 000000000..169cfa001 --- /dev/null +++ b/design-notes/failure-diagnosability-2026-08-18.md @@ -0,0 +1,635 @@ +# A failure must not destroy its own cause + +**Status:** intake. No requirement proposed. Written from five production +failures observed on 2026-08-18, two of them fixed the same day. +**Date:** 2026-08-18 + +## Why this note exists + +Five failures in about one day. All five have the same shape: **something went +wrong, and the evidence needed to act on it was destroyed by the code that +handled it.** In four cases the failure itself may well have been transient and +harmless. What made them cost a day was that nothing downstream could tell. + +1. **`[object Object]`.** `packages/polyfill-connectors/src/reference-blob-uploader.ts` + built its error text with `String(body.error ?? statusText)`. The RI host + always shapes `error` as an object — `pdppError` in + `reference-implementation/server/request-helpers.ts:92` writes + `{code, message, type}` — so `String()` produced the literal + `"[object Object]"` for every host-side failure. That discarded the cause of + 24 quarantined Gmail attachment gaps, each recorded as + `blob upload failed (503): [object Object]`. Fixed in `457e23e93`. +2. **Silent statement_timeout.** `observeConnectorSummaryEvidence`'s outer catch + treated a typed `PostgresStatementTimeoutError` like any other error and + routed it to `markAllConnectorSummaryEvidenceDiscoveryFailed`, durably + writing `record_snapshot_state='failed'` across every row in scope. Neither + that path nor `repairCandidatePostgres` logged anything. 25 of 29 evidence + rows degraded in production with **zero** log output. Fixed in `1d8995b0f`. +3. **Empty `failure_reason`.** A failed ChatGPT run (`run_1787075769450`) wrote + zero log lines matching its own `run_id`, and `run_history.failure_reason` is + empty on every failed row. Only `terminal_reason` and a + `connector_error_json` blob survived. Not fixed. +4. **`[REDACTED]`.** HEB connection `cin_c875ca3ec8b6ce2c283a4288` failed with + `connector_error_json = {"code": null, "message": "heb_session_failed: [REDACTED]", "retryable": false}`. + The cause is literally the string `[REDACTED]`. Partly fixed same-day in + `46887c2e8`, which populates the `code` channel; the `message` channel is + the subject of the proof-of-concept below. +5. **Unpublished dependency.** Every published `@pdpp/local-collector` + (1.5.1–1.5.4) has `import ... from "@pdpp/reference-contract/common"` as line + 1 of `dist/polyfill-connectors/src/local-device-client.js`, but that package + is not in `dependencies` and does not exist on npm. Every install crashes + with `ERR_MODULE_NOT_FOUND` on any invocation, including `--version`. It + works in the monorepo because pnpm resolves it through the workspace link. + +Numbers 1–4 are error handling. Number 5 is packaging, and it belongs to a +different family; it is addressed separately at the end. + +## First, the scale numbers are wrong + +The intake for this note claimed ~246 bare catches and ~282 stringified-error +coercions. Both were re-measured. The raw catch count is **higher** than +claimed and the story is **much smaller**. + +| metric | intake claim | measured | +|---|---|---| +| bare `catch {` (non-test) | ~246 | **439** | +| bare `catch {` (tests, separate) | — | 207 | +| `catch (e) {}` empty body (non-test) | — | **0** | +| `String(err)`-style on error-named values | ~282 | **173** | +| `REDACTED` (non-test, in scope) | ~58 | 35 | + +The original grep almost certainly used `catch\s*\{`, which also matches +`catch (e) {`. It conflated two populations while undercounting the one it +named. + +Then 55 of the 439 bare catches were read and classified, one per file, across +55 distinct files: + +| class | count | share | +|---|---|---| +| benign cleanup | 8 | 15% | +| benign optional parse | 24 | 44% | +| benign probe | 21 | 38% | +| **fault-swallowing** | **0** | **0%** | +| **degrades durably** | **2** | **3.6%** | + +**About 97% of bare catches are benign, and the fault-swallowing class is +empty.** The dominant patterns are `JSON.parse` → `return null` on stored +manifests and cursors, `new URL(x)` → `return false` in SSRF validators, and +telemetry taps explicitly commented "must never break the streaming path." +Several carry a comment justifying the swallow. This is a deliberate house +style, not neglect. **There is no catch-block crisis here, and a campaign to +migrate 439 call sites would be almost entirely waste.** + +What the sample did find is three real defects, verified end-to-end to their +persistence points: + +- `packages/polyfill-connectors/src/statement-content-fingerprint.ts:142` — + any PDF text-extraction failure returns the all-null fingerprint, which is + then persisted as statement `content`. The record durably says "no extractable + content" whether the PDF is genuinely empty or the extractor crashed. USAA's + sibling path at `statement-pdfs.ts:502` shows the fix: it emits `onSkip` with + `structuralErrorDiagnostic(err)`. +- `packages/polyfill-connectors/connectors/gmail/index.ts:1418` — an IMAP body + fetch failure returns null bodies and the message is emitted anyway, with no + `DETAIL_GAP` and no coverage marker. Given the measured ~4.65 KB/s IMAP + throttle on this connector, transient failures are expected, so bodyless + messages land durably with nothing distinguishing them from empty mail. +- `reference-implementation/server/routes/as-grant-revoke.ts:157` — + `String(e?.message ?? hookErr)`, where the catch neither rethrows nor changes + the response. A failed grant-revoke side effect vanishes into one unreadable + log line while the caller gets a success envelope. + +Both durable-degradation sites are the same shape: **a fail-closed default that +is indistinguishable from a legitimate empty result.** Neither needs a `try` +restructured; each needs one extra field saying why the value is null. + +The `457e23e93` archetype has **no surviving siblings** — a targeted search for +`String(...)` over parsed-JSON HTTP error bodies returns zero hits. And all 35 +`REDACTED` occurrences are redaction *mechanism* (regex constants, scrub tables, +sanitizers), not a bare `message: "[REDACTED]"` standing in for a lost cause. + +So the honest headline is **three fixes, not 246.** The rest of this note is +about why those three happened and what makes the next one impossible. + +## What this codebase already gets right + +There is a good, half-built convention here, and it is worth naming precisely +because the answer is to finish it rather than import something foreign. + +**Two channels with opposite disciplines.** Stated outright in +`packages/polyfill-connectors/src/terminal-error.ts:44-57` and implemented once, +correctly, in `buildTerminalConnectorFields` +(`reference-implementation/runtime/index.ts:2846`): + +- `code` is a **typed** channel. It is *validated, never redacted*, and fails + closed to null. `boundConnectorErrorCode` tests it against + `/^[a-z][a-z0-9_]{1,63}$/` and drops anything malformed. +- `message` is a **prose** channel. It is *redacted and truncated*, never + trusted — `boundConnectorErrorMessage` runs `redactStderrTail` then caps at + 500 characters. + +The security argument for the asymmetry is explicit in the source: `code` is +exempt from redaction *only because* the charset makes it incapable of carrying +a credential, a URL, or a stack trace. That is a genuinely good design, and it +is the thing to generalize. + +Three more pieces are already right: + +- **`recovery_hint` is a closed vocabulary** of 8 actions + (`runtime/connector-gap-bounding.ts:117`), with the design intent stated + well: *"a connector requests an ACTION this way; it never gets to pick one by + shaping its `code` or free-form `message` text."* +- **Structured failure evidence on the row.** `search_index_dirty` records + `last_error`, `attempts`, and `next_attempt_at` atomically + (`queries/search/index-dirty/record-failure.sql`), with the comment + *"observable evidence, not just a console.warn line."* This is the right + instinct and the right place to put it. +- **The typed-error → reason-code mapping** the `1d8995b0f` fix introduced: + `err instanceof PostgresStatementTimeoutError ? REASON_CODES.STATEMENT_TIMEOUT : default`. + That is exactly the shape the terminal design needs, written once. + +### Where the convention is only half-built + +The pattern exists. Its *closure* does not. + +- **Closure is upheld by hand-copied `Set`s and tests, not by types.** + `RECOVERY_ACTIONS` is a `Set`. Both `REASON_CODES` objects are + unexported `as const` with no derived type. `codeToStatus` + (`routes/ref-error-status.ts:89`) is `Record`, so an + unregistered error code compiles fine and silently becomes a 500. Only + `SharedConnectionConditionReason` is a real derived union — and it is the one + vocabulary with a single producer and a single consumer. +- **It has already drifted.** `scripts/stream-health-audit/authority.ts` + maintains duplicate `Set`s that omit `repair_statement_timeout` and contain + `"summary_evidence_unavailable"`, which exists in no const. + `"update_connector"` is emitted at `runtime/connection-health.ts:1982,2747` + and is not in `RECOVERY_ACTIONS`. +- **~170 error subclasses, no shared base.** The de-facto common field is + `code: string`, but the HTTP-status carrier is variously `statusCode`, + `httpStatus`, and `status`. Discrimination splits three ways: 134 `instanceof` + sites, 74 `.code ===` sites, and 3 fragile `.name === "..."` string + comparisons. +- **The typed channel degrades into prose-sniffing.** When a recovery hint is + absent or invalid, `inferRecoveryAction` + (`runtime/connector-gap-bounding.ts:739`) regex-matches the free-form message + to guess an action. `connector-coverage-policy.ts:167` matches connector + reason strings by substring. These are the seams where a structured design + silently becomes a guess. +- **There is no logger.** No module exports a log API. Pino exists but is wired + to exactly one call site (`transport.ts:335`) and is never exported, so no + library module can reach it. Everything else is `console.*` with a + `[module-tag]` prefix — **19 calls in the entire server, none carrying + `run_id`.** That is the mechanical reason incident 3 produced zero lines + matching its own run id: there is no facility that would have written one. + The real correlation channel is the spine (`lib/spine.ts:483`), which carries + `run_id`, `trace_id`, `request_id`, and `grant_id`, and hard-rejects a + malformed event. `packages/polyfill-connectors/src/` has no spine access at + all, which makes it the least observable surface in the system. + +## The invariant + +The candidate invariant from intake was: + +> A failure must never lose the information needed to act on it. Every failure +> that crosses a durability boundary carries a machine-readable code, a +> human-readable cause that is not a stringified object, and a PII-safe +> diagnostic detail. A catch that discards a fault is a bug, not a style choice. + +The last sentence should go. The measurement says the fault-swallowing class is +empty and 97% of bare catches are correct, so "a catch that discards a fault is +a bug" indicts a house style that is not what broke. It would also push toward +a 439-site migration that buys nothing. + +The rest is close but describes a payload rather than a property. What actually +failed in all four cases is narrower and more testable: + +> **A failure that crosses a durability boundary must carry a cause that is +> reconstructable from what is written down.** Every failure persisted to a row, +> returned to a caller, or shown to the owner carries (a) a machine-readable +> code from a closed vocabulary, and (b) a human-readable cause. Any transform +> applied on the way out — coercion, redaction, truncation, classification — +> must be **category-preserving**: it may drop detail, but it may never leave a +> value whose failure class can no longer be told apart from a different one. + +The operative word is *category-preserving*. `[object Object]`, `[REDACTED]`, +`''`, and `record_snapshot_state='failed'` with no reason are all the same +defect under this rule: each is a value that survived the boundary while +becoming indistinguishable from every other failure that produced the same +placeholder. + +### Testing it against the five incidents + +| # | prevented? | why | +|---|---|---| +| 1 `[object Object]` | **yes** | The coercion is not category-preserving: every distinct host error maps to one string. Caught by the rule directly. | +| 2 silent timeout | **yes** | A cancelled read and a genuinely bad row both wrote `failed`. Distinguishing them is exactly what `1d8995b0f` added, and it is what the rule requires. | +| 3 empty `failure_reason` | **partly** | The rule forces the field to be populated. It does **not** by itself produce a correlated log line — that needs a logger this codebase does not have. | +| 4 `[REDACTED]` | **yes** | Redaction that collapses distinct reasons to one token is not category-preserving. | +| 5 unpublished dep | **no** | Nothing was mishandled. No failure crossed a boundary; the artifact never ran. Different family. | + +**Where it would not have helped, honestly:** + +- It would not have prevented incident 5 at all. +- For incident 3 it fixes the durable row but not the missing log. Someone + debugging by `grep run_id` still finds nothing until a logger exists. +- It says nothing about failures that are never caught in the first place, or + about a correct code attached to a wrong diagnosis. +- It does not address the two durable-degradation sites found by measurement + (`statement-content-fingerprint.ts:142`, `gmail/index.ts:1418`). Those write a + *successful-looking record* with a silently-null field; no failure crosses a + boundary, so the invariant never engages. They need a different rule — a + fail-closed default must be distinguishable from a real empty result — and + that rule is worth stating separately rather than stretching this one. + +## The mechanism + +One small thing, not a framework. The two-channel design already exists and is +already correct; what is missing is that its vocabularies are open and its +transforms are not category-preserving. Three changes, in order of leverage. + +**1. Close the vocabularies with types, not `Set`s.** Every reason vocabulary +becomes an exported `as const` with a derived union: + +```ts +export const RECOVERY_ACTIONS = { + RETRY_BY_RUNTIME: "retry_by_runtime", + // ... +} as const; +export type RecoveryAction = (typeof RECOVERY_ACTIONS)[keyof typeof RECOVERY_ACTIONS]; +``` + +This is a mechanical change with immediate payoff: `"update_connector"` and the +drifted audit `Set`s become type errors rather than silent divergence, and +`codeToStatus` stops turning an unregistered code into a 500 by default. It +requires no call-site migration — only the declarations move. + +**2. Make redaction category-preserving.** This is the subject of the +proof-of-concept below, and the finding there is the most interesting result in +this note. + +**3. Populate `failure_reason` from the terminal event.** The empty field in +incident 3 is not a bug in the ordinary sense. It is a hardcoded literal, in +both backends, with a comment explaining why: + +```ts +// reference-implementation/server/stores/run-history-writer.ts:275 and :366 +const terminalReason = typeof event.data.reason === "string" ? event.data.reason : null; +const failureReason: string | null = null; +``` + +The comment says `failure_reason` is a scheduler-only classification and is +"left null rather than fabricated, since no Slice A reader depends on it for +non-scheduled runs." That reasoning was sound when written and is now false — +`ref-spine-correlations-list` and the console both read it. **This is worth +dwelling on: the information was dropped deliberately, on a reader-side +assumption that later stopped holding.** No lint rule catches that. It is an +argument for making the *durable schema* carry the classification unconditionally, +so a writer cannot decide on a reader's behalf that a cause is not worth keeping. + +### The gate that makes regression impossible + +This repo already has the right mechanism, used twice, and it is better than a +linter rule for this purpose. `lefthook.yml` runs +`check-direct-prepare-conformance.ts`, which pins grandfathered sites at exact +`(path, line)` in a checked-in allowlist and fails on three divergences: a +**new** hit not in the allowlist, a **stale** row whose site moved or was +migrated, and a **duplicate** row. The polyfill-connectors `noAwaitInLoops` gate +has the same shape. The comment states the property that matters: + +> The rule still fires on any NEW direct-prepare anywhere, INCLUDING a new one +> inside an already-allowlisted file, and additionally fails on a STALE +> allowlist row so exceptions cannot be carried silently. + +That is precisely the migration mechanism this design needs, and it already +exists. A new `check-error-envelope-conformance.ts` in the same shape would +enforce the one defect worth banning outright: **`String(...)` applied to a +caught value or a parsed HTTP error body.** That is a real, narrow, mechanically +detectable pattern with exactly one known instance left +(`as-grant-revoke.ts:157`), so the allowlist starts at ~1 entry rather than 439. + +Biome cannot express this. It is version 2.5.6 extending `ultracite`, and the +repo already works around missing rules with grep gates — the `no-double-cast` +job exists because "Biome/Ultracite has no equivalent to typescript-eslint's +`consistent-type-assertions` yet." A `String()`-on-caught-value rule needs type +information about the argument, which is why the grep gate is the honest answer +here rather than a stopgap. + +**What should not be banned:** empty `catch {}`. The measurement says 97% are +correct and the fault-swallowing class is empty. A lint rule there would +generate 439 suppression comments and teach people to ignore the gate. + +## Proof of concept: category-preserving redaction + +Incident 4 was chosen over incident 3 because another agent is already working +in `run-history-writer.ts`, and because it turned out to have the more +interesting answer. + +While this note was being written, `46887c2e8` landed a **complementary** fix +for the same incident from the other side: it populates `TerminalError.code` on +the session-establishment path, so the *typed* channel carries the reason even +when the prose channel is destroyed. That is the right first move and it +confirms the two-channel design is the one to generalize. It does not restore +the `message`, which is what an owner reads and what `inferRecoveryAction` +consumes, so the two fixes stack rather than compete. The proof-of-concept below +touches `runtime/stderr-redact.ts`, which `46887c2e8` does not; both were +verified green together. + +**The defect, reproduced exactly.** `LONG_OPAQUE_RE` +(`runtime/stderr-redact.ts:43`) is `/\b[A-Za-z0-9_-]{24,}\b/g` — an **entropy** +heuristic, aimed at unlabelled API keys in stack traces. Categorical reason +tokens match it too: + +``` +"heb_session_failed: login_form_never_appeared" -> "heb_session_failed: [REDACTED]" +"usaa_session_failed: source_unavailable" -> "usaa_session_failed: source_unavailable" +``` + +`login_form_never_appeared` is 25 characters and carries no PII whatsoever. +`source_unavailable` is 18 and survives. **Whether a failure stayed diagnosable +was decided by the length of its reason token.** Run through the real production +boundary, `boundConnectorErrorMessage` reproduces the exact production string +`heb_session_failed: [REDACTED]`. + +There is a second half to this. Once the message is destroyed, +`inferRecoveryAction` regex-matches the *redacted* text to choose a recovery +action, and returns `"unknown"`. So the redaction did not just cost a human +reader the cause — it silently degraded the machine-actionable output too. + +**The finding: shape cannot fix this.** The obvious fix is a smarter pattern — +preserve alphabetic `snake_case`, redact anything with entropy. Tested against +real secret shapes, it separates cleanly: + +``` +login_form_never_appeared kept sk_live_ redacted +heb_verification_code_not_provided kept eyJhbGciOiJIUzI1NiIsInR5cCI6... redacted +two_factor_challenge_unrecognized kept a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6 redacted +``` + +And then it fails, in the way that matters: + +``` +tim_nunamaker_gmail_com kept <- a personal name +``` + +**A name is alphabetic snake_case too.** No regex distinguishes a declared +reason token from a person's name, because the difference is not spelling — it +is *provenance*. That is the load-bearing conclusion, and it is why the +mechanism is an allowlist rather than a better pattern. + +So: a token survives only if the connector **declared** it in advance. The +declaration is reviewable in the connector's source, where a human reading +`login_form_never_appeared` can see it is a constant, and would see a name for +what it is. + +```ts +// reference-implementation/runtime/stderr-redact.ts +next = next.replace(LONG_OPAQUE_RE, (match) => (declared?.has(match) ? match : "[REDACTED]")); +``` + +The full change is 36 lines, most of it the comment explaining why shape does +not work. Six tests in +`reference-implementation/test/stderr-redact-declared-reasons.test.ts` pin the +four properties the fix must have, including the two that say what it +deliberately does not do: + +``` +✔ regression: the production defect — a declared reason token is destroyed by length alone +✔ a declared reason token survives redaction, so the owner sees the real cause +✔ secrets are still redacted even when a declaration set is supplied +✔ an UNdeclared reason token still redacts — declaration is the safety property, not spelling +✔ callers that do not opt in are byte-identical to the previous behaviour +✔ disclosed pre-existing gap: this redactor is not a PII control +``` + +**Verification.** 18/18 tests pass across the new file and the existing +`stderr-redact` suite; 20/20 across `connector-gap-bounding` and +`device-exporter-sanitize` consumers; 16/16 on the systemic-failure redaction +and stderr-tail oracles. `tsc --noEmit` clean, `biome check` clean. The +declaration set is optional and defaults to empty, so every existing call site +is byte-identical — pinned by its own test, since a redaction change that +silently widened what escapes would be much worse than the bug it fixes. + +**Disclosed while building it.** Today's redactor already passes +`tim.nunamaker@example.com` and `tim_nunamaker_example` through untouched, with +no options involved — both are under the 24-character threshold. This is +pre-existing and independent of the change, but it should be recorded plainly: +`LONG_OPAQUE_RE` is an entropy heuristic and **is not a PII control**, though +its name and position invite reading it as one. The declared-token change can +only ever reduce what escapes, never widen it. Whether a real PII boundary is +needed here is a separate question this note does not answer. + +## Incident 5 is a different family + +Nothing was mishandled. No failure crossed a boundary. The artifact simply never +ran anywhere except where it was built — the same shape already recorded in +`connector-sidecar-packaging-2026-08-17.md`, which put it well: + +> Whatever ships must be verified against the runtime it will execute on, at +> build time, by executing it. + +The pleasant surprise is that **the smoke test already exists and is already +correct.** `packages/local-collector/scripts/pack-install-run.ts` packs the +tarball, installs it into a clean temp npm project outside the workspace with an +isolated `HOME` and cache, and then *executes the installed binary* — `advertise`, +`enroll`, and `run --connector codex` against an in-process reference server. It +asserts forbidden packages are absent and the bin is executable. An `npm install` +of a package whose dependency does not exist on npm fails at the install step. +**It would have caught this.** + +It is wired to `pnpm --filter @pdpp/local-collector run verify`, and `verify` is +`pnpm test && pnpm validate:package`. `pack-install-run` is not in it. CI calls +`verify` at `.github/workflows/semantic-release.yml:130`. + +So the fix is one line — add `pack-install-run` to `verify` — and the rule is: + +> **A package's release gate must install the packed artifact in a clean +> environment and execute it.** Not "the build passed," not "the types check" — +> those both passed while every published version was unstartable. + +The right place is the existing `verify` script, so it runs in +`semantic-release.yml` before publish and stays available locally. The same gap +should be checked for `@pdpp/cli` and `@pdpp/mcp-server`, which have sibling +`verify` scripts. Worth noting separately: `pnpm` workspace linking is what +*hid* this, so any check that resolves through the workspace is structurally +incapable of finding it. That is the same lesson as the sidecar note, one layer +up the stack. + +## Cost, and what stays broken + +Hundreds of call sites cannot and should not be migrated. The measurement is +what makes this affordable — the real work is small: + +1. **Close the vocabularies** (types only, no call-site changes). Highest + leverage, lowest risk. Turns existing drift into compile errors. +2. **Wire `pack-install-run` into `verify`** for all three published packages. + One line each; closes incident 5 permanently. +3. **Land the redaction change** and give connectors a place to declare reason + tokens. Already proven; needs the declaration plumbed from the manifest. +4. **Populate `failure_reason`** in `run-history-writer.ts` for non-scheduler + runs. Needs coordination — another agent is in that file. +5. **Add the conformance gate** for `String()` on caught values, starting from a + ~1-entry allowlist. +6. **Fix the three measured defects** — `statement-content-fingerprint.ts:142`, + `gmail/index.ts:1418`, `as-grant-revoke.ts:157`. + +New code is forced onto the new path by step 5, which is the only step that has +to be right the first time; the rest are additive. + +**What stays broken in the meantime:** + +- There is still no logger, so incident 3's "zero lines matching the run_id" + stays true even after `failure_reason` is populated. A structured logger with + correlation fields is a real piece of work and is not scoped here. The spine + is the closest existing thing and connectors cannot reach it. +- `inferRecoveryAction`'s prose-sniffing fallback and + `connector-coverage-policy.ts`'s substring matching both stay. They are the + seams where the typed design degrades into a guess, and closing them means + deciding what happens when a connector supplies no hint at all. +- The ~170 error classes keep their three discrimination styles and their + `statusCode`/`httpStatus`/`status` drift. The duck-typed `.code` convergence at + the HTTP boundary works well enough that a base-class migration is hard to + justify on today's evidence. +- The two fail-closed-null sites are real but need their own rule; the invariant + in this note does not reach them. + +## Open questions + +- Where do connectors declare their reason vocabulary? The manifest is the + obvious home, and `reason-display-messages.ts` already keys on + `(connector_key, reason_code)` with an anti-parrot rule — but its + exhaustiveness is enforced only by an AST-scanning test, which is the same + open-vocabulary weakness described above. +- Should `failure_reason` be a closed union rather than free text? There is + already an unexported closed union in `runtime/classify-runtime-failure.ts:14` + that never returns empty. Exporting it may be most of the answer. +- Is a real PII boundary needed where `redactStderrTail` currently sits, given + it passes plain email addresses through today? +- Do `@pdpp/cli` and `@pdpp/mcp-server` have the same unpublished-dependency + defect? Only `local-collector` was checked. + +## Provenance + +Written from five production failures on 2026-08-18. Incidents 1 and 2 fixed +same-day in `457e23e93` and `1d8995b0f`. Scale numbers re-measured against +`deploy/prod-plus-fixes-0817`; the catch classification is a 55-site sample +across 55 files (12.5% of 439), so the two durable-degradation sites are +verified end-to-end but any extrapolation from them should be treated as an +order-of-magnitude hint, not a census. The redaction proof-of-concept and its +six tests are working code, not a sketch. + +Related: `connector-sidecar-packaging-2026-08-17.md` (incident 5 is the same +"verified where it was built, not where it runs" shape) and +`summary-evidence-projection-controller-2026-08-18.md`, whose subject is the +sweep that produced incident 2. + +--- + +## Addendum: the wider class is failures nothing reports + +Added 2026-08-18 after the note was applied to code outside this repo. + +An engineer migrating this instance's Postgres read the draft and found the +same defect in their own verification gate within five minutes: + +```bash +tgt_rows=$(psql -Atqc "SELECT count(*) FROM ${tbl};" 2>/dev/null || echo "ERROR") +``` + +That is `[object Object]` in bash. The gate fails correctly and exits non-zero +while collapsing permission denied, a missing table, a dropped connection, and +genuine data loss into one string. On migration night that is the difference +between a five-minute fix and a torn-down restore. It appeared twice, and the +fix was the same: capture stderr and log it before failing. + +Worth recording because it says the rule is not TypeScript-specific and not +about error objects. It is about any transform on the way out of a failure +path that keeps the fact of failure and discards which failure. + +### The count that reframes it + +That engineer's session produced four significant findings, and only one was a +thing breaking loudly: + +- Postgres crash-recovering behind `RestartCount: 0` and green healthchecks +- meilisearch crash-looping behind a healthcheck that only probed its own port +- promtail dying on a full disk without telling anyone +- `pg_dump` exiting 0 while producing an unrestorable dump + +Four of five were *nothing reported it*, not *it broke*. Our five incidents +were the narrower shape — something reported a failure and destroyed its cause. +Both belong to one family, and the wider one is the more dangerous half, +because the narrow shape at least leaves a row to investigate. + +So the invariant needs a second clause. The first is already stated above: a +failure that crosses a durability boundary must carry a code, a human cause, +and a PII-safe detail. The second: **a component whose failure is survivable +must still be observable — a supervisor that restarts, a probe that recovers, +or a process that degrades silently is not thereby healthy, and something must +say so.** `RestartCount: 0` is the canonical false negative: the container +never died, so every container-level signal stays green while the process +inside it crashed and recovered. + +This is why the missing logger matters more than its size suggests. Nineteen +`console.*` calls in the server and none carrying `run_id` is not merely +inconvenient; it means the only detection layer for an in-process crash is the +database's own log, which on this host was reaching no aggregator at all — +promtail scrapes the systemd journal and has neither docker discovery nor +socket permission. The blind spot was total, not partial, and nothing reported +that either. + +### What this does not change + +The remediation scope stays three sites. The measurement in this note stands: +439 catches, 97% benign, the `[object Object]` archetype with no surviving +siblings. Adding a second clause to the invariant does not widen the migration +— it widens what counts as evidence when deciding whether a component is +healthy, which is a monitoring question, not a refactor. + +### A third variant: reported correctly, in a field nobody read + +The migration engineer, having just added the second variant, produced the +third by making the mistake himself an hour later. He checked whether the +maintenance sweep's backlog had drained: + +```sql +SELECT record_snapshot_state, count(*) FROM connector_summary_evidence GROUP BY 1; +-- current | 29 +``` + +Twenty-nine of twenty-nine current, so he reported the backlog clean and +warned that a post-deploy measurement could not distinguish the fix from the +status quo. The real backlog at that moment was nine rows, and the sweep had +logged seventy consecutive no-progress passes. `dirty` is a different column. + +Nothing was hidden, destroyed, or masked. The failure was fully and correctly +recorded, in a field he did not read. + +This is the same error made earlier the same day from the other direction — +reading `connector_summary_evidence` columns and reporting "15 of 21 green" +while the Sources page showed otherwise. Both readers were competent, both +queried real data, and both were confidently wrong. + +The structural cause is that this table carries seven signals — `dirty`, +`state`, `record_snapshot_state`, `terminal_facts_state`, +`manifest_declaration_state`, `retained_bytes_state`, +`list_summary_projection_state` — none of which is the verdict, while the +actual verdict lives in `isHealthyConditionSet` +(`reference-implementation/runtime/connection-health.ts:1739`), ten conditions +evaluated together. Any reader who samples one column gets a plausible answer +that is not the answer. + +**A system with N independent health signals and no single authoritative one +invites every reader to pick a different signal and be confidently wrong.** + +The remedy is not better logging — the logging was perfect. It is that a +verdict must have exactly one source, and the raw signals must be hard to +mistake for it. That is a naming and API problem: `record_snapshot_state` reads +like the state of the record snapshot, which it is, and like the state of the +row, which it is not. + +Worth noting what this shares with the other two variants and what it does +not. All three end with an operator holding a wrong conclusion. Only the first +involves anything being destroyed, and only the second involves anything being +unreported. The third needs neither — it is sufficient to offer several true +answers to slightly different questions and let the reader choose. diff --git a/design-notes/fused-source-status-2026-08-22.md b/design-notes/fused-source-status-2026-08-22.md new file mode 100644 index 000000000..4ea9dfdea --- /dev/null +++ b/design-notes/fused-source-status-2026-08-22.md @@ -0,0 +1,133 @@ +# The fused source status line — why it wasn't there, and what it cost + +2026-08-22 + +The owner asked on 2026-08-19 why the console has no product-standard fused +status string — the "Last updated 3 hours ago / Syncing now" line every mature +sync product shows — and asked for the underlying design problem explained +rather than patched over. This is that explanation, and the design that shipped. + +## What the owner saw + +On `/sources`, each row showed one colored glyph (`●◐⊘○◌⏸`) and nothing else +about state. The status *text* existed but was `sr-only` — announced to screen +readers, invisible to eyes. Freshness never reached the row at all. Whether a +source was syncing right now was used only to disable buttons and set a polling +interval; it was never displayed. + +So the row could not answer any of the three questions an owner actually has: +what is this source's state, when did it last update, and is it working right +now? + +## Why this is not a one-line fix + +The tempting fix is to concatenate three fields. That fails because **these are +three independent axes that routinely disagree**, and the disagreements are +exactly the cases that matter: + +| Situation | Freshness | Activity | Verdict | What the owner needs to hear | +|---|---|---|---|---| +| Sync running against a broken connector | stale | syncing | blocked | **Blocked** — the run will not save it | +| Healthy source mid-refresh | fresh | syncing | healthy | Working, syncing now | +| Stale because every run fails | stale | idle | blocked | Blocked, and here is how long | +| Paused with a stale in-flight run flag | stale | "syncing" | n/a | **Paused** — it is not syncing | +| Never connected | none | idle | unknown | Never updated | + +A source can be fresh and failing. It can be stale and syncing. It can look busy +and be dead. Any fused string has to *resolve* those conflicts, and the +resolution rule is the entire design — not the concatenation. + +## What the code actually did: last-writer-wins + +`deriveRenderedSourceStatus` in `apps/console/src/app/(console)/lib/source-actionability.ts` +resolves the conflict by returning early: + +```ts +if (running) { + return { dot: "◌", freshnessNote: null, kind: "pending", label: "Syncing", tone: "muted" }; +} +``` + +An in-flight run **erased both the freshness note and the health verdict**. A +source that was blocked, stale, and had a doomed retry in flight rendered as +`"Syncing"` in a muted tone — the calmest thing on the page. + +That is the fabricated-green defect class this program exists to kill, in +miniature. Nobody wrote "pretend it is fine"; the most reassuring axis just +happened to be evaluated last and win the single available slot. **Fabricated +green is usually an architecture accident, not a lie somebody typed.** One slot +plus three axes forces a silent choice about which truth to discard, and the +convenient one wins by default. + +## The rule that shipped + +> **Activity is additive, never substitutive.** + +"Syncing" is something a source is *doing*, not something it *is*. So: + +- The **state slot** always holds the worst honest verdict. +- **Freshness** gets its own slot and is never erased by activity. +- **Syncing** is appended as a separate clause and owns no color of its own. + +The line therefore reads `Blocked · Last refreshed 6 days ago · Syncing now` — +which is uncomfortable, and correct. The old rendering of that same source was +the single word `Syncing`. + +Implementation: `apps/console/src/app/(console)/lib/fused-source-status.ts`. +Axes are ranked worst-to-best (`SEVERITY_BY_KIND`) and the worst wins the slot +by comparison, not by branch order — so adding a state later cannot silently +reintroduce last-writer-wins. + +### Recovering the discarded verdict + +Because `deriveRenderedSourceStatus` throws the verdict away on `running`, the +fused line needs it back. `deriveSourceVerdictStatus` re-derives the +verdict-only status, and `projectSourceActionability` passes it as +`verdictFallback` — but **only for the `running` collapse**. `revoked`, +`paused`, and `pending` are lifecycle facts that outrank any verdict: a revoked +source is revoked no matter how its last verdict read. Passing the fallback for +those would let a stale "Blocked" overwrite "Revoked". + +### Honest absence + +A missing freshness note is not "fine". When a source has never had a +successful run, the line says `Never updated` rather than omitting the slot — +omission reads as "not applicable", which is a stronger claim than the evidence +supports. When a source *has* succeeded but carries no annotation, the slot is +genuinely absent rather than guessed. + +The server already folds activity into its own freshness annotation +(`rendered-verdict.ts` emits `"Refreshing now."` when `badges.syncing`). That +phrasing is dropped here and re-added from the real activity flag, so the +activity slot has exactly one owner and cannot double up. + +## What is guarded + +`fused-source-status.test.ts` pins the disagreement cases, and the rule is +mutation-proven three ways: restoring last-writer-wins, inverting the severity +comparison, and letting paused/revoked show as syncing each fail tests that name +the specific dishonesty. + +CSS state rules change **color only** — the geometry-bearing rules are +state-independent — so the list never reflows as sources change health. This is +enforced by the pre-existing "status state does not create separate row +geometry" invariant. + +## Coordination note + +This work deliberately does **not** restructure +`apps/console/src/app/(console)/lib/connection-evidence.ts`, which the +`labels-naming-0822` lane is changing. The fused line *composes* the label +strings that module and the server verdict produce; it defines no owner-facing +label vocabulary of its own beyond the two connective strings `"Syncing now"` +and `"Never updated"`. If that lane renames a pill label, the fused line picks +it up with no change here. + +## What is not addressed + +- Freshness granularity is still the server's day-level prose + (`"3 days ago"`), not minute-level. Mixing the client's `formatRelative` + (minutes/hours) with the server's day buckets would read inconsistently, so + one source of truth was kept. A finer server annotation would improve this. +- The detail page still renders the axes separately. Only the `/sources` list + row is fused so far. diff --git a/design-notes/graceful-drain-verdict-2026-08-22.md b/design-notes/graceful-drain-verdict-2026-08-22.md new file mode 100644 index 000000000..8cd08e924 --- /dev/null +++ b/design-notes/graceful-drain-verdict-2026-08-22.md @@ -0,0 +1,86 @@ +# Graceful drain on SIGTERM: do not re-attempt + +**Verdict: rejected, with production evidence. `fix/graceful-drain-0821` should +not be landed.** + +This note exists because the idea is genuinely appealing and has already been +built twice. Without a durable record it will be rediscovered — the reasoning +below is what stops the third attempt. + +## What was tried + +`fix/graceful-drain-0821` (uncommitted work in +`~/.tmp/graceful-drain-0821`) makes `drainActiveRuns` CANCEL every in-flight +run on SIGTERM and then await its terminalization within a 5s budget, so the +dying process writes each run's own terminal event instead of leaving it to be +adjudicated at the next boot. It also closes run admission during shutdown and +adds a `shutdown_drained` terminal reason. + +The engineering is careful and the diagnosis of the read-side problem is right +(a restart-ended run should not be reported as `run.failed`). The mechanism is +what does not work. + +## Why it does not work + +**The budget cannot cover the work.** Production sets no `--stop-timeout`, so +Docker's 10s default governs, and `--stop-timeout` is fixed at container +creation — Docker has no equivalent of systemd's runtime `EXTEND_TIMEOUT_USEC=`. +Real runs take minutes: production medians span 9s (github) to 1221s (ynab). The +gap is two orders of magnitude and is not closable by tuning. + +**It was measured failing exactly that way.** A production shutdown logged: + +``` +drained:0, elapsedMs:5000, timedOut:1 +``` + +The drain spent its entire budget and abandoned the run anyway, having consumed +half the window before SIGKILL while making the failure look handled. That is a +negative win — worse than not trying, because it delays the storage close that +still has to happen. + +**Correctness never depended on it, and cannot.** A `kill -9`, an OOM kill, or a +power loss gets no drain at all. Any design that needs the dying process to +write its own terminal state has an unhandled case by construction. The +successor must adjudicate regardless, so the drain is a redundant second +mechanism guarding a case the first already covers. + +This is the standard layering, not a local shortcut: Temporal's +`WorkerStopTimeout` defaults to 0s and the service writes the terminal state on +a timer; Kafka recovers a dead producer's transaction through the successor's +`InitProducerId` epoch bump. + +## What replaced it + +`2ddcca1b8` ("perf(shutdown): drop the connector drain from the SIGTERM path") +removed the shutdown call site and left `drainActiveRuns` on the controller, +where it means "await in-flight runs" for the watchdog, `awaitRun`, and the test +suite. Correctness lives in `reconcileOrphanedRunsAtBoot` +(`lib/controller-boot.ts`), which writes `run.abandoned` for any run whose owner +epoch is not the current one. + +That commit is an ancestor of the current base. **The drain branch predates it +and does not contain it**, so landing the branch would re-introduce a mechanism +this repo already removed on evidence. + +## The valid insight, and where it went instead + +The drain branch was right that a restart-ended run must not be reported as a +connector failure. That belongs on the read side, where it works for every +restart shape including `kill -9`: + +- `8609f37a8` — restart-abandoned runs no longer classify connection health; + they defer to the last run that actually observed something. +- `3cf21fd04` — restart-ended runs no longer reset the schedule anchor, so a + restart stops delaying the next run by a full interval. + +Both apply regardless of how the process died, which is precisely the property +the drain could not offer. + +## If someone still wants a drain + +The only version worth discussing would need all of: a stop timeout raised well +beyond 10s at container creation, evidence that the raised timeout does not get +the process SIGKILLed mid-storage-close, and a reason why successor +adjudication is insufficient despite handling the `kill -9` case the drain +cannot. Absent all three, this is settled. diff --git a/design-notes/local-collector-coverage-architecture-2026-08-22.md b/design-notes/local-collector-coverage-architecture-2026-08-22.md new file mode 100644 index 000000000..b0003fcb4 --- /dev/null +++ b/design-notes/local-collector-coverage-architecture-2026-08-22.md @@ -0,0 +1,171 @@ +# Local-collector coverage: two parallel systems, not one dropped field + +Investigation of PR #166 owner-feedback item C1: four local-collector sources +(peregrine Claude Code, peregrine Codex, Simon VM Claude Code, vivid fish Claude +Code) emit coverage facts with blank `covered`/`considered` while server-run +connectors emit real numbers. + +**Read this before "fixing" the blank denominators.** The obvious fix is wrong +and would fabricate a false coverage claim. The disproof is in §3. + +## Verdict + +The owner's reasoning — "local collector runs a connector the same way the +server does, so denominators should depend on the CONNECTOR, not the runner" — +is sound in principle but does not describe how this is built. There is no +single runner with a dropped field. There are **two parallel coverage systems** +with different units, vocabularies, and transports. + +This is a **per-connector gap plus a transport that cannot carry counts**, not a +runtime silently discarding numbers the connector emitted. `claude_code` and +`codex` emit no `DETAIL_COVERAGE` message at all — zero occurrences in either +`connectors/*/index.ts`. + +| | Server path | Local path | +|---|---|---| +| Signal | `DETAIL_COVERAGE` message | `coverage_diagnostics` records | +| Unit | per-record counts | per-**store** status enum | +| Vocabulary | `covered` / `considered` | `collected`, `inventory_only`, `missing`, `deferred`, `excluded`, `unsupported`, `unaccounted` | +| Verdict source | `covered` vs `considered` | `localCoverageConditionForStatus` | + +## 1. Three independent barriers + +Any one of these alone is sufficient to produce blank denominators. All three +are present, so a partial fix changes nothing. + +**Barrier 1 — the producer has no numeric field.** +`packages/polyfill-connectors/src/local-source-inventory.ts:168-181`. +`CoverageRecord` carries `status`, `store`, `stream`, `reason` — no count. +`coverageStatus()` (:541) answers "does this store exist and what is its +policy", one status per store. Counts that *are* measured get stringified into +prose at :245: + +```ts +return `enumeration complete, ${input.examined} examined (${input.emitted} emitted)`; +``` + +**Barrier 2 — the wire contract forbids counts.** +`packages/reference-contract/src/reference/index.ts:1839-1848`: + +```ts +const DeviceTerminalRunFactSchema = { + additionalProperties: false, + properties: { + coverage_statuses: { items: { minLength: 1, type: "string" }, minItems: 1, type: "array" }, + scoped: { type: "boolean" }, + stream: { minLength: 1, type: "string" }, + }, + required: ["coverage_statuses", "stream"], +``` + +`additionalProperties: false` — numeric coverage is actively rejected, not +merely absent. `canonicalTerminalRunCommitEnvelope` +(`packages/reference-contract/src/common/terminal-run-commit.ts:32-45`) +reconstructs each fact from exactly these three fields, so anything else is +dropped before signing. + +**Barrier 3 — the server hard-codes null.** +`reference-implementation/operations/local-device-terminal-collection.ts:249-258`: + +```ts +const fact = readRuntimeCollectionFact({ + checkpoint: "committed", + collected: 0, + considered: null, + coverage_statuses: coverageStatuses, + covered: null, + ... +``` + +This exactly reproduces the observed rows: `covered: null`, `considered: null`, +`collected: 0`, `checkpoint: "committed"`. + +## 2. The local path is NOT verdict-less + +Worth stating because "coverage unknown" was over-scoped in the ledger. The +server has a working local coverage authority: +`reference-implementation/server/ref-control.ts:3829-3848` maps +`collected -> "complete"`, `inventory_only -> "inventory_only"`, +`unaccounted -> "gaps"`. `coverageTone` (`runtime/rendered-verdict.ts:560`) +renders `complete`, `deferred`, and `inventory_only` all **green**. + +`connector-coverage-policy.ts:246-249` treats `observed_collected` (derived from +`coverage_statuses`) as legitimate non-numeric coverage proof, and +`verdict.proven` can return `"complete"` with no denominator at all. + +So blank denominators do not automatically mean a red or unknown stream. Check +the rendered verdict per stream before scoping work. + +## 3. Why `examined` is NOT the denominator (the trap) + +The prose already contains real numbers, so piping them into `considered` looks +like a one-line fix. It is a fabricated denominator. Production disproof +(peregrine Codex, `cin_ece4bfe5096b8bf67a1468c2`): + +``` +coverage_diagnostics reason: "enumeration complete, 566589 examined (0 emitted)" +actual stored records: 758,345 +``` + +**Examined (566,589) is LOWER than stored (758,345).** `scanLocalJsonl` +(`packages/polyfill-connectors/src/local-jsonl-cursor.ts:139-172`) is +incremental: on a `fast_skip` or `append` decision `onLine` fires only for new +bytes. `examined` is therefore a **per-run delta**, not a corpus size. + +Rendering that as `considered` would show "566589 of 566589 — complete" for a +run that skipped most of the corpus. That is precisely the fabricated-watermark +defect class this program exists to eliminate. + +It is also not uniform. `sessions` reports `"declared rollout source"` — **no +count exists at all**. Any fix must be per-stream. + +## 4. What an honest denominator would be + +For these inventory-style local collectors: **corpus discovered vs corpus +parsed**, measured at the enumeration site independently of the incremental +cursor — e.g. sessions/transcript files discovered on disk vs successfully +parsed. That satisfies the standard `DetailCoverageParams` already documents +(`connector-runtime.ts:494-509`): measured at the enumeration site, never +aliased to the collected/emitted count. + +Today's counters cannot supply it. `messagesExamined` +(`connectors/claude_code/index.ts:1447-1470`) is honestly measured per-line and +classified independently of emission — it is a *good* number, just a +per-run-delta one. Getting a real denominator means teaching the connectors to +measure corpus size on a full pass, separate from the cursor. + +Streams that genuinely cannot have one (a store whose whole content is +`inventory_only` by policy) should keep saying so precisely via +`coverage_statuses` rather than reporting a blank number. + +## 5. Cross-repo scope + +The end-to-end fix spans three places, one of which is not this repo: + +- **data-connect** (`PDP-Connect/data-connect @ 9155e57`) — the collector + runtime and `@pdpp/connector-protocol` that build `terminal_facts`. Vendored + here as `vendor/pdpp-collector-runtime-0.0.1.tgz` and + `vendor/pdpp-connector-protocol-0.0.1.tgz`; not editable in this repo. +- **this repo, reference-contract** — the wire schema AND + `canonicalTerminalRunCommitEnvelope`, which is the **commit-id / replay hash + authority**. Widening the fact shape changes envelope hashes, so it is a + compatibility break requiring a versioned migration, not an additive field. +- **this repo, reference-implementation** — `normalizeTerminalFacts` and the + coverage policy. + +Recorded as a known architectural gap. Not attempted here: a hash-contract break +across repos needs deliberate scoping. + +## 6. Related finding: freshness is a separate defect + +Ledger C3 suspects local sources write no run history. **They do** — 537 +succeeded runs for peregrine Claude Code, 380 for peregrine Codex, all recent. +C3 is right only for Signal. + +Local-device runs are deliberately discarded for health +(`ref-control.ts:5215-5221`, `localDeviceBacked ? null : ...`), so freshness +rests entirely on the heartbeat gate. A single dead-lettered row out of 10,001 +forced `blocked` -> `stalled` -> no freshness proof -> RED "can't collect" on a +2.5-million-record source whose collector was healthy. Fixed separately in +`localDeviceFreshnessHeartbeatAt`; see +`test/local-device-terminal-backlog-freshness.test.ts`. diff --git a/design-notes/manual-import-coverage-receipt-2026-08-19.md b/design-notes/manual-import-coverage-receipt-2026-08-19.md new file mode 100644 index 000000000..feeea8d1f --- /dev/null +++ b/design-notes/manual-import-coverage-receipt-2026-08-19.md @@ -0,0 +1,164 @@ +# A finished import has no way to prove what it ingested + +**Status:** superseded 2026-08-20. The premise below was wrong on the facts, and +the proposed build is unnecessary. See "What was actually true" at the end. +Written 2026-08-19 after a related fix was deliberately scoped to exclude this +case. + +## The state that has no honest verdict + +Two sources on this instance hold real data and can never be green: + +- Google Maps Timeline Import — 299,248 records +- WhatsApp — 120,042 records + +Both are `source_kind='manual'`, paused, with zero runs ever and zero rows in +`connector_detail_gaps`, `run_history`, or `acquisition_batches`. They are +finished one-time file imports. Nothing will run again. + +`isHealthyConditionSet` requires `SourceCoverageComplete` to be true. Coverage +evidence is produced by a collection run. No run, no evidence, no green — on a +source holding 419k records the owner can read today. The page says "Not +measured", which he reads as broken. + +## Why this is not the same as the case next to it + +An adjacent problem looks identical from the pill and is not. Gmail has 32 +terminal gaps, each carrying `observed_size_bytes > configured_limit_bytes` — +a specific attachment, a recorded size, a recorded cap. Gmail measured its +shortfall and can enumerate it. That case is being handled by +`unfillableAccounted`, a flag whose contract requires **every** outstanding gap +to carry durable per-item evidence. + +These two imports fail that contract on the facts. There is no `terminal_gap` +axis to account for, because there are no gaps. Their coverage axis is +`unknown` — never measured — not `terminal_gap` — measured and partly +impossible. + +**"Measured, and this part is provably impossible" and "never measured, and +never will be" are different states and must not share a signal.** Setting the +Gmail flag for an import would assert that every outstanding gap is proven +unfillable over an empty set, inferred from the total absence of evidence. +That is the exact false-green the anti-green tests exist to catch. + +An earlier framing in this work treated both as one design. That was wrong, and +the evidence above is what corrected it. + +## The line that already exists, and holds + +`Fresh` can be satisfied by `not_applicable` (`conditionIsSettledSatisfied`). +`SourceCoverageComplete` cannot — it is gated by `conditionIsTrue` +(`connection-health.ts:1830`). That asymmetry is deliberate, and +`source-state-truth-2026-08-18.md` states why: a completed import buys +exemption from a *freshness* proof, never from proving it ingested what it +claimed. Relaxing coverage would let any source with no evidence read as +complete. + +So the fix is not to exempt coverage. It is to let an import **prove** its +coverage. + +## The shape a proof would take + +`connector-coverage-policy.ts` already declares a `snapshot_import_receipt` +coverage strategy alongside `checkpoint_window` and `full_inventory`. No +connector emits evidence for it. The placeholder is the design, unbuilt. + +What it needs, roughly: + +1. The manual-upload route writes an `acquisition_batches` receipt at import + time recording what the file claimed to contain and what was ingested. The + table exists and is empty for both connections. +2. The stream's manifest declares `coverage_strategy: snapshot_import_receipt`. +3. The coverage projection grows a branch that reads that receipt as proof, the + way it reads a run's coverage report today. + +Then a finished import satisfies coverage the same way a collecting source +does — by evidence, not by exemption — and the terminal label already built for +it (`Fresh: not_applicable`, "Import complete") becomes reachable. + +## Cost and scope + +This touches the manifest schema, the upload route, and the coverage read path. +It is a separate change from the Gmail work deliberately: bundling them would be +two problems in one story, and the Gmail fix is narrow and provable on its own. + +Worth stating plainly: until this exists, these two sources stay honestly red. +That is the correct outcome. The alternative — green by exemption — would mean +the page can no longer distinguish a source that proved its completeness from +one that never tried. + +## Open questions + +- Does the receipt record the file's own claim (a manifest inside the export, + a row count) or only what PDPP ingested? A receipt that only records what was + ingested proves nothing about what was missed. +- Are existing imports retrofittable, or is this only correct for imports made + after it ships? Both sources here predate any receipt, so they may need a + one-time backfill with explicitly weaker provenance — and that weaker + provenance should be visible, not silently equal to a real receipt. +- Does a partial import (an interrupted upload) produce a receipt that honestly + reports incompleteness, or none at all? + +--- + +## What was actually true (2026-08-20) + +Three of the four load-bearing claims above are false. Steps 1-3 should not be +built. + +**"No connector emits evidence for `snapshot_import_receipt`" — false.** Both +connectors already emit an artifact-grounded coverage declaration, and both +manifests already declare the strategy (`google_maps.json:166,246`, +`whatsapp.json:140,208`). `google_maps` counts every point/segment element the +parser produced, at the parse site, and reports `covered < considered` when an +element had no usable id or timestamp (`finishPoints`/`finishSegments`, +hardened by `e1b92b36c` the same week this note was written). `whatsapp` counts +the archive's own file and attachment listing and subtracts media the bounded- +read policy dropped. Neither denominator is recomputed from the survivors. + +**"Nothing will run again" — false, and this is the load-bearing one.** The +uploaded artifact is durable and is re-read on **every** run, not just at setup: +`ManualUploadDurableSourceBinding` carries `import_dir`/`import_dir_env_var` +specifically so they survive promotion, and the run orchestrator injects the +directory as the connector-declared env var. These connections are re-runnable +against the file already on disk. + +**"Coverage evidence is produced by a collection run" — true, and sufficient.** +`connector_summary_evidence` is keyed by `connector_instance_id`, but is only +ever populated by folding a terminal run event's `collection_facts`. Zero runs +therefore means `stream_latest_facts_json IS NULL` and axis `unknown` — which is +the correct and honest reading of "never measured", not a defect. + +Derivation over the exact shapes these two connectors emit, through the real +`deriveStreamCoverageCondition`: + +| fact | axis | +| --- | --- | +| `considered=299248, covered=299248` (fully reconciled) | `complete` | +| `considered=299248, covered=299243` (5 elements unaccounted) | `partial` | +| whatsapp attachments, media dropped by read policy | `partial` | +| no run ever — the live state today | `unknown` | + +So the proof path is built and works end-to-end, including the honest partial. +**The remedy is operational — re-run each connection once against its stored +artifact — not a receipt, a manifest change, or a read-side branch.** A backfill +would be strictly worse: it would synthesize a denominator from records already +ingested, which is the fabricated-denominator anti-pattern (`covered == +considered` recomputed from survivors) that the coherence contract exists to +reject, and it could never produce the `partial` verdict a real re-run can. + +One genuine defect did come out of this review, fixed separately: `CredentialsValid` +had no branch for a connector that authenticates to nothing, so both sources sat +at `credentials_not_probed`/`unknown` forever. That is now `not_applicable`, +derived from the manifest declaration. + +What remains unprovable either way, and is worth not overclaiming: a reconciled +artifact proves the run ingested everything the *file* contained. It says +nothing about whether the file is a complete export of the owner's history — +the provider decides what goes into it, and no check on this side can see past +that. + +## Related + +`source-state-truth-2026-08-18.md` — the `not_applicable` design and the +deliberate decision not to extend it to coverage. diff --git a/design-notes/semantically-bounded-consent-2026-08-07.md b/design-notes/semantically-bounded-consent-2026-08-07.md new file mode 100644 index 000000000..41d63a444 --- /dev/null +++ b/design-notes/semantically-bounded-consent-2026-08-07.md @@ -0,0 +1,78 @@ +# Semantically Bounded Consent (derived streams vs dynamic selectors) + +Status: captured +Owner: Tim +Created: 2026-08-07 +Related: spec-core Grant semantics; derived subset streams aside (non-normative); +spec-deferred predicate scoping; openspec change harden-pdpp-authorization-and-0-1-migration +(critical-extension and seam-spike gates); inbox/8-7-26-chatgpt-convo.txt + +## Question + +A user wants consent bounded by a subjective rule ("my accountant may read financial +documents, excluding items my agent flags as private"). Can PDPP express this without +changing the grant model or sync semantics, and what minimal seams should exist so a +future extension can carry it? + +## Context + +Two designs were compared, independently by two analyses (this repo, 2026-08-05; an +external ChatGPT session with its own red-team, 2026-08-07), converging on the same +answer. + +Dynamic selectors: grants carry typed, monotonically narrowing constraints evaluated +per request, possibly by a model. Rejected for Core: it converts the immutable grant +from the complete authorization into a maximum bound, leaks excluded records through +side surfaces (counts, aggregations, search, expansion), risks per-grant membership +state at platform scale, and produces interoperability in name only when evaluator +contracts differ. + +Derived streams: an evaluator materializes a subset stream upstream; the recipient +receives an ordinary deterministic grant to that stream. Core is untouched, side +surfaces are contained because excluded records are absent from the granted stream, +and existing mutable-stream sync carries membership changes. + +Honest limit of the derived-stream design: the grant fully describes authorization +only syntactically. Stream membership changes at the evaluator's discretion, so the +indeterminism moves behind the stream name rather than disappearing. The real +arguments are Core stability, side-channel containment, and reuse of existing sync. + +Evaluator placement is a deployment property, and the spec stays deployment-agnostic. +Where the evaluator is co-located with the data (a personal server, or the provider +itself), no second disclosure occurs. A remote evaluator is a second grantee and needs +its own grant. An extension should state this trust consequence explicitly. + +## Stakes + +Low until an implementer wants it. The protocol-design payoff is flexibility: the same +seams cover role changes, household membership, classification, and jurisdiction, well +beyond AI evaluators. + +## Current Leaning + +1. Prototype subjective consent as a materialized derived stream. No Core change. +2. One near-term semantic clarification worth owner review before or after the v0.1.0 + freeze: on subset streams, a tombstone signals membership removal and does not + assert source deletion. This ambiguity exists today without any evaluator, and the + two claims carry different recipient obligations. One sentence, optionally a reason + field later. +3. Reserve a namespaced critical-extension mechanism (an enforceable constraint an RS + must reject when unrecognized, distinct from ignorable capabilities). PR #77's + accepted proposal already moves in this direction; keep the reservation, publish no + selector grammar. +4. Revisit a Dynamic Disclosure Profile only after derived streams fail against + several real use cases, and require: hard Core boundary, evaluator identity, + decisions tied to record versions, fail-closed behavior, and authoritative + resynchronization rules. + +## Promotion Trigger + +A second implementer asks for subjective or externally evaluated consent, or derived +streams demonstrably fail a real deployment (per-recipient stream explosion, consent +legibility complaints, or re-consent churn on stream redefinition). + +## Decision Log + +- 2026-08-07 — Captured from convergent internal (2026-08-05) and external analyses. + Owner direction: keep the protocol flexible and cohesive; no build planned; the + tombstone clarification is the only near-term action candidate. diff --git a/design-notes/source-state-truth-2026-08-18.md b/design-notes/source-state-truth-2026-08-18.md new file mode 100644 index 000000000..9165ba3d2 --- /dev/null +++ b/design-notes/source-state-truth-2026-08-18.md @@ -0,0 +1,328 @@ +# What a source's state should tell its owner + +**Status:** intake. Terminal model proposed, not owner-ratified. One case +implemented as proof; the rest is unbuilt. +**Date:** 2026-08-18 + +## The evidence + +The owner's goal is every source green, honestly — green only if genuinely +collecting, never by loosening a condition. Today `/sources` shows 23 sources in +five display states, and four of the five lie in a way the owner can catch: + +| shown | reality | +|---|---| +| `○ Not measured · Fresh today` | claude-code holds 2,408,082 records and is collecting right now | +| `○ Not measured · Freshness has not been measured yet` | Google Maps Timeline Import holds 299,248 records; the import finished and will never run again | +| `◐ Needs refresh · Review: Resume schedule` | Chase is fine; an operator disabled its schedule to stop an OTP loop | +| `⊘ Can't collect` + `Last successful refresh today` + 2,129 records | USAA, all three simultaneously true | + +Verified in production today: + +``` +claude-code local_device 2408082 stale (3 instances: 2.4M, 38k, 20k) +codex local_device 1299535 fresh +google-maps manual 299248 fresh +whatsapp manual 120042 fresh +usaa account 2129 fresh +``` + +Both `manual` sources have **zero rows in `run_history` and zero schedules**. +There is nothing to run, and nothing that will ever run. + +The four failures have one shape. `isHealthyConditionSet` +(`reference-implementation/runtime/connection-health.ts:1755`) collapses ten +conditions to one boolean, and it requires three of them to be affirmatively +`true`: + +``` +CollectionSucceeded === true +SourceCoverageComplete === true +Fresh === true +``` + +Three of those ten are also required *not* to be `false`, and `BacklogClear` +must not be `error`. The predicate has exactly one caller, `classifyHealthy`, +the last of fourteen ordered classification steps. + +The collapse is not the bug by itself. The bug is that the predicate cannot +distinguish **"we don't know"** from **"the question doesn't apply here"**, so +it treats both as not-green. A finished import can never produce a freshness +proof, so it can never be green, no matter what the owner does. + +## What the code already knows + +This codebase already diagnosed this problem and solved it one layer too high. +`ConnectionConditionStatus` (`connection-health.ts:107`) has four values, and +the doc comment on the fourth is worth quoting: + +> `not_applicable` : the condition cannot apply to this connection at all, +> because the evidence source it reads does not exist here. This is a *settled* +> answer, not a pending one. +> +> `not_applicable` exists so the projection stops encoding certainty as doubt. + +And then, three lines later: + +> Classification treats `not_applicable` exactly as it treated the `unknown` it +> replaces: it is never `true` and never `false`, so no headline state, axis, or +> healthy-set predicate changes. **Only presentation changes.** + +That last sentence is the decision to revisit. The concept is right and already +shipped; it was deliberately confined to cosmetics. Making it load-bearing in +the healthy predicate is a smaller change than inventing anything new. + +Two more pieces already exist: + +- **`source_kind`** is a real column with a CHECK constraint over `account`, + `local_device`, `browser_collector`, `manual` + (`server/postgres-storage.ts:885`). The two "never measured" sources are + exactly the two rows with `source_kind = 'manual'`. The manifest already + carries enough to decide this — nothing new needs to be declared. +- **`COVERAGE_UNKNOWN_STALE_COLLECTOR`** (`connection-health.ts:2716`) already + says *"This local collector build predates coverage evidence the server now + requires. Update the collector."* That is the honest sentence for the 2.4M-record + case. It exists, it is correct, and the sources list does not show it. + +## The dimensions, derived from the incidents + +Not a taxonomy invented for symmetry — each of these is a distinct axis because +a real source varies on it independently of the others. + +1. **Is data arriving?** claude-code: yes, 2.4M records. Independent of whether + we can prove anything about it. +2. **Is coverage provable?** Separate from (1). The stale collector emits data + but not the stores that prove coverage. Data flowing and proof complete are + genuinely orthogonal — that pair is the whole "Not measured · Fresh today" + contradiction. +3. **Is currency meaningful, and if so, is it current?** Two questions, and the + model only asks the second. For a finished import the first answer is *no*, + which makes the second a category error. +4. **Who can resolve the blocker?** Connector maintainer, owner, operator, or + external provider. USAA's detail page says "Connector code needs a fix"; the + list says "Can't collect". The useful sentence is the one not shown. +5. **Is this source finished by design?** No state expresses it. There is no + terminal state at all. + +## Is a single green/not-green verdict the right shape? + +**Yes — keep the boolean, and fix which conditions are required versus +inapplicable per source.** I considered the alternatives seriously. + +**Two-axis (data-flowing × proof-complete)** describes the claude-code case +exactly, and it is the model I most wanted to adopt. I rejected it because it +does not generalize: it has nothing to say about the operator-paused case or +the provider-down case, so those would need a third and fourth axis, and the +owner would be reading a vector. The owner's stated goal is *every source +green*. A goal phrased as a scalar needs a scalar answer. + +**A state machine with a terminal Archived/Complete state** is the wrong +primitive because completeness is not a state a source *transitions* into +through the health pipeline — it is a property of the source's kind, known at +creation. Google Maps Timeline Import was complete the moment its import +finished. Modeling it as a reachable state implies a transition that never +fires, and this codebase has already been bitten by exactly that: the +`terminal_facts_historical` exclusion in +`summary-evidence-projection-controller-2026-08-18.md` stranded three +production rows behind an exit condition that was unreachable by construction. + +**Keeping the boolean, fixing the required set** wins because the boolean was +never actually the problem. The problem is that "required" is currently a fixed +list of ten conditions applied uniformly to every source, when some conditions +are unanswerable for some source kinds. Green should mean *every condition that +applies to this source is satisfied* — which is what the owner already thinks it +means. + +**What it costs.** The predicate stops being a fixed list, so reading it no +longer tells you the whole rule; you must also know which conditions the source +kind marks inapplicable. That is real complexity and I am not going to pretend +otherwise. The mitigation is that inapplicability is derived from `source_kind` +and the manifest — both durable, both already there — rather than from +per-source configuration an operator can get wrong. The failure mode to guard +is a condition marked inapplicable when it is merely unproven, which would +manufacture exactly the false green the owner refuses to accept. Hence the rule +below. + +### The rule + +> A source is green when every condition that **applies** to it is satisfied. +> `not_applicable` is satisfaction. `unknown` is not. +> +> A condition may be marked `not_applicable` only from durable evidence that the +> question is meaningless for this source — never from the absence of an answer. + +The second sentence is the entire safety property. "We couldn't measure it" and +"there is nothing to measure" must never collapse, or this design becomes the +loosening the owner rejected. + +## The manual-import case + +**Settled: `Fresh` is `not_applicable`, not `false` and not `unknown`, for a +source whose acquisition is complete.** + +Not `true`. Claiming a finished 2023 WhatsApp export is "fresh" replaces one lie +with another. The honest statement is that freshness does not apply. + +Coverage is deliberately **not** relaxed. A completed import must still prove it +ingested what it claimed. Unknown or gapped coverage keeps it out of green — the +completeness declaration buys exemption from a freshness proof only. + +`source_kind = 'manual'` already carries this and is written at exactly one +place (`server/routes/ref-manual-upload-draft-connection.ts:687`). The health +input takes a new `acquisition: { complete: true }` evidence field rather than +reading `source_kind` directly, matching how every other signal reaches +`computeConnectionHealth` — the projection trusts caller-supplied evidence and +never reads storage itself. + +## The operator-paused case + +**Settled: this is already correct in the health model and wrong only in the +rendering. Do not touch the health model.** + +`classifyOwnerPaused` (`connection-health.ts:1356`) runs third of fourteen +steps, before every failure classifier, and routes a disabled schedule to +`idle` — not `degraded`, not `blocked`. `isDegradingCondition` explicitly +excludes `ScheduleEligible`. The model already says an operator pause is not a +source defect. + +The damage is done downstream: `idle` + disposition `owner_refresh_due` renders +the amber pill `"Needs refresh"` (`runtime/rendered-verdict.ts:432`), and the +console prefixes the CTA with a hardcoded `"Review: "` +(`apps/console/.../sources/sources-view.tsx:338`). Amber plus "Review" reads as a +defect for a source that has none. + +The fix belongs in the pill vocabulary — an operator-paused source is not amber +— and I am explicitly not making it here, because `rendered-verdict.ts` is the +same file the actor vocabulary below would rewrite, and both should land +together. + +## The actor vocabulary + +"Can't collect" names no actor, so it cannot be acted on. Every state must name +who resolves it. All five derive from evidence that already exists: + +| state | meaning | derived from | +|---|---|---| +| **Collecting** | green | the healthy predicate above | +| **Complete** | finished by design, final | `acquisition.complete` (from `source_kind = 'manual'`) | +| **Needs your login** | owner action | `CredentialsValid` false, `CREDENTIAL_REQUIRED` / `CREDENTIAL_REJECTED` | +| **Needs a collector upgrade** | owner action, distinct from the above | `COVERAGE_UNKNOWN_STALE_COLLECTOR` — exists today, unshown | +| **Needs a connector fix** | maintainer action, not the owner's | `terminalCoverageCta`, `audience: "maintainer"` | +| **Paused by operator** | operator action, not a defect | `SCHEDULE_PAUSED` | +| **Provider is down** | nobody's action; wait | `REMOTE_SURFACE_FAILED`, `EXTERNAL_TOOL_UNAVAILABLE` | + +Every row maps to a reason code already in `CONNECTION_CONDITION_REASONS`. This +is a presentation vocabulary over existing evidence, not new derivation — which +is why it is cheap, and why it is worth doing before anything more ambitious. + +Note "Needs a collector upgrade" is the sentence the owner most needs today: it +covers 2.4M + 1.3M + 38k + 20k records currently labelled "Not measured", and +the string already exists in the codebase. + +## Proof of concept + +The manual-import case, implemented end to end in +`reference-implementation/runtime/connection-health.ts`. It is the cleanest test +of the model because it is the case with no possible workaround — no owner +action can ever make a finished import fresh. + +New test: `reference-implementation/test/connection-health-completed-import.test.ts`. + +**Fail before** (against unmodified `connection-health.ts`) — this reproduces +the production symptom exactly: + +``` +✖ a completed one-time import reports Fresh as not_applicable, not unknown + + actual 'unknown' - expected 'not_applicable' +✖ a completed one-time import is healthy without a Fresh=true proof + 'idle' !== 'healthy' +``` + +**Pass after** — 6/6: + +``` +✔ a completed one-time import reports Fresh as not_applicable, not unknown +✔ a completed one-time import is healthy without a Fresh=true proof +✔ a completed import still needs complete coverage to be healthy +✔ a completed import with a terminal coverage gap is not healthy +✔ acquisition completeness does not leak into recurring sources +✔ a recurring source that is genuinely stale is never rescued by this path +``` + +The last three tests are the ones that matter. They prove the change cannot +manufacture a false green: coverage is still required, and a recurring source +without the completeness declaration behaves exactly as before. + +The change is 90 lines, of which the load-bearing edit is **one**: + +``` +- conditionIsTrue(conditions, "Fresh") && ++ conditionIsSettledSatisfied(conditions, "Fresh") && +``` + +where `conditionIsSettledSatisfied` accepts `true` or `not_applicable`, and +pointedly not `unknown`. The other 89 lines are the new +`ConnectionAcquisitionEvidence` type, one branch in `freshCondition`, and one +branch in `collectionSucceededCondition` that mirrors the existing +`localDeviceCollection.verdict` precedent for sources that legitimately write no +spine run. + +**Regression evidence:** 305 existing tests pass unchanged — +`connection-health.test.ts` 151/151, `connection-health-acceptance.test.ts` +70/70, `rendered-verdict.test.ts` 84/84 — and `tsc --noEmit` is clean. + +Not wired to the read path. `projectConnectorSummaryConnectionHealth` in +`server/ref-control.ts` would need to pass `acquisition` from the instance's +`source_kind`, and that file is being actively edited by another agent. The +runtime model is proven; the wiring is one line in a file I did not touch. + +## What I deliberately left alone + +- **The other nine conditions stay required.** Only `Fresh` gained a + not-applicable path, and only for one source kind. Extending this to coverage + is where a false green would come from, so it needs its own evidence and its + own argument. +- **`classifyOwnerPaused` and the classification order.** Already correct. The + paused-source damage is in the pill vocabulary, not the model. +- **`rendered-verdict.ts`.** The actor vocabulary rewrites it; the paused-pill + fix rewrites it; doing either piecemeal now means doing it twice. +- **`hasAffirmativePassiveRecoveryEvidence`** (`connection-health.ts:1751`) — + the scheduler's passive-recovery authority. It independently requires + `axes.freshness === "fresh"` and `Fresh === "true"`. I did not touch it: a + completed import has no schedule and no next attempt, so it can never reach + that path, and relaxing a scheduler predicate to fix a display problem would + be scope I cannot justify. It is, however, the second place the same + fixed-required-list assumption lives, and it will need the same treatment if + this model is adopted. +- **The `dirty`/projection layer.** Orthogonal, and owned by + `summary-evidence-projection-controller-2026-08-18.md`. +- **Production data.** Nothing deployed, nothing committed, no database written. + +## Cost and risk + +**What breaks if the predicate changes.** Less than feared — `isHealthyConditionSet` +is private with exactly one caller. The blast radius is `classifyHealthy`, and +from there whatever reads `state === "healthy"`. The real risk is not +mechanical; it is that every future `not_applicable` is a potential false green. +That is why the rule above forbids deriving inapplicability from a missing +answer, and why the proof-of-concept spends half its tests on that boundary. + +**The honest residual risk.** `not_applicable` is now load-bearing, so a bug +that marks a condition inapplicable is a bug that turns a source green. Before +this change such a bug was cosmetic. That is a genuine increase in the cost of +being wrong, accepted because the alternative is a permanently dishonest display +on 420k records that are complete and correct. + +**Confidence.** That the manual-import fix is right: high — it is proven by +test, and the case admits no other honest answer. That the same shape extends +cleanly to the stale-collector and paused cases: moderate — the evidence exists +and the vocabulary maps, but neither is implemented, and the paused case needs a +pill-vocabulary decision I did not make. + +## Related + +`upstream-disclosure-window-2026-08-17.md` names the same failure from the other +side — "Imports have no upstream. They need to be first-class *not applicable*, +not zero — the same failure this codebase already has with 'Not measured'." That +note wanted this primitive and could not assume it. This note builds it for +freshness; the boundary case will want it too. diff --git a/design-notes/summary-evidence-projection-controller-2026-08-18.md b/design-notes/summary-evidence-projection-controller-2026-08-18.md new file mode 100644 index 000000000..1305ebf24 --- /dev/null +++ b/design-notes/summary-evidence-projection-controller-2026-08-18.md @@ -0,0 +1,207 @@ +# One generation row per connection, not a generic job queue + +**Status:** intake. Terminal design proposed by an independent reviewer, not yet +owner-ratified. Deliberately NOT scoped into the corrective branch that prompted it. +**Date:** 2026-08-18 + +## Why this note exists + +Five starvation bugs were found in the bounded maintenance sweep inside about +twenty-four hours. All five have the same shape: **work that cannot progress +consumes a shared budget, and work that can progress never runs.** + +1. **Checkpoint floor.** The fold read from `min(checkpoint)` across participants. + Three rows sat at checkpoint 0, so the floor was 0 against a 1.44M-event log. + Every 2s pass restarted at 0, read zero qualifying events, and repeated. +2. **Zero-vs-null.** The first fix guarded `null`. The rows stored a literal `0`. +3. **Phase starvation.** "Missing" discovery consumed the whole budget before + "generic" — the only path that classifies a row as dirty — could run. +4. **Post-deadline skip.** Discovery expired the deadline, so the repair loop + skipped every candidate. 16 classified, 16 skipped, 0 repaired, on an idle + database. +5. **Permanent exclusion.** The fix for #2 excluded `terminal_facts_historical` + rows at checkpoint 0 from the fold. Its stated exit condition was unreachable: + nothing marks such a row dirty, and the checkpoint advances only via the fold + the row is excluded from. Three production rows stranded, one an active + connection that could not recover. + +Number 5 is the one that matters most for design purposes. It was introduced *by* +a starvation fix, written immediately after fixing the previous one, and it +converted a livelock into a permanent exclusion. That is not an attention failure. +It is what happens when fairness is an emergent consequence of phase order, cursor +position, and exception paths rather than an explicit durable invariant. + +## The reviewer's verdict + +> Replace the scheduling model; keep the shipped fix only as incident mitigation. + +Confidence that the current model produces more bugs of this family: **0.96**. +Confidence that a small durable reconcile queue is the right terminal shape: **0.90**. + +Crucially, the preferred design is **not** a generic durable job framework with +fold, missing-repair, generic-repair, and audit job types. That would preserve the +task-kind zoo that produced the bugs. It is: + +> A level-triggered, generation-based projection controller keyed by connection. + +## The design + +One durable projection-state row per connection: + +``` +connector_instance_id +desired_generation +applied_generation +target_event_seq +folded_event_seq +applied_contract_version +dirty_since +next_attempt_at +last_attempt_at +attempt_count +last_outcome +``` + +Every canonical change that could affect a connection's summary increments +`desired_generation`, **preferably in the same transaction as the change**. +Multiple changes coalesce into the same row — the row *is* the durable +deduplicating queue entry. There is no separate queue table to keep in sync. + +One bounded, idempotent operation reconciles a connection: + +``` +reconcile(connection_id): + snapshot desired_generation + fold at most N indexed events for this connection + persist fold progress and yield if more remain + read bounded canonical facts + compute and write the complete desired summary + set applied_generation to the generation that was reconciled +``` + +If the connection changes mid-reconciliation, `desired_generation` advances past +`applied_generation`, so it stays eligible automatically. A deferred or failing +connection gets `next_attempt_at`/backoff and cannot permanently hold first +position. + +## What this deletes + +These concepts stop existing, and with them the bugs they produced: + +- "missing" versus "generic" discovery phases +- the shared minimum participant checkpoint +- the rotating page cursor +- process-local phase alternation +- the first-candidate deadline exemption + +Missing, dirty, and code-version-stale collapse into two conditions: + +``` +applied_generation < desired_generation +applied_contract_version != CURRENT_VERSION +``` + +**Both of those conditions would have prevented a bug this codebase actually +shipped.** `applied_generation < desired_generation` is derived, not remembered, +so bug #5 is unrepresentable — a row cannot be stranded by a predicate that +forgot to let it back in. And `applied_contract_version != CURRENT_VERSION` is +exactly the check that was missing when production ran fold logic version 5 while +every committed branch was at 4: a clean build shipped a binary older than its own +data, the version guard failed closed, and 26 of 28 evidence rows went unreadable +with no signal beyond a fleet of grey pills. + +## Bounds are still required + +The controller shape does not remove the need for hard bounds: + +- bounded indexed event pages +- PostgreSQL `statement_timeout` and `lock_timeout` +- bounded SQLite query shapes, or interruption where the driver allows it +- a soft pass admission deadline +- a maximum number of units per wake + +The reviewer's P1-2 stands independently of the redesign: the current 2000ms +`maxDurationMs` is a cooperative admission hint, not a wall-clock or database +occupancy bound. Measured on production *after* removing an unrelated CPU +contention problem, a pass still reported `repair_duration_ms: 5322` with skipped +candidates. If a unit cannot be hard-cancelled, it does not belong inside a +claimed 2-second maintenance loop. + +## The three invariants + +The reviewer rejected the single-invariant framing ("a pass that finds candidates +must repair at least one") as insufficient — it conflates repair success with +scheduling and does not bound a pathological unit. Three independent, separately +testable invariants are required: + +**A. Bounded yield.** No operation may run between durable yield points unless its +worst-case work is bounded or it has enforceable cancellation. + +**B. Monotonic outcome.** Every attempted work item must durably advance a cursor, +complete, defer with a future eligibility time, back off, or terminate. An +identical no-op retry cannot repeat forever. + +**C. Bounded fairness.** Every continuously eligible item and nonempty task class +must receive an attempt within a defined number of scheduler turns, **across +process restarts**. + +Invariant C is the one the current implementation cannot satisfy: fairness lives +in module-local variables (`nextDirtyAfterId`, `nextFirstObservationPhase`) whose +convergence bound vanishes on restart. + +## The audit becomes a backstop + +The hot path should not recompute fleet-wide aggregates. Expensive facts such as +record counts should be maintained incrementally where practical and verified by a +slower paged audit. The periodic sweep stops being the primary repair engine and +becomes what it should have been: a detector of missed invalidations that marks +connections behind. Orphan detection belongs there too, not in the latency- +sensitive loop. + +## Scope discipline + +The reviewer was explicit that this redesign should **not** be added to the +corrective branch. That branch finishes a bounded list: + +- advance fairness from the last *attempted* candidate, not the last fetched + page member — **done** (`ab28764f2`) +- repair `terminal_facts_historical` re-entry and boundary stamping — **partially + done** (`078b72e3a` prevents new stranding; already-stranded rows still need a + one-time re-entry path, in progress) +- the four adversarial tests, plus below-page-limit, above-page-limit, and + restart cases — **partially done**, restart case outstanding +- hard per-query/per-unit bounds, and an honest name for the pass deadline +- no-progress telemetry and alerting +- durable fairness, or fairness derived from durable per-item attempt state — + **deliberately deferred to this note's design**, since it needs a schema change + and a different discovery query shape + +Durable fairness is the item that most clearly belongs here rather than there: +implementing it in the current model means adding per-item attempt columns and +reshaping the discovery query, which is most of the projection-state row anyway. +Doing it twice would be waste. + +## Open questions + +- Does `desired_generation` increment in the same transaction as every canonical + change, or is a trigger acceptable? Same-transaction is stated as preferred; + the cost is touching every writer. +- What is the migration path for the existing `connector_summary_evidence` rows, + including the three currently stranded at checkpoint 0? +- Does the audit backstop need its own fairness guarantee, or is a slow full + rotation sufficient given it is no longer the primary repair path? +- SQLite parity: `statement_timeout` has no direct equivalent. Is a bounded query + shape provably sufficient, or is driver-level interruption required? + +## Provenance + +Independent design review of the bounded maintenance sweep, 2026-08-18, conducted +against `sweep-design-review-20260818.zip` (the four starvation bugs, the shipped +minimum-one fix, and the supporting patches). The reviewer retracted one +production measurement from that packet after it was shown to be confounded by an +uncapped embedding transformer competing with PostgreSQL — the code-level +counterexamples and the structural conclusion were unaffected. + +Related: `connector-sidecar-packaging-2026-08-17.md` and +`upstream-disclosure-window-2026-08-17.md` share the shape of a component whose +correctness depends on a peer's version with nothing verifying the pair. diff --git a/design-notes/upstream-disclosure-window-2026-08-17.md b/design-notes/upstream-disclosure-window-2026-08-17.md new file mode 100644 index 000000000..1a62e931b --- /dev/null +++ b/design-notes/upstream-disclosure-window-2026-08-17.md @@ -0,0 +1,129 @@ +# Surfacing a shrinking upstream disclosure window + +**Status:** intake. Not an OpenSpec change; no requirement is proposed yet. +**Date:** 2026-08-17 + +## The observation that prompted this + +PDPP holds two H-E-B orders for the owner, both captured in the same scan on 2026-07-15: + +| id | date | status | total | +|---|---|---|---| +| `HEB20169324473` | 2023-08-09 | Order canceled (`SHORTED`) | $293.98 | +| `HEB20607368035` | 2023-08-19 | Delivered | $382.67 | + +H-E-B now displays only the second one to that account. + +The owner's inference is the load-bearing one: **PDPP scrapes what the account UI shows, so if it +captured the first order, H-E-B was showing it then.** The record did not move. The provider's +disclosure did. + +That is the product working exactly as intended — PDPP holds data the provider no longer surfaces. +But the app cannot say so. Its dashboard shows "2 orders" today and would show "2 orders" after a +fresh run that finds nothing. Something important happened and the product is silent about it. + +For a tool whose purpose is outrunning deletion, "the source is disclosing less than it used to" +is not noise. It is the alarm. + +## What the system already has + +- Per-source checkpoint: `{"checkpoint": "2023-08-19", "fingerprints": {...}}` +- `fetched_at` on every record +- Scan-termination reasons in the H-E-B connector distinguishing `pagination_exhausted` from + `selector_drift`, `pagination_metadata_absent`, `source_auth_or_challenge` + (`packages/polyfill-connectors/connectors/heb/index.ts:320-335`) + +What is missing is durable evidence of *why a scan stopped* and *how far back it reached*. For +`cin_c875ca3ec8b6ce2c283a4288` no such evidence was stored, and there is no run history at all — +so today we cannot distinguish "H-E-B showed us everything" from "we hit a wall." + +## The proposed primitive: an observed boundary, not availability + +Per successful run, one value: **the oldest item the provider displayed**, recorded only when the +scan proves it reached the end of its range. Comparing across runs yields a moving frontier. + +The product could then say something entirely factual: + +> H-E-B showed orders back to 2023-08-09 in July 2026; today it goes back to 2023-08-19. +> 1 stored order is no longer displayed. + +Every clause is an observation. The conclusion — *their retention window is closing* — is the +owner's to draw, and they can draw it, because they know whether they shopped in 2024. + +## Hard constraints + +**1. A moving boundary is information about the provider, never an annotation on a record.** +This is the one absolute. Nothing in this design may mark a stored record deleted, stale, or +suspect. Deliberately: no connector in the fleet emits deletions today — verified, zero occurrences +of a delete/tombstone emission across `packages/polyfill-connectors/connectors/` — so provider +erasure structurally cannot propagate into the owner's copy. Introducing a path that annotates +records based on absence would give that up for a signal that is frequently wrong. + +**2. Absence is evidence only when the scan proves it covered the range.** +A run ending `pagination_exhausted` makes absence meaningful. A run ending `selector_drift` or +`source_auth_or_challenge` makes it meaningless. Without a stored termination reason there is no +signal, and the correct output is "unknown." + +**3. Do not encode provider retention policies.** +"H-E-B keeps 18 months" is undocumented, changes silently, and varies by account and region. A +wrong constant produces confident lies. The empirically observed window is strictly better: it is +measured, not asserted, and it survives the provider changing policy without telling anyone. + +## Edge cases that make a *general* solution hard + +- **Silent auth degradation.** A session expiring into a logged-out-but-200 view returns no items, + reports pagination exhausted, and yields a boundary of nothing — indistinguishable from a total + purge. The nastiest false positive, and the reason constraint 1 is absolute. +- **Not every source has an ordering.** Contacts have no time axis. Notion pages are edited, so + recency is not age. Gmail's "oldest visible" depends on the query. A single scalar frontier fits + perhaps half the fleet and produces meaningless numbers for the rest. +- **Scope change mimics retention.** Leaving a Slack channel removes its history from view; a plan + downgrade hides older data; a provider splitting history by store looks like shrinkage. No + boundary comparison can tell these from deletion. +- **Retention is rarely uniform.** Amazon keeps orders but drops invoice PDFs; Slack's free tier + hides messages but keeps files; Gmail retains mail but purges trash at 30 days. One per-source + boundary cannot express "text kept, attachments gone," and per-stream boundaries multiply the + connector burden. +- **Imports have no upstream.** Google Maps and the WhatsApp exports are one-shot files. They need + to be first-class *not applicable*, not zero — the same failure this codebase already has with + "Not measured" (see `add-honest-uncollected-source-states`). +- **Contiguity assumptions are false.** "Orders are sequential, so a gap means deletion" breaks on + a month with no shopping. The owner's own H-E-B data is two orders ten days apart and then + nothing — a gap that is a fact about their life, not their provider. + +## Recommended shape + +**Opt-in per connector, not a universal contract.** Connectors with a genuine monotonic frontier +and a provable pagination stop — orders, transactions, messages — report a boundary. Everything +else reports nothing, and nothing is fine. A signal present on eight connectors and honest beats +one present on twenty-four and wrong. + +**Runtime owns the bookkeeping.** The author declares the scanned range and the termination reason; +the runtime derives the boundary and its movement. A connector must never assert that something is +gone. + +**Fail closed.** No termination reason means no boundary claim. A lazy or broken connector produces +"unknown," never a false purge. + +**Machine reports, human interprets.** State the observation; leave the conclusion to the owner. +That is not a cop-out — it puts the inference where the context actually lives. The owner knew +instantly that PDPP could not have collected an invisible order; no rule authored here would have +encoded that. + +## Honest caveat on feasibility + +This is reasoned from one connector and a day of code reading. The fleet has not been surveyed for +how many connectors could actually satisfy constraint 2. Today's evidence argues for pessimism: +Slack emitted duplicate coverage for every multi-archive run since inception, and a Codex source +was rendered unmeasurable by a single stale store name. If the existing, simpler coverage contract +is unevenly met, a boundary contract will be too. Expect "unknown" from a meaningful fraction of +the fleet for a long while — and prefer that to false confidence. + +## Relationship to existing work + +Same failure shape as several open items: a derived value that goes quietly stale because nothing +watches whether its source changed. Compare `add-projection-contract-versioning` (input checkpoints +cannot see a formula change), `make-local-coverage-tolerate-unexpected-stores` (a stale store name +discards a valid proof), and `add-durable-connection-account-identity` (identity derived from a +provisional binding key). The recurring principle is worth stating once, somewhere durable: +**derive nothing durable from a value that may be provisional, and watch anything you do derive.** diff --git a/docker-compose.yml b/docker-compose.yml index ca9a3ebdb..85aa387d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,17 @@ services: # below — the env guard exists precisely to refuse boot if this policy # (or an equivalent one) is missing. restart: unless-stopped + # An unconstrained container reports the WHOLE host to + # effectiveCpuCount()/effectiveMemoryBudgetBytes(), so + # resolveEmbeddingConcurrency() sizes the transformer pool against every + # core on the machine. On a 24-core host that derives workLimit=8 x + # intraOpNumThreads=3 = 24 native ONNX threads, which then contend with + # postgres and the web service in this same stack. The sizing math is + # correct; it was being handed a budget this service does not actually own. + # Declare the share explicitly and let operators raise it on dedicated + # hardware. + cpus: ${PDPP_REFERENCE_CPUS:-4} + mem_limit: ${PDPP_REFERENCE_MEM_LIMIT:-4g} environment: AS_PORT: "7662" RS_PORT: "7663" @@ -179,6 +190,14 @@ services: # the storage backend. To re-run the migration tool, do it on the # host with the SQLite file at `./packages/polyfill-connectors/ # .pdpp-data/pdpp.sqlite` and the host-published Postgres port. + # Browser profiles live under PDPP_BROWSER_PROFILE_ROOT + # (/var/lib/pdpp/browser-profiles, baked into the image). Without this + # mount they sit in the container's ephemeral layer and are destroyed on + # every restart, so every browser-backed connector -- ChatGPT, Reddit, + # Amazon, Chase -- demands a fresh interactive login after each deploy. + # deploy/docker/docker-compose.yml already mounts this; the owner stack + # did not, and twelve deploys in one day cost the owner that many logins. + - pdpp-data:/var/lib/pdpp - pdpp-transformers:/var/cache/pdpp/transformers - pdpp-home:/root/.pdpp # File connector imports. Override the host-side paths with @@ -225,6 +244,33 @@ services: # `PDPP_POSTGRES_BIND_HOST` AND set non-default credentials. ports: - "${PDPP_POSTGRES_BIND_HOST:-127.0.0.1}:${PDPP_POSTGRES_PORT:-55432}:5432" + # Defaults are sized for a small demo database, not for the ingest volume a + # real instance reaches. Two symptoms this addresses, both observed live: + # thousands of "checkpoints are occurring too frequently" warnings during + # bulk ingest (max_wal_size too small), and autovacuum taking 19+ minutes on + # a 4.2M-row search index while starving the maintenance sweep + # (maintenance_work_mem at the 64MB default). Override per deployment. + # Docker gives a container 64MB of /dev/shm by default. Postgres uses + # shared memory for parallel query workers, and on a multi-million-row + # table that is not enough -- a parallel VACUUM or aggregate fails with + # "could not resize shared memory segment ... No space left on device", + # which reads as a disk problem and is not one. Observed on a 5.5M-row + # records table. + shm_size: ${PDPP_POSTGRES_SHM_SIZE:-1g} + command: + - postgres + - -c + - max_wal_size=${PDPP_POSTGRES_MAX_WAL_SIZE:-8GB} + - -c + - min_wal_size=${PDPP_POSTGRES_MIN_WAL_SIZE:-1GB} + - -c + - checkpoint_timeout=${PDPP_POSTGRES_CHECKPOINT_TIMEOUT:-15min} + - -c + - shared_buffers=${PDPP_POSTGRES_SHARED_BUFFERS:-512MB} + - -c + - effective_cache_size=${PDPP_POSTGRES_EFFECTIVE_CACHE_SIZE:-1536MB} + - -c + - maintenance_work_mem=${PDPP_POSTGRES_MAINTENANCE_WORK_MEM:-512MB} volumes: - pdpp-postgres-data:/var/lib/postgresql/data healthcheck: @@ -267,6 +313,7 @@ services: - "${PDPP_WEB_PORT:-3002}:3000" volumes: + pdpp-data: pdpp-transformers: pdpp-home: pdpp-postgres-data: diff --git a/docs/design-system/ink-carbon/project/explorer/data.js b/docs/design-system/ink-carbon/project/explorer/data.js index 2a21d68f2..97ed0f0b1 100644 --- a/docs/design-system/ink-carbon/project/explorer/data.js +++ b/docs/design-system/ink-carbon/project/explorer/data.js @@ -1379,7 +1379,7 @@ const deepGmail = [ { id: rid("gm"), thread_id: "th8", - from: "Stanford Alumni ", + from: "Northcrest Alumni ", to: ["the owner@example.com"], subject: "Class of 2014 — 5-year reunion", snippet: "Save the date: October 19, 2019. Memorial Auditorium.", diff --git a/docs/design-system/ink-carbon/project/explorer/data.ts b/docs/design-system/ink-carbon/project/explorer/data.ts index 75e9c5270..65b525bcb 100644 --- a/docs/design-system/ink-carbon/project/explorer/data.ts +++ b/docs/design-system/ink-carbon/project/explorer/data.ts @@ -1413,7 +1413,7 @@ const deepGmail = [ { id: rid("gm"), thread_id: "th8", - from: "Stanford Alumni ", + from: "Northcrest Alumni ", to: ["the owner@example.com"], subject: "Class of 2014 — 5-year reunion", snippet: "Save the date: October 19, 2019. Memorial Auditorium.", diff --git a/docs/design/media-acquisition.md b/docs/design/media-acquisition.md index b030f929a..e326adb6a 100644 --- a/docs/design/media-acquisition.md +++ b/docs/design/media-acquisition.md @@ -157,5 +157,5 @@ written, and Immich is the gate. --- -Signed-off-by: Tim Nunamaker +Signed-off-by: PDPP Maintainers Assisted-by: AI diff --git a/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md b/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md index df0a8d4ac..1cb090855 100644 --- a/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md +++ b/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md @@ -94,7 +94,7 @@ neko-specific input seam). Default: **defer** to stay inside the 1–2h box. is a behavior change — STOP, flag it, do not proceed). 3. `pnpm --filter types:check` green. 4. Full console test suite green. -5. Committed as `tnunamak@gmail.com`, one lane per commit, message states which local code +5. Committed as `maintainers@example.com`, one lane per commit, message states which local code was deleted and which RS primitive replaced it. ### Hard stops (escalate, do not improvise) diff --git a/docs/operator/local-collector-runbook.md b/docs/operator/local-collector-runbook.md index 2ab6646bc..9d5cc1144 100644 --- a/docs/operator/local-collector-runbook.md +++ b/docs/operator/local-collector-runbook.md @@ -1,8 +1,10 @@ -# Local Collector Runbook (Claude Code / Codex) +# Local Collector Runbook (Claude Code / Codex / Signal) Status: reference-experimental operator surface. Not PDPP Core or Collection Profile protocol. -This is the single-page operator runbook for running Claude Code and Codex local collectors against a PDPP Docker reference deployment, with resumable connector state. It supersedes the bare `bin/local-device-exporter.ts` flow in `reference-implementation/docs/local-device-exporter.md` — that script remains as a compatibility shim but does not participate in STATE sync. +This is the single-page operator runbook for running Claude Code, Codex, and Signal Desktop local collectors against a PDPP Docker reference deployment, with resumable connector state. It supersedes the bare `bin/local-device-exporter.ts` flow in `reference-implementation/docs/local-device-exporter.md` — that script remains as a compatibility shim but does not participate in STATE sync. + +Steps 1–5 below apply to every local-collector connector, including Signal — swap `--connector claude_code` for `--connector signal` in Steps 2 and 4. Signal carries one additional prerequisite (the `sigtop` sidecar binary) and one structural constraint (it only runs on the owner's own logged-in desktop session) that Claude Code/Codex do not have; see "Signal Desktop prerequisites" below before Step 1. ## What you are setting up @@ -40,6 +42,95 @@ State is authoritative on the server. Before each connector pass the runner fetc current published release. See `docs/reference/local-collector.md`§"Deployment Posture: Published vs Dev". +## Signal Desktop prerequisites + +Signal is a **local-collector-only** connector: unlike Slack (`slackdump`) or +Google Messages (`gmcli`), its sidecar tool cannot ship inside the Core +server image. The manifest declares a `desktop_session` runtime binding, and +the engine resolves that to `local_device` placement and refuses server-side +placement outright. This is by design, not a temporary gap — see +`design-notes/connector-sidecar-packaging-2026-08-17.md` for the full +evidence trail (a container hits four successive failures: missing file, +no D-Bus session, uid mismatch, AppArmor denial). + +**Why:** Signal Desktop's SQLCipher database key is stored encrypted in +`~/.config/Signal/config.json` (`encryptedKey`) and unwraps only through a +session-bound OS keyring — KWallet/GNOME Keyring (`libsecret`) on +Linux, Keychain on macOS, DPAPI on Windows. Check which backend your +Signal Desktop uses: + +```bash +cat ~/.config/Signal/config.json +``` + +- `"safeStorageBackend": "kwallet6"` (or `gnome_libsecret`, etc.) — the + key only unwraps inside the owner's own logged-in desktop session. Run the + collector directly on that machine, logged in as that user, outside any + container. +- `"safeStorageBackend": "basic_text"` — the key is stored unwrapped. + A server-side path is technically possible in this configuration, but it + is not what this connector is built or tested for; treat it as a + documentation note, not a supported deployment target. + +**Install `sigtop`** (github.com/tbvdm/sigtop, ISC license) — the CLI +this connector spawns as an arms-length subprocess to decrypt and read +Signal Desktop's database, the same "sidecar" pattern the `slack` connector +uses for `slackdump`: + +```bash +# Linux: pkg-config needs the libsecret headers to build the safeStorage +# unwrap. If you cannot install system-wide (no root), download the .deb +# with apt-get download (works without sudo) and extract it locally, then +# point PKG_CONFIG_PATH/CGO_LDFLAGS at the extracted tree instead of +# installing system-wide. +sudo apt install libsecret-1-dev pkg-config # Debian/Ubuntu +# or: dnf install libsecret-devel pkgconf-pkg-config # Fedora + +GOBIN=~/.local/bin go install github.com/tbvdm/sigtop/cmd/sigtop@latest +``` + +Note the real import path is `github.com/tbvdm/sigtop/cmd/sigtop` — +`go install github.com/tbvdm/sigtop@latest` (without `/cmd/sigtop`) fails +with "module ... found, but does not contain package ...". + +**Verify the binary actually works** — `sigtop -v` and `sigtop version` +are NOT valid subcommands (there is no version flag at all); use a real +subcommand instead: + +```bash +sigtop check-database # fast SQLCipher integrity check against the local DB; + # exits 0 with no output on success +``` + +If Signal Desktop is running, close it first — sigtop needs unlocked +read access to `db.sqlite`/`db.sqlite-wal`/`db.sqlite-shm`. + +**Point the collector at `sigtop`** if it is not on `PATH` (a custom +`GOBIN`, a non-standard install location, etc.): + +```bash +export SIGTOP_BIN=/absolute/path/to/sigtop # default: "sigtop" on PATH +``` + +`resolveSigtopBin`/`runSigtop` in `packages/polyfill-connectors/connectors/signal/index.ts` +implement this resolution; a missing binary fails fast with a message +naming both the install command and the `SIGTOP_BIN` override, rather than +an opaque `ENOENT`. + +**Building `@pdpp/local-collector` from source for Signal support**: if your +installed `@pdpp/local-collector` predates Signal (`advertise` does not list +`signal` under `bundled_connectors`), rebuild from a checkout that has the +connector. The collector's `tsconfig.build.json` `include` list is the +package's actual shipping manifest — a connector must be listed there +(and its test-only fixture files that pull in `better-sqlite3` must be +listed under `exclude`, the same way `imessage/fixtures.ts` and +`signal/fixtures.ts` are) or its compiled `.js` never reaches the tarball, +even if the connector is registered in `collector-registry.ts` and shows up +under `advertise`. A collector shipping only `collector-definition.js` for a +connector (no `index.js`) will advertise it but fail at spawn time with +`spawn tsx ENOENT` (falling back to running uncompiled `.ts` source, which +needs a `tsx` binary this package deliberately does not depend on). + ## Step 1 — Confirm collector runtime capabilities On the host with Claude/Codex data: @@ -55,10 +146,14 @@ Expected output (capabilities may grow): "runtime": "collector", "bindings": ["network", "filesystem", "local_device"], "collector_protocol_version": "1", - "bundled_connectors": ["claude_code", "codex"] + "bundled_connectors": ["claude_code", "codex", "google_takeout", "imessage", "apple_photos", "google_messages", "signal"] } ``` +If `signal` is missing from `bundled_connectors`, see "Signal Desktop +prerequisites" above — your installed build predates Signal support +and needs rebuilding from a checkout that has it. + Both `claude_code` and `codex` require the `filesystem` binding, which the collector advertises by default. The published package intentionally does not bundle the `browser` binding; browser-bound connectors stay in the monorepo until each has its own publishability review. A connector that requires a binding the collector does not advertise will fail before spawn with `runtime_capability_mismatch` — you do not need to discover that empirically. ## Step 2 — Mint an enrollment code @@ -67,7 +162,7 @@ In a browser, open `/device-exporters` on the reference deployment, signed in as Use the "Create enrollment code" form: -- Connector id: `claude_code` (or `codex`). +- Connector id: `claude_code` (or `codex`, `signal`, ...). - Local binding: a stable name like `personal-laptop` or `ci-runner-eu-1`. Used by the server to namespace the connection id. Existing server responses still expose this compatibility field as `source_instance_id`. - Display name: optional, propagates as the device label. @@ -129,7 +224,7 @@ PDPP_CONNECTION_ID=si_... \ --connector claude_code ``` -Swap `--connector claude_code` for `codex` to ingest Codex CLI history/skills/etc. +Swap `--connector claude_code` for `codex` to ingest Codex CLI history/skills/etc., or for `signal` to ingest Signal Desktop messages/conversations/reactions/attachments (see "Signal Desktop prerequisites" above first — `sigtop` must be installed and Signal Desktop must not be running). Live progress prints to stderr as the connector finds records (phase, running counts, and a final summary), so a large local archive no longer looks stuck diff --git a/docs/reference/stream-evidence-inventory.md b/docs/reference/stream-evidence-inventory.md index 61a891a78..60d218723 100644 --- a/docs/reference/stream-evidence-inventory.md +++ b/docs/reference/stream-evidence-inventory.md @@ -142,8 +142,8 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif | stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state | | --- | --- | --- | --- | --- | --- | --- | -| timeline_points | checkpoint_window | manual_as_of | — | true | — | — | -| timeline_segments | checkpoint_window | manual_as_of | — | false | — | — | +| timeline_points | snapshot_import_receipt | manual_as_of | — | false | — | — | +| timeline_segments | snapshot_import_receipt | manual_as_of | — | false | — | — | ## polyfill/google-maps-data-portability @@ -273,6 +273,15 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif | --- | --- | --- | --- | --- | --- | --- | | orders | checkpoint_window | manual_as_of | — | true | — | — | +## polyfill/signal + +| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state | +| --- | --- | --- | --- | --- | --- | --- | +| messages | snapshot_import_receipt | manual_as_of | — | true | — | — | +| conversations | snapshot_import_receipt | manual_as_of | — | false | — | — | +| reactions | parent_detail_accounting | manual_as_of | — | false | — | — | +| attachments | parent_detail_accounting | manual_as_of | — | false | — | — | + ## polyfill/slack | stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state | diff --git a/openspec/changes/add-connection-pause-resume/design.md b/openspec/changes/add-connection-pause-resume/design.md new file mode 100644 index 000000000..35323e284 --- /dev/null +++ b/openspec/changes/add-connection-pause-resume/design.md @@ -0,0 +1,54 @@ +## Context + +`connector_instances.status` admits `active`, `paused`, `revoked`, `draft`. Before this change, `paused` was reachable only by data transplant or direct SQL, and escapable only through `resumePausedConnectionAfterCredentialCapture` — a side effect of the static-secret credential-capture route. + +The owner asked for pause and resume directly, and two stranded production connections proved the state needed a real exit, not just an entrance. + +## Goals + +- One coherent state machine: `active <-> paused`, `revoked -> active` via reactivate. +- Wrong-state refusals are typed and distinguishable, never a generic inactive error. +- A paused connection is visible in the console and carries its own way back. + +## Decisions + +### Pause ships as an owner action + +The alternative was to ship resume alone (enough to rescue the stranded rows) and defer pause until an owner asked twice. Rejected: a state that can be left but not entered is exactly the incoherence that produced this defect, and the state, the storage constraint, and the owner-facing spec language for it already existed. Shipping resume alone would have left `paused` reachable only by transplant — still a trapdoor, just a monitored one. + +Pause is genuinely distinct from revoke and not redundant with it. Revoke means "stop collecting and treat the authorization as withdrawn". Pause means "stop collecting, keep the credential, resume when I say". A source you can pause without re-authorizing is the feature the owner named. + +### Pause and resume are zero-cascade + +Both are pure status flips on the connector-instance row, matching revoke/reactivate. Neither touches records, credentials, grants, schedules, or the spine. Pause deliberately does NOT revoke the credential — retaining it is the entire difference from revoke, and it is what makes resume a single click rather than a re-authorization. + +### Wrong-state refusals get their own codes + +The namespace resolver reports any status mismatch as `connector_instance_inactive` (400). Both routes re-label it: resume against a non-paused target returns `connector_instance_not_paused` (409); pause against a non-active target returns `connector_instance_not_active` (409). This mirrors reactivate's existing `connector_instance_not_revoked` and lets a caller distinguish "wrong state" from "no such connection" without parsing prose. + +### The explicit owner route is not restricted by binding kind; the automatic hooks still are + +The shared `applyResume` primitive takes an optional `requireSourceBindingKind`. The automatic resume hooks — credential capture and run admission — pass `historical_archive`, keeping the implicit, non-owner-initiated path as narrow as it was. The explicit owner routes pass nothing: when the owner clicks Resume, the binding kind is not the server's business, and restricting it would have re-created the trap for every other paused connection. + +### Credential freshness is delegated to the next run + +Resume does not validate or supply a credential, matching reactivate. A resumed connection whose credential has expired surfaces a typed credential error on its next run through the existing health projection. Validating at resume time would duplicate that machinery and could refuse a resume the owner legitimately wants (for example, a file-import connection that authenticates to nothing). + +### A missing import directory is an infrastructure fault, not owner data loss + +`isManualUploadBinding` returning null for an unusable binding meant the run silently received no import-dir env var and reported `source_incomplete` — which reads as "your archive was incomplete" when the truth is "this server was told to read a path that is not on this disk". The resolver now stats the directory and throws `manual_upload_import_dir_missing` naming the path, the env var, and the connection. This is the silence that let two intact archives sit unnoticed. + +## Risks / Tradeoffs + +- **Pause could be mistaken for revoke.** Mitigated by console copy stating that data and the credential are retained, and by keeping the two actions visually and textually distinct. +- **A paused connection stops collecting silently.** Mitigated by rendering `paused` as a first-class status and surfacing the connection with a Resume action rather than hiding it as revoked connections are hidden. +- **Statting the import directory adds a filesystem call per manual-upload run.** Negligible against a run that then reads the archive, and it converts a silent misattribution into an actionable error. + +## Acceptance Checks + +- Resuming a paused connection returns 200 and the row becomes `active`; a second resume returns `connector_instance_not_paused` (409). +- Pausing an active connection returns 200 and the row becomes `paused`; a second pause returns `connector_instance_not_active` (409). +- Pause and resume leave record counts, credentials, grants, and schedules unchanged. +- A paused connection renders a `paused` status and a Resume action in the console. +- A manual-upload run whose `import_dir` is absent fails with `manual_upload_import_dir_missing` naming the missing path and env var. +- A transplanted `historical_archive` binding is not claimed by the manual-upload resolver. diff --git a/openspec/changes/add-connection-pause-resume/proposal.md b/openspec/changes/add-connection-pause-resume/proposal.md new file mode 100644 index 000000000..dec6b2006 --- /dev/null +++ b/openspec/changes/add-connection-pause-resume/proposal.md @@ -0,0 +1,24 @@ +## Why + +`paused` has been a valid `connector_instances.status` since the schema's first CHECK constraint, and the owner-facing specs already promise pause and resume as per-instance actions. Neither existed. No production code path ever wrote `paused`, and only one path ever cleared it — a side effect of storing a static-secret credential. + +That left `paused` a state with no entrance and, for any connection that reached it another way, no exit. Two of the owner's connections (419,290 records) arrived pre-paused from an archive transplant and could not be recovered: `run` refused them as inactive, `reactivate` refused them as not-revoked, the console had no control because `SourceStatusKind` had no `paused` member, and file-import connections authenticate to nothing so the credential-capture side effect never fired. + +## What Changes + +- Add owner-initiated `pause` (active -> paused) and `resume` (paused -> active) as first-class connection actions, on both the owner-agent bearer surface and the owner-session reference surface. +- Add `connector_instance_not_paused` (409) and `connector_instance_not_active` (409) so each action refuses a wrong-state target with a typed, distinguishable code. +- Make `paused` a first-class `SourceStatusKind` the console renders and explains, with a Resume control on a paused connection and a Pause control on an active one. +- Keep the narrow automatic resume for recovered `historical_archive` connections (credential capture and run admission) unchanged, so repairing a credential still resumes that row without a second owner step. +- Fail loudly when a manual-upload binding's `import_dir` is absent on the host, naming the missing path and env var instead of reporting a bare `source_incomplete`. + +## Capabilities + +Modified: +- `reference-connector-instances` + +## Impact + +- Pause and resume are zero-cascade status flips. Records, credentials, grants, schedules, and the audit spine are untouched by both. +- Pause is deliberately NOT a substitute for revoke: it stops collection while keeping the credential, so it carries no credential-revocation or grant-narrowing semantics. +- A manual-upload run whose artifact directory is missing now fails with a typed, actionable error rather than an owner-blaming coverage verdict. diff --git a/openspec/changes/add-connection-pause-resume/specs/reference-connector-instances/spec.md b/openspec/changes/add-connection-pause-resume/specs/reference-connector-instances/spec.md new file mode 100644 index 000000000..c9b4d5b5d --- /dev/null +++ b/openspec/changes/add-connection-pause-resume/specs/reference-connector-instances/spec.md @@ -0,0 +1,74 @@ +## ADDED Requirements + +### Requirement: Owner-Initiated Connection Pause And Resume + +The reference implementation SHALL expose connection pause and connection resume as owner-initiated, connection-scoped actions keyed on exactly one `connector_instance_id`. + +Pause SHALL transition a connection from `active` to `paused`. Resume SHALL transition a connection from `paused` to `active`. Both SHALL be zero-cascade status flips: they SHALL NOT delete, rewrite, or hide already-collected records; SHALL NOT revoke, rotate, or erase stored credentials; and SHALL NOT alter disclosure grants, schedules, or the audit spine. + +Pause SHALL remain distinct from revoke. Pause stops future collection while retaining the connection's stored credential so that resume requires no re-authorization. Revoke SHALL continue to express a withdrawn authorization. + +Both actions SHALL be reachable on the owner-agent bearer control plane and on the owner-session reference control plane. Neither SHALL be reachable over `/mcp` or by a client grant token. + +A paused connection SHALL be refused by run admission until it is resumed. Resume SHALL NOT itself validate or supply a credential; a resumed connection whose credential is missing or expired SHALL surface a typed credential error on its next collection run. + +#### Scenario: Owner pauses an active connection + +- **WHEN** the owner pauses a connection whose status is `active` +- **THEN** the connection's status SHALL become `paused` +- **AND** its collected records, stored credential, disclosure grants, schedule, and audit spine SHALL be unchanged +- **AND** subsequent run admission for that connection SHALL be refused while it remains paused + +#### Scenario: Owner resumes a paused connection + +- **WHEN** the owner resumes a connection whose status is `paused` +- **THEN** the connection's status SHALL become `active` +- **AND** its collected records SHALL be unchanged +- **AND** the connection SHALL become eligible for run admission + +#### Scenario: Wrong-state pause and resume are typed and distinguishable + +- **WHEN** the owner resumes a connection whose status is not `paused` +- **THEN** the reference SHALL refuse with `connector_instance_not_paused` +- **AND** **WHEN** the owner pauses a connection whose status is not `active` +- **THEN** the reference SHALL refuse with `connector_instance_not_active` +- **AND** neither refusal SHALL mutate the connection + +#### Scenario: Unknown or foreign connection + +- **WHEN** a pause or resume names a connection that does not exist or belongs to another owner +- **THEN** the reference SHALL refuse with `connector_instance_not_found` +- **AND** SHALL NOT disclose whether the connection exists + +#### Scenario: Paused is an owner-visible status with a way back + +- **WHEN** the owner views a connection whose status is `paused` +- **THEN** the console SHALL render `paused` as a first-class status distinct from revoked, syncing, and setup-in-progress +- **AND** SHALL present a resume action for that connection +- **AND** SHALL state that collected data is retained + +### Requirement: Manual Upload Import Directory Absence Is A Typed Failure + +The reference implementation SHALL verify that a manual-upload connection's `import_dir` exists as a directory on the host before a run is given the binding's import-directory environment variable. + +When the directory is absent or is not a directory, the reference SHALL fail the run environment resolution with a typed `manual_upload_import_dir_missing` error naming the missing path, the binding's `import_dir_env_var`, and the `connector_instance_id`. The reference SHALL NOT report this condition as a bare coverage or completeness verdict, because the fault is host or binding configuration rather than an incomplete owner-supplied archive. + +A source binding that is not a manual-upload binding SHALL continue to resolve to no manual-upload environment fragment without raising this error. + +#### Scenario: Missing import directory names what is missing + +- **WHEN** a manual-upload connection's `import_dir` does not exist on the host +- **THEN** run environment resolution SHALL fail with `manual_upload_import_dir_missing` +- **AND** the error SHALL name the missing path, the import-directory environment variable, and the connection +- **AND** the run SHALL NOT report the condition as an incomplete owner-supplied source + +#### Scenario: Import path that is not a directory + +- **WHEN** a manual-upload connection's `import_dir` exists but is not a directory +- **THEN** run environment resolution SHALL fail with `manual_upload_import_dir_missing` + +#### Scenario: Non-manual-upload binding is unaffected + +- **WHEN** a connection's source binding is not a manual-upload binding +- **THEN** manual-upload run environment resolution SHALL yield no environment fragment +- **AND** SHALL NOT raise `manual_upload_import_dir_missing` diff --git a/openspec/changes/add-connection-pause-resume/tasks.md b/openspec/changes/add-connection-pause-resume/tasks.md new file mode 100644 index 000000000..2ac63eefa --- /dev/null +++ b/openspec/changes/add-connection-pause-resume/tasks.md @@ -0,0 +1,43 @@ +## 1. Resume action + +- [x] 1.1 Add the shared `applyResume` primitive with an optional `requireSourceBindingKind` guard. +- [x] 1.2 Add owner-agent bearer resume routes addressed by `connection_id` and by `connector_id`. +- [x] 1.3 Add the owner-session reference resume route, unrestricted by binding kind. +- [x] 1.4 Keep the automatic `historical_archive` resume hooks (credential capture, run admission) narrow. +- [x] 1.5 Pin `connector_instance_not_paused` (409) in the error-status table. +- [x] 1.6 Publish the resume contract manifests and regenerate the OpenAPI/route docs. + +## 2. Pause action + +- [x] 2.1 Add the shared `applyPause` primitive (active -> paused). +- [x] 2.2 Add owner-agent bearer pause routes addressed by `connection_id` and by `connector_id`. +- [x] 2.3 Add the owner-session reference pause route. +- [x] 2.4 Pin `connector_instance_not_active` (409) in the error-status table. +- [x] 2.5 Publish the pause contract manifests and regenerate the OpenAPI/route docs. + +## 3. Console + +- [x] 3.1 Add `paused` to `SourceStatusKind` and render it distinctly from revoked/syncing/pending. +- [x] 3.2 Surface a paused connection with a Resume action instead of hiding it. +- [x] 3.3 Add a Pause action to an active connection's detail page. +- [x] 3.4 Keep the recovered-archive reconnect journey routed to credential repair. + +## 4. Manual-upload import directory + +- [x] 4.1 Verify `import_dir` exists before returning the import-dir env fragment. +- [x] 4.2 Fail with `manual_upload_import_dir_missing` naming path, env var, and connection. +- [x] 4.3 Confirm a non-manual-upload binding still resolves to null without raising. + +## 5. Stranded transplanted bindings + +- [x] 5.1 Add a dry-run-default repair tool that lifts `original_source_binding` to the top level, restores `kind`, and rewrites `import_dir` to a discovered on-disk path. +- [x] 5.2 Refuse on zero and on ambiguous discovery candidates rather than guessing. +- [x] 5.3 Back up the pre-image binding in the same transaction as the write, guarded on the pre-image still being current. + +## 6. Validation + +- [x] 6.1 Unit-test the repair tool's envelope recognition, lift, discovery, and refusals. +- [x] 6.2 Unit-test the import-directory guard, including mutation-checking that removing it turns the tests red. +- [x] 6.3 Test the resume and pause routes, including wrong-state and foreign-target refusals. +- [x] 6.4 Typecheck polyfill-connectors, reference-implementation, console/operator-ui, and packages/mcp-server. +- [x] 6.5 Run `npx biome check` on every touched file. diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/.openspec.yaml b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/.openspec.yaml new file mode 100644 index 000000000..d160e09cf --- /dev/null +++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-21 diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/design.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/design.md new file mode 100644 index 000000000..965d30553 --- /dev/null +++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/design.md @@ -0,0 +1,202 @@ +## Context + +PDPP already had the right mechanism and the right terminal vocabulary. +`run.abandoned` is a first-class terminal event in `check-run-terminal.sql`, +`run-history-writer.ts`, and the `terminal_status` contract in +`openspec/specs/reference-implementation-architecture/spec.md`. A boot +reconciler already wrote it. The design was disabled by one identity defect, +and the surrounding machinery — a drain, a wall clock, a second reconciler — +existed to compensate for the resulting silence. + +This change is therefore mostly subtraction. The one substantive addition is +durable identity; the deletions are its consequence. + +## Goals / Non-Goals + +**Goals:** + +- Every run reaches exactly one durable terminal state, written by exactly one + writer. +- An interrupted run is named `abandoned`, never `failed`. +- A successor container can adjudicate its predecessor's orphans. +- No adjudication decision depends on a tuned time threshold. + +**Non-Goals:** + +- Finishing in-flight work during shutdown. +- Committing staged cursors under interruption. +- A generic cross-table "interrupted work engine". Six boot reconcilers should + share a *predicate*, not an executor; a unified engine would need per-table + SQL, projections, and terminal vocabularies injected into it, which is + relocation rather than decomplecting. + +## Decisions + +### The successor adjudicates; the dying owner writes nothing + +The dying process is the wrong writer because it cannot be relied on to run at +all. A `kill -9` gets no shutdown path, so any design that needs the owner to +write its own terminal state has an unhandled case by construction. + +This is the layering mature systems ship. Temporal's server has no +crash-detection channel and converts worker silence into a recorded outcome +with a timer alone; its `WorkerStopTimeout` — "the time delay before hard +terminate worker" — defaults to **0s**, so it does not wait for in-flight work +by default. Kafka's successor performs the recovery: `InitProducerId` "Bumps up +the epoch of the PID, so that the any previous zombie instance of the producer +is fenced off" and "Recovers (rolls forward or rolls back) any transaction left +incomplete by the previous instance." Primary sources for both, with quotes and +access dates, are in the research entry +`ai/research/pdpp/interrupted-work-needs-an-owner-fenced-terminal-state-not-a-graceful-shutdown-because-dockers-10s-stop-is-shorter-than-the-work.md` +(claims A, sources `sdk-go-worker-base`, `temporal-activity-failures`, +`kip-98`). + +Alternative: retarget the controller path to write `run.abandoned` instead of +deleting it. Rejected because two writers racing for one run's terminal event +is worse than one. The boot reconciler is the better writer: it reads the +append-only spine rather than the `controller_active_runs` flight table, it is +idempotent on `caused_by_event_id` through the +`spine_run_abandoned_cause_unique` partial index, and it aborts boot on error +instead of swallowing it. + +### `abandoned` stays distinct from `failed` + +Interruption is Kubernetes' `Unknown`, not `False`. Kubernetes keeps the two +distinct because they drive different remediation — node `Unknown` yields an +`unreachable` taint, `False` yields `not-ready`. The same holds here: `failed` +on a bank connector means ask the human; `abandoned` means nobody knows, and +the normal schedule will pick it up. Collapsing them is what pages an owner for +a deploy. + +The cost of the collapse is measured, not theoretical: of 134 production runs +recorded as `run.failed`/`controller_restarted`, 55 had staged a cursor and 34 +had durably ingested a batch before being written down as plain failures. + +### Durable identity beats an env var + +Setting `PDPP_CONTROLLER_ID` in the run command would fix the filter. It is +rejected as the *default* because its failure mode is silent. Production runs +from a hand-rolled `docker run`; an identity that depends on an operator +reproducing a flag on every container recreation is one omission away from +reopening this exact defect, with no signal that it has reopened. That is how +the defect hid for three months. Reading the value from the same database that +holds the runs makes the correct answer the default. `PDPP_CONTROLLER_ID` is +kept as an override so a genuine multi-controller deployment can still +partition ownership. + +`os.hostname()` survives only as the seed for the first row, never as the live +identity. The boot epoch still advances per boot; only the identity is stable, +so adjudication still distinguishes "a prior incarnation owned this" from "I +own this". + +### The drain is deleted, not lengthened + +Rejected on measurement. Production sets no `--stop-timeout` +(`docker inspect pdpp-core-prod-drain` returns `StopTimeout=`), so +Docker's 10s default governs, and `--stop-timeout` is fixed at container +creation — Docker has no equivalent of systemd's runtime `EXTEND_TIMEOUT_USEC=` +extension. Only 2 of 17 connectors finish inside 10s at p95; slack's p95 is +4052.8s. The gap is two orders of magnitude and is not closable by tuning. + +The drain was observed in production failing exactly that way, logging +`{"drained":0,"elapsedMs":5000,"timedOut":1,"msg":"connector run drain complete"}`. +It is a negative, not a small win: it consumes half the SIGKILL budget doing +nothing while making the failure look handled. + +Only the shutdown call site is removed. `drainActiveRuns` stays on the +controller, where it means "await in-flight runs" — 137 references exist on the +base branch and 136 remain, the single removal being the shutdown call. Sidekiq +draws the same line between *quiet* and *drain*. + +### The epoch fence, not a wall clock, decides in-flight ownership + +A wall-clock reaper must guess a threshold and can be wrong in both directions. +River documents the cost in its own config comment — "this can result in repeat +or duplicate execution of a job that is not actually stuck but is still +working" — and Oban's Lifeline carries the same caveat. An epoch fence needs no +guess: a unit stamped with epoch E, observed by epoch E' ≠ E, is *provably* +orphaned. + +A `NULL` owner epoch must be swept, since no live process claims it. On +PostgreSQL this arm is spelled out explicitly rather than left to +`IS DISTINCT FROM`, because `owner_epoch IS DISTINCT FROM NULL` reduces to +`owner_epoch IS NOT NULL` and would spare exactly the legacy rows that most +need reclaiming. + +## Corrections to the research entry + +The research entry is `settled` and its prior-art layer holds, but +implementation measurement refuted four of its claims. The spec deltas reflect +the measurements, not the entry. + +1. **"The leak is live and accruing" — false.** The entry inferred acceleration + from re-measuring 121 as 123 forty minutes later. Those two extra rows were + runs the live container had started minutes earlier — live work, not new + orphans. The newest true orphan is 2026-07-10. The leak stopped because the + *dishonest* path started catching what the honest one could not see; two + defects were masking each other. + +2. **Deleting `MANUAL_UPLOAD_IN_FLIGHT_STALE_MS` rests on an epoch that did not + exist.** The entry said the wall clock answered "a question the epoch + answers exactly", assuming an epoch was available on that table. It was not: + `manual_upload_artifacts` had no epoch column, confirmed absent on the live + database. Deleting the clock without adding the column would have been a + straight regression — the sweep would have had nothing left to distinguish + live work from dead. The column is the substantive change; the deletion is + the consequence. + +3. **"The two reconcilers race on the same runs" — false.** They are fully + disjoint: zero of the 134 `controller_restarted` runs ever also received a + `run.abandoned`. This strengthens the case for deleting the controller path + rather than weakening it — there was no overlap proving the boot path + already covered those runs, so the identity fix had to land first. + +4. **The §11 gate is answered NO.** The entry asked whether connectors emit + bounded `DETAIL_COVERAGE` with `covered == considered` and a non-null + boundary. Zero of 34,928 `run.detail_coverage_declared` events in production + carry `boundary`, `slice_start`, or `slice_end`. Committing staged cursors + under `INTERRUPTED` would therefore fabricate denominators. That stage is + correctly not shipped and is an explicit non-goal. + +## Risks / Trade-offs + +- [A backfill adjudicates live work] -> The repair tool excludes runs belonging + to the newest `controller.booted` epoch. This is load-bearing: the first dry + run against production reported 123 because it picked up two runs the live + container had started ninety seconds earlier. Adjudicating those would have + declared live work abandoned and freed the connection for a competing run, + reintroducing the exact duplicate-execution hazard the epoch fence exists to + avoid. Excluding the newest epoch fixes it with no threshold to tune; the + dry run then reported 121, matching the measured backlog. +- [Deleting the controller path loses stale-claim cleanup] -> It does not. The + function is retained as `releaseAbandonedControllerRunClaims` and still + releases stale `controller_active_runs` rows, because + `reconcileBrowserSurfaceLeasesAfterBoot` reads that table to decide which + leases are still held. Releasing a claim and reporting on the work are + separate jobs. +- [No drain means Chromium residue leaks] -> Chromium residue is still cleaned + at next boot by `profile-lock.ts`. +- [A multi-controller deployment adjudicates a peer's live runs] -> The boot + reconciler still filters on `controller_id`, and `PDPP_CONTROLLER_ID` + partitions ownership. Only the repair tool ignores `controller_id`, and it is + documented as single-controller-only, with `--connector` scoping for anyone + else. +- [Legacy rows without an epoch] -> Both new columns are NULL-tolerant. A NULL + owner epoch is treated as unclaimed and swept; a NULL `controller_id` is + treated as ours under the single-controller assumption, which is the + pre-existing behavior. + +## Migration Plan + +1. Land durable identity first. It is behavior-preserving on a host whose + hostname was already stable, and on a container host it only makes the + existing ownership filter match reality. +2. Run the repair tool's dry run, confirm the scope, then `--apply` with + pre-image snapshots. Reversible: it only adds terminal events where none + exist, and never edits or deletes an existing event. +3. Delete the controller failure path only after identity is durable, so the + boot reconciler demonstrably covers the cases it covered. +4. Delete the drain and fence the manual-upload sweep. Both are independent of + the above. +5. Roll back by restoring the controller path and the drain call site. Already + written `run.abandoned` events remain valid and correct. diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/proposal.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/proposal.md new file mode 100644 index 000000000..8d0edfb46 --- /dev/null +++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/proposal.md @@ -0,0 +1,107 @@ +## Why + +A run interrupted by container replacement never received an honest terminal +state, and the mechanism built to give it one was silently disabled. + +`resolveControllerId` fell back to `os.hostname()`, which under Docker is the +container id and is fresh on every `docker run`. `PDPP_CONTROLLER_ID` is unset +in production, so the boot reconciler's ownership filter +`COALESCE(data_json->>'controller_id', $2) = $2` excluded every prior +container's orphans — a lost-and-found that only accepted items it had lost +itself. Production holds 1,231 `controller.booted` events across 1,153 distinct +controller ids; the one host with a stable id (`peregrine`, non-Docker, 18 boots +under one id) is the only place the mechanism ever worked. + +The measured cost: 121 `run.started` events spanning 2026-05-15 to 2026-07-10, +across 106 distinct controller ids, carried no terminal event of any kind. A +second, cruder path picked up part of the slack with the wrong vocabulary — +of 134 runs it recorded as `run.failed`/`controller_restarted`, 55 had staged a +cursor and 34 had durably ingested a batch first. Those were interruptions +recorded as failures, and the two states have different remedies: `failed` on a +bank connector means ask the human, `abandoned` means nobody knows and the +normal schedule will pick it up. + +The shipped 5s SIGTERM drain cannot close this. Production sets no +`--stop-timeout`, so Docker's 10s default governs, and only 2 of 17 connectors +finish inside 10s at p95. It was observed in production logging +`{"drained":0,"elapsedMs":5000,"timedOut":1}` — burning its whole budget and +abandoning the run anyway. + +## What Changes + +- Make controller identity durable in a `controller_identity` row, seeded from + the hostname on the first boot that finds the table empty and read back + unchanged thereafter, so a successor container inherits the ownership filter + that lets it adjudicate its predecessor's orphans. `PDPP_CONTROLLER_ID` still + wins when set. The boot epoch still advances per boot. +- Stop recording interrupted runs as failures. The controller path no longer + emits `run.failed`/`controller_restarted`; it retains only its stale-claim + release, renamed to say what it does. The boot reconciler becomes the single + writer of an interrupted run's terminal state. +- Delete the connector drain from the SIGTERM path. `drainActiveRuns` stays on + the controller, where it means "await in-flight runs" — 136 of its 137 + references carry that meaning and keep working. +- Fence in-flight manual-upload artifacts by owner epoch instead of the + 10-minute `MANUAL_UPLOAD_IN_FLIGHT_STALE_MS` wall clock. This requires adding + an `owner_epoch` column to `manual_upload_artifacts`, which did not exist. +- Add an owner-operated repair tool that adjudicates the already-stranded + backlog, dry-run by default, with pre-image snapshots taken inside the write + transaction. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `reference-implementation-architecture`: Define durable controller identity, + the successor-adjudicates ownership rule and its newest-epoch exclusion, the + single-writer rule for an interrupted run's terminal state, epoch-fenced + in-flight artifact sweeps, and the absence of a shutdown-path drain. + +## Impact + +- `reference-implementation/lib/controller-boot.ts`, + `runtime/controller.ts`, `server/index.ts`, + `server/stores/manual-upload-artifact-store.ts`, + `server/routes/ref-manual-upload-draft-connection.ts` +- New `controller_identity` table and additive `manual_upload_artifacts.owner_epoch` + column on SQLite and PostgreSQL. Both are NULL-tolerant, so existing databases + migrate without a backfill. +- New owner-only repair script + `reference-implementation/scripts/repair/adjudicate-orphaned-runs.ts`. +- Shutdown is faster and its failure mode is honest. Interrupted runs terminalize + as `abandoned` at the next boot rather than as `failed` or not at all. +- Does not change record ingestion, checkpoint commit, or any connector. + +## Non-Goals + +- **Committing staged cursors under interruption.** The research entry gated + this on whether connectors emit bounded `DETAIL_COVERAGE` with + `covered == considered` and a non-null boundary. Measured on the live spine, + the answer is no: zero of 34,928 `run.detail_coverage_declared` events carry + `boundary`, `slice_start`, or `slice_end`. Committing staged cursors under an + interrupted terminal state would fabricate denominators, so it is out of scope + until that evidence exists. +- **The checkpoint-contract / interval-claim design.** Out of scope here, and + now carried by `qualify-connectors-for-incremental-checkpoint-commit`. An + earlier draft of this non-goal described the prototype as not surviving + contact with 2 of 3 connectors and therefore "not ready for OpenSpec". That + reading is corrected: the contract is a **qualification standard**, and a + connector failing it is the qualifier working. `slack` is a genuine + disqualifier (no ordered scan at any granularity); `heb` fails at fine + granularity but qualifies at day granularity under a closed-day rule. See + that change for the verdicts and their evidence. +- **Any Node version change.** +- **Auto-resume or auto-retry of interrupted runs.** `chase`, `usaa`, `venmo`, + `heb`, `amazon`, and `reddit` need an interactive human sign-in; adjudication + is silent by design and lets the normal schedule pick the work up. + +## Residual risks + +- Owner-authorized live verification remains: after one deploy in a replaced + container, a `SIGKILL` mid-run should produce exactly one terminal event for + that run and it should be `run.abandoned`. This is live-environment + verification, not an implementation task. diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/specs/reference-implementation-architecture/spec.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/specs/reference-implementation-architecture/spec.md new file mode 100644 index 000000000..5c632e5dd --- /dev/null +++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/specs/reference-implementation-architecture/spec.md @@ -0,0 +1,242 @@ +## ADDED Requirements + +### Requirement: Every started run SHALL reach exactly one durable terminal state + +For every `run.started` event in the spine, either a terminal event SHALL exist +for that run, or the run's `boot_epoch` SHALL equal the current process's boot +epoch. The canonical terminal set is `run.completed`, `run.failed`, +`run.browser_surface_failed`, `run.cancelled`, and `run.abandoned`. + +Exactly one component SHALL write a given run's terminal state. A run +interrupted by process death SHALL be adjudicated by the boot reconciler +reading the append-only spine. No other reconciler SHALL emit a terminal event +for an interrupted run, and no component SHALL emit a second terminal event for +a run that already has one. + +Adjudication SHALL be idempotent: each emitted terminal event SHALL carry +`caused_by_event_id`, and repeated adjudication passes SHALL produce exactly +one terminal event per orphan. + +#### Scenario: A run interrupted by process death is terminalized at the next boot + +- **WHEN** a run has a `run.started` event, no terminal event, and a `boot_epoch` + that is not the current process's boot epoch +- **THEN** the boot reconciler SHALL emit exactly one terminal event for that run +- **AND** the run's `terminal_status` SHALL become non-null + +#### Scenario: Repeated adjudication does not duplicate terminal events + +- **WHEN** two successive reconcile passes observe the same orphaned run +- **THEN** exactly one terminal event SHALL exist for that run + +#### Scenario: A second writer does not emit a competing terminal event + +- **WHEN** a reconciler other than the boot reconciler observes a stale + `controller_active_runs` claim for a run +- **THEN** it SHALL release the stale claim +- **AND** it SHALL NOT emit any terminal event for that run + +### Requirement: An interrupted run SHALL terminalize as abandoned, never as failed + +A run whose owner process died without reporting SHALL be recorded as +`run.abandoned` with reason `controller_terminated_before_run_finished`. It +SHALL NOT be recorded as `run.failed`, and SHALL NOT be assigned any +failure reason that asserts an observed failure. + +`abandoned` and `failed` SHALL remain distinct terminal states because they +carry different remedies: `failed` indicates an observed failure that MAY +require owner attention, while `abandoned` indicates that no owner will ever +report on the unit and the normal schedule will pick the work up. + +Adjudicating an interrupted run SHALL NOT request owner attention, SHALL NOT +emit an owner notification, and SHALL NOT re-run, re-queue, or retry the work. + +Adjudication SHALL NOT delete or edit any existing event, SHALL NOT modify +ingested records, and SHALL NOT revise `records_emitted`. Records durably +ingested before the interruption SHALL stay committed. + +#### Scenario: An interrupted run with staged cursors is not called a failure + +- **WHEN** a run staged a cursor or durably ingested a batch and was then + interrupted by process death +- **THEN** its terminal event SHALL be `run.abandoned` +- **AND** no `run.failed` event SHALL exist for that run + +#### Scenario: Adjudication is silent + +- **WHEN** an interrupted run for a connector that requires an interactive human + sign-in is adjudicated +- **THEN** zero owner-attention rows SHALL be created +- **AND** zero owner notifications SHALL be sent +- **AND** the run SHALL NOT be automatically retried + +#### Scenario: Durable records survive adjudication + +- **WHEN** a run is adjudicated as abandoned after durably ingesting records +- **THEN** those records SHALL remain committed +- **AND** the run's reported `records_emitted` SHALL NOT be revised + +### Requirement: Controller identity SHALL be durable across container replacement + +The reference implementation SHALL resolve controller identity from +`PDPP_CONTROLLER_ID` when set, and otherwise from a durable single-row +`controller_identity` record stored in the same database that holds the runs. +The record SHALL be seeded from the host name on the first boot that finds it +absent, and SHALL be read back unchanged on every later boot. + +The host name SHALL NOT be used as the live controller identity. Under a +container runtime the host name is the container id and is fresh on every +container creation, which makes an ownership filter keyed on it exclude every +prior container's orphans. + +The boot epoch SHALL continue to advance on every boot. Only the identity is +stable, so adjudication can still distinguish a prior incarnation's work from +the current process's work. + +An identity that must be supplied by operator configuration SHALL NOT be the +default, because omitting it fails silently and reopens the ownership defect +with no signal. + +#### Scenario: A replacement container inherits its predecessor's identity + +- **WHEN** the reference implementation boots in a new container against a + database whose `controller_identity` row already exists +- **THEN** it SHALL adopt the stored controller id rather than its host name +- **AND** its ownership filter SHALL select orphans left by the prior container + +#### Scenario: First boot seeds the identity + +- **WHEN** the reference implementation boots against a database with no + `controller_identity` row and no `PDPP_CONTROLLER_ID` +- **THEN** it SHALL write one row seeded from the host name +- **AND** every later boot SHALL read that same value back + +#### Scenario: The operator override still partitions ownership + +- **WHEN** `PDPP_CONTROLLER_ID` is set +- **THEN** it SHALL take precedence over the stored row +- **AND** a multi-controller deployment SHALL remain isolated by controller id + +### Requirement: A successor SHALL adjudicate only units whose owner epoch is not its own + +A unit of work SHALL carry, in the same durable write that starts it, the +identity of the owner epoch entitled to finish it. A successor epoch SHALL +adjudicate a unit only when that unit's owner epoch is not the successor's own +epoch. + +A successor SHALL NOT adjudicate any unit belonging to the newest boot epoch. +A unit started by the process that is still running is live work, not an +orphan: it lacks a terminal event for the ordinary reason that it has not +finished. Adjudicating it would declare live work abandoned and free its +resource for a competing run, reintroducing the duplicate-execution hazard the +epoch fence exists to prevent. + +Eligibility SHALL be decided by epoch comparison, not by an age threshold. A +unit with a `NULL` owner epoch SHALL be treated as unclaimed and SHALL be +eligible, since no live process claims it. + +An owner-operated repair tool MAY ignore the controller-identity filter, since +that field is the defect being healed, but SHALL NOT ignore the newest-epoch +exclusion. + +#### Scenario: Live work in the newest epoch is never adjudicated + +- **WHEN** an adjudication pass observes a unit whose owner epoch equals the + newest `controller.booted` epoch +- **THEN** it SHALL NOT adjudicate that unit +- **AND** it SHALL NOT release or reassign that unit's resource + +#### Scenario: A prior epoch's unit is adjudicated without a time threshold + +- **WHEN** an adjudication pass observes a unit whose owner epoch is neither its + own nor the newest epoch +- **THEN** it SHALL adjudicate that unit regardless of the unit's age + +#### Scenario: A unit with no recorded owner epoch is eligible + +- **WHEN** an adjudication pass observes an in-flight unit whose owner epoch is + `NULL` +- **THEN** it SHALL treat that unit as unclaimed and eligible for adjudication + +### Requirement: In-flight manual-upload artifacts SHALL be swept by owner epoch + +The manual-upload artifact store SHALL record the owner epoch of the process +that created an artifact, written in the same INSERT that creates it. An +artifact left in an in-flight state SHALL be eligible for sweep when its owner +epoch is not the current epoch, or when its owner epoch is `NULL`. + +Sweep eligibility SHALL NOT be decided by a wall-clock staleness threshold. A +guessed threshold can be wrong in both directions: it can sweep a slow but live +validation out from under itself, or leave a genuinely dead upload in place. + +An artifact whose owner epoch matches the current epoch SHALL NOT be swept. +Claiming an artifact for sweep SHALL remain an atomic compare-and-swap and +SHALL stamp the claiming epoch on a win, so a concurrent second claim loses. + +The owner-epoch column SHALL be additive and `NULL`-tolerant on both backends so +existing databases migrate without a backfill. + +#### Scenario: A live in-flight artifact is never swept + +- **WHEN** the boot sweep observes an in-flight artifact whose owner epoch is the + current epoch +- **THEN** it SHALL leave that artifact untouched regardless of its age + +#### Scenario: An orphaned in-flight artifact is swept immediately + +- **WHEN** the boot sweep observes an in-flight artifact whose owner epoch is not + the current epoch +- **THEN** it SHALL claim and sweep that artifact without waiting for any + staleness interval + +#### Scenario: Legacy artifacts written before the column existed are swept + +- **WHEN** the boot sweep observes an in-flight artifact whose owner epoch is + `NULL` +- **THEN** it SHALL treat it as unclaimed and sweep it +- **AND** the backend predicate SHALL NOT reduce to one that spares `NULL` rows + +#### Scenario: Concurrent claims resolve to one winner + +- **WHEN** two processes attempt to claim the same in-flight artifact for sweep +- **THEN** exactly one SHALL win the compare-and-swap and stamp its epoch +- **AND** the loser SHALL NOT sweep that artifact + +### Requirement: Shutdown SHALL NOT attempt to drain in-flight connector runs + +The reference implementation SHALL NOT wait for in-flight connector runs on the +`SIGTERM` path. Interrupted runs SHALL be adjudicated by the successor at the +next boot instead. + +A shutdown drain SHALL NOT be reintroduced as the mechanism for terminalizing +interrupted work. The container runtime's stop grace period is fixed at +container creation and is shorter than a connector run, so a drain cannot +complete the work, and any drain consumes the grace period that remains before +forced termination. A forced termination receives no shutdown path at all, so a +design that requires the dying process to write its own terminal state has an +unhandled case by construction. + +Removing the drain SHALL NOT remove the controller's ability to await in-flight +runs. That capability SHALL remain available to the run watchdog and to +callers awaiting a specific run. + +Resources that outlive an abrupt termination, such as browser profile locks, +SHALL continue to be reclaimed at the next boot. + +#### Scenario: Shutdown does not wait for a running connector + +- **WHEN** the reference implementation receives `SIGTERM` while a connector run + is in flight +- **THEN** it SHALL NOT block shutdown awaiting that run +- **AND** the run SHALL be adjudicated as abandoned at the next boot + +#### Scenario: Awaiting in-flight runs remains available to other callers + +- **WHEN** the run watchdog or a caller awaiting a specific run needs in-flight + run completion +- **THEN** the controller SHALL still provide that capability + +#### Scenario: Browser profile locks are reclaimed after an abrupt termination + +- **WHEN** the process is terminated without running any shutdown path +- **THEN** stale browser profile locks SHALL be reclaimed at the next boot diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/tasks.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/tasks.md new file mode 100644 index 000000000..04e5f193e --- /dev/null +++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/tasks.md @@ -0,0 +1,86 @@ +## 1. Durable controller identity + +- [x] 1.1 Add a `controller_identity` table on SQLite and PostgreSQL holding one + row, seeded from `os.hostname()` on the first boot that finds it empty. +- [x] 1.2 Resolve controller identity as `PDPP_CONTROLLER_ID`, then the durable + row, so the hostname survives only as a first-boot seed and never as the + live identity. +- [x] 1.3 Keep the boot epoch advancing per boot, so adjudication still separates + "a prior incarnation owned this" from "I own this". +- [x] 1.4 Add `test/controller-identity-durability.test.ts` proving identity + survives a simulated container replacement and that a successor's + ownership filter then selects the predecessor's orphans. + +## 2. Single writer for an interrupted terminal state + +- [x] 2.1 Remove the `run.failed`/`controller_restarted` emission from the + controller reconciliation path. +- [x] 2.2 Retain the stale-claim release as + `releaseAbandonedControllerRunClaims`, since + `reconcileBrowserSurfaceLeasesAfterBoot` depends on + `controller_active_runs` to decide which leases are still held. +- [x] 2.3 Update `list-active-runs.sql` and `check-run-terminal.sql` and the + SQLite/collection-store scheduler drivers for the new terminal vocabulary. +- [x] 2.4 Update tests to the new contract rather than relaxing them: the restart + test asserts `terminal_status === "abandoned"` and that no `run.failed` + exists for the run. + +## 3. Drop the shutdown drain + +- [x] 3.1 Remove the connector drain from the SIGTERM path in `server/index.ts`. +- [x] 3.2 Verify `drainActiveRuns` itself is retained on the controller and that + exactly one reference is removed — 137 references on the base branch, 136 + after — so the watchdog and `awaitRun` keep working. + +## 4. Epoch-fence in-flight manual uploads + +- [x] 4.1 Add a NULL-tolerant `owner_epoch` column to `manual_upload_artifacts` + on both backends, written in the same INSERT that creates the artifact. +- [x] 4.2 Replace the `MANUAL_UPLOAD_IN_FLIGHT_STALE_MS` eligibility predicate + with the epoch predicate, spelling the NULL arm out explicitly on + PostgreSQL so `IS DISTINCT FROM` does not reduce to `IS NOT NULL` and + spare the legacy rows. +- [x] 4.3 Keep the atomic compare-and-swap in `claimForSweep` unchanged in kind, + stamping `owner_epoch` on a win so a concurrent second claim loses. +- [x] 4.4 Update crash-recovery and artifact-store tests on both backends. + +## 5. Repair the stranded backlog (operational) + +- [x] 5.1 Add `scripts/repair/adjudicate-orphaned-runs.ts` following the sibling + repair tools: dry-run by default, `--apply` required to write, full scope + printed on every invocation, payload-free output, `--limit` bounding. +- [x] 5.2 Select orphans without filtering on `controller_id` — that field is + what broke — but exclude runs belonging to the newest + `controller.booted` epoch so live work is never adjudicated. +- [x] 5.3 Snapshot the pre-image of every targeted `run.started` event and every + re-projected `run_history` row into an `aor_backup` table inside the same + transaction as the write. +- [x] 5.4 Add `test/adjudicate-orphaned-runs.test.ts` covering idempotency, + newest-epoch exclusion, and backup completeness. +- [x] 5.5 Run the dry run against production first and confirm the scope before + `--apply`. Verified: dry run reported 121 after the newest-epoch exclusion + (123 before it, the two extras being live runs started 90 seconds + earlier). +- [x] 5.6 Apply. Verified on the live instance: backup table + `aor_backup_ad8166a2__all__20260821161222` holds 121 spine-event + pre-images spanning 2026-05-15 to 2026-07-10 across 106 distinct + controller ids, and the orphan predicate now returns 0. + +## 6. Validation + +- [x] 6.1 Confirm the corrections against the live database rather than the + research entry: newest true orphan is 2026-07-10 (the leak is not + accruing); `manual_upload_artifacts` had no epoch column; zero of the 134 + `controller_restarted` runs also received `run.abandoned` (the reconcilers + never raced); zero of 34,928 `run.detail_coverage_declared` events carry + `boundary`, `slice_start`, or `slice_end`. +- [ ] 6.2 Run focused controller, controller-boot, manual-upload, and repair-tool + suites on both backends, plus typecheck, formatting, and strict OpenSpec + validation. +- [ ] 6.3 Owner-authorized live verification: after one deploy in a replaced + container, `SIGKILL` the server mid-run, restart, and assert the run has + exactly one terminal event and it is `run.abandoned`, with zero + `needs_human` attention rows and zero pushes. This must run the successor + in a *new container*, not just a new process — every pre-existing + boot-orphan test shared a `controller_id` with the orphan it created, + which is what let the defect survive. diff --git a/openspec/changes/allow-resumable-checkpoints-after-restart/proposal.md b/openspec/changes/allow-resumable-checkpoints-after-restart/proposal.md new file mode 100644 index 000000000..ef91c0536 --- /dev/null +++ b/openspec/changes/allow-resumable-checkpoints-after-restart/proposal.md @@ -0,0 +1,127 @@ +## Why + +A run interrupted by a server restart resumes from zero. The owner's standard +for #166 is "if I restart the server, my data doesn't all go unhealthy"; this +is the largest remaining gap against it, and it needs a spec change, so it is +proposed rather than implemented. + +`commitState` has exactly two call sites, both inside `handleDoneClose` +(`runtime/index.ts`). `handleStateMessage` only stages cursors in the +in-process `newState` map, which dies with the process. **A run that never +reaches DONE commits no cursor at all, however long it ran.** + +Concrete cost. A Slack archive walk takes ~54 minutes. Killed at minute 50, it +re-fetches all 50 minutes of work on the next run. If deploys land closer +together than one walk length, such a connection can never finish — it is not +slow, it is non-converging. 45 runs were ended by restarts between 2026-08-15 +and 2026-08-22 (28 `controller_terminated_before_run_finished` + 17 +`controller_restarted`), concentrated in the longest walks: 9 Slack, 4 Gmail, +3 YNAB, plus Amazon and Google Maps. + +**This is bounded, not catastrophic.** `records` carries a UNIQUE CONSTRAINT +`records_connector_instance_stream_key (connector_instance_id, stream, +record_key)`, so ingest is idempotent: a re-walk re-upserts rather than +duplicating, and no already-collected record is lost. The cost of the current +rule is **wasted work and staleness, never data loss.** Any change here must +preserve that property — the current rule is conservative in the right +direction, and a careless relaxation would trade a real-but-bounded cost for an +unbounded one. + +## The rule today, and why it exists + +`spec-collection-profile.md` states it twice, and deliberately: + +- Line 151: "The runtime MUST NOT persist STATE checkpoints from a run that + terminates in the `failed` state, except for the certified stream-scoped + failure described under DONE. State is otherwise persisted only after a + successful DONE." +- Line 497: "A missing or mismatched terminal code, a missing or untargeted + skip, an out-of-scope stream, a protocol violation, an invalid terminal count + or exit code, **a process exit without valid DONE**, or cancellation MUST + preserve the default fail-closed rule and persist no staged STATE." + +Line 497 covers a restart directly — a restart *is* a process exit without +valid DONE. There is no `abandoned` loophole; the case was considered and +closed. The rule is correct in its purpose: a cursor must not advance past +records whose detail coverage was never proven. `DETAIL_COVERAGE` is evaluated +at DONE, and a detail stream may emit coverage for a parent long after that +parent's own STATE message. Committing a parent early could advance past +unhydrated detail that no later run would revisit — converting today's honest +re-fetch into silent data loss. + +An implementation attempt confirmed this is load-bearing, not incidental: +committing at STATE time broke 19 existing tests in +`test/collection-profile.test.ts`, including the explicit contract "STATE is +only committed when DONE status is succeeded". It was reverted. + +## What Changes + +A narrow exception, for a restart only, limited to checkpoints whose coverage +is already PROVEN at the moment the STATE message is handled. + +- Add a terminal disposition for a run ended by controller restart, distinct + from `failed`: the run's outcome was never observed, rather than observed to + be bad. (This mirrors the distinction already drawn on the read side, where + restart-abandoned runs no longer classify connection health.) +- Permit a runtime to persist a staged checkpoint at STATE time **only when + every one of these holds**: + 1. The checkpoint stream is not named as a detail parent by any in-scope + stream's manifest `parent_streams`/`state_stream` declaration — so no + DONE-time `DETAIL_COVERAGE` verdict can ever apply to it. This is + decidable from the manifest before the run starts, and is exactly the + predicate `missingDetailCoverageReports` already uses. + 2. The records the cursor covers are already durably ingested. + `handleStateMessage` already awaits `flushBatch(stateStream)` before + staging, so this holds today at that point. + 3. The connector declared no gap for that stream this run. +- Keep the fail-closed default for every other case: any stream that could + shortfall, any protocol violation, any connector-reported failure, and any + owner cancellation continue to persist nothing. + +Under this exception a restart-interrupted Slack walk resumes near minute 50 +instead of zero, while a list+detail connector like ChatGPT — whose +`conversations` checkpoint gates `messages` detail — is unaffected and keeps +committing only at DONE. + +## What this does NOT propose + +- No change to the `failed`-run rule. A connector that reports failure still + commits nothing beyond the existing certified stream-scoped exception. +- No change to cancellation. An owner cancel still persists nothing. +- No weakening of the coverage gate. A stream that could shortfall is excluded + by construction, not by a runtime judgement call. +- No trust in connector self-declaration. Eligibility is derived from the + manifest and the runtime's own flush ordering, never from a connector flag — + a manifest claim of "safe to commit early" would be voluntary honesty, which + this program has been burned by before. + +## Alternatives considered + +- **Do nothing.** Defensible, because idempotency bounds the cost to wasted + work. Rejected as the default answer because a walk longer than the interval + between deploys never converges — the owner cannot get a complete Slack + archive, which is a data-completeness failure, not a performance one. +- **Shorten the walks instead.** Real, and worth doing independently, but it + does not fix restart-during-walk; it only narrows the window. +- **Graceful drain on SIGTERM.** Already built, deployed, measured, and + removed in `2ddcca1b8` — production logged `drained:0, elapsedMs:5000, + timedOut:1`. Docker's stop timeout is 10s and fixed at container creation + while real runs take minutes, and a `kill -9` gets no drain at all. Not a + viable path; see `design-notes/graceful-drain-verdict-2026-08-22.md`. + +## Impact + +- Affects `spec-collection-profile.md` (normative), and `runtime/index.ts` + `handleStateMessage` once the spec permits it. +- The eligibility predicate is manifest-derived and computable before the run + starts, so conformance is testable offline. +- Verification should follow D15: kill a run mid-walk, then prove the next run + re-fetches nothing before the last committed boundary AND that a + detail-parent checkpoint did not advance. + +## Owner decision required + +The owner gates spec changes. The question is narrow: **may a checkpoint whose +coverage is already proven survive a restart, when the alternative is that +long walks never converge?** The safety property that makes this askable is +that eligibility is decided from the manifest, not from connector claims. diff --git a/openspec/changes/allow-resumable-checkpoints-after-restart/specs/reference-implementation-architecture/spec.md b/openspec/changes/allow-resumable-checkpoints-after-restart/specs/reference-implementation-architecture/spec.md new file mode 100644 index 000000000..37a0b436c --- /dev/null +++ b/openspec/changes/allow-resumable-checkpoints-after-restart/specs/reference-implementation-architecture/spec.md @@ -0,0 +1,69 @@ +## MODIFIED Requirements + +### Requirement: Checkpoint Persistence Across a Controller Restart + +The runtime SHALL continue to fail closed by default: a run that ends in a +protocol violation, a connector-reported failure, an owner cancellation, or any +process exit without a valid DONE SHALL persist no staged STATE, except as +provided below. + +As a narrow exception, the runtime MAY persist a staged checkpoint at the time +its `STATE` message is handled, so that a run interrupted by a controller +restart resumes from that checkpoint rather than from zero. The runtime SHALL +apply the exception to a checkpoint stream ONLY when all of the following hold: + +1. No in-scope stream declares that checkpoint stream as a detail parent, via + the manifest's `state_stream` or `parent_streams` declaration. Eligibility + SHALL be derived from the manifest alone and SHALL be decidable before the + connector is spawned. The runtime SHALL NOT accept a connector-supplied + claim of eligibility. +2. Every record the cursor covers is already durably ingested at the moment the + checkpoint is persisted. +3. The connector has reported no gap for that stream in the current run. + +A checkpoint stream that fails any condition SHALL remain staged and SHALL +commit only under the existing successful-DONE rule, so that a DONE-time +`DETAIL_COVERAGE` verdict can never be bypassed. + +The runtime SHALL make an eagerly persisted checkpoint distinguishable from one +that is merely staged, so a reader of the run timeline can tell which +checkpoints survive a restart. + +A run ended by a controller restart SHALL be reported with a terminal +disposition distinct from `failed`, reflecting that its outcome was never +observed rather than observed to be bad. + +#### Scenario: An interrupted run resumes from its last eligible checkpoint + +- **WHEN** a connector emits records for an eligible checkpoint stream, then a + `STATE` message for that stream, and the process then exits without a valid + DONE +- **THEN** the runtime SHALL have persisted that stream's cursor durably +- **AND** the next run SHALL resume from that cursor rather than from zero +- **AND** the next run SHALL NOT re-fetch records committed before that cursor + +#### Scenario: A detail parent's checkpoint never commits early + +- **WHEN** an in-scope stream declares a checkpoint stream as its detail parent +- **AND** the connector emits a `STATE` message for that parent stream +- **THEN** the runtime SHALL NOT persist that parent's cursor at `STATE` time +- **AND** the parent's cursor SHALL commit only after a successful DONE whose + `DETAIL_COVERAGE` accounting is complete + +#### Scenario: Fail-closed defaults are unchanged + +- **WHEN** a run terminates with a connector-reported failure, an owner + cancellation, an invalid terminal count or exit code, or a protocol violation +- **THEN** the runtime SHALL persist no staged STATE beyond the existing + certified stream-scoped failure exception +- **AND** an ineligible checkpoint stream SHALL persist nothing in every such + case + +#### Scenario: Re-collection after a restart cannot duplicate records + +- **WHEN** a run interrupted by a restart re-collects a range whose records were + already durably ingested +- **THEN** ingest SHALL upsert on the existing record identity rather than + create duplicates +- **AND** the observable cost of an interrupted run SHALL be repeated work only, + never lost or duplicated records diff --git a/openspec/changes/allow-resumable-checkpoints-after-restart/tasks.md b/openspec/changes/allow-resumable-checkpoints-after-restart/tasks.md new file mode 100644 index 000000000..45b97ccf6 --- /dev/null +++ b/openspec/changes/allow-resumable-checkpoints-after-restart/tasks.md @@ -0,0 +1,53 @@ +## 1. Owner decision (blocking) + +- [ ] Owner rules on the narrow exception in `proposal.md`. Nothing below starts + until the spec change is accepted — the current rule is normative and was + deliberately written to cover this case (`spec-collection-profile.md` + lines 151 and 497). + +## 2. Spec + +- [ ] Amend `spec-collection-profile.md` to admit the restart exception, stating + the three eligibility conditions and keeping the fail-closed default for + every other terminal shape. +- [ ] Name the restart disposition explicitly in the spec's terminal vocabulary, + distinct from `failed` (outcome never observed vs. observed bad). + +## 3. Implementation + +- [ ] Add the manifest-derived eligibility predicate (a checkpoint stream named + as a detail parent by any in-scope stream is ineligible), reusing the same + reading of `parent_streams`/`state_stream` that + `missingDetailCoverageReports` already applies. +- [ ] Commit an eligible checkpoint from `handleStateMessage`, after the + existing `flushBatch(stateStream)` await. +- [ ] Report the checkpoint as already-durable in `run.state_staged`, so the + timeline distinguishes a checkpoint that survives a restart from one that + only survives to DONE. +- [ ] Leave the DONE-time commit path unchanged for every ineligible stream. + +## 4. Tests + +- [ ] Prove a connector that emits STATE and then exits WITHOUT DONE leaves its + cursor durable — read back over `GET /v1/state/:connectorId`, the same + surface the next run reads. Simulate the restart by killing the child + after a flushed write; do not use a sleep or any timing proxy. +- [ ] Prove a detail-parent checkpoint does NOT commit early, so an unproven + coverage verdict can never be skipped. +- [ ] Prove the existing contracts still hold: `DONE(failed)`, + `DONE(cancelled)`, and every protocol-violation case in + `test/collection-profile.test.ts` still commit nothing. +- [ ] Mutation-prove each new test in both directions (remove the eager commit; + remove the safety predicate). A predicate mutation MUST fail the + detail-parent test — an earlier attempt at this work had a safety test + that passed with the guard removed, because its fixture manifest was + being rejected and the assertion never exercised the path. + +## 5. Validation + +- [ ] D15 canary: kill a real run mid-walk, then verify the next run re-fetches + nothing before the last committed boundary and that no detail-parent + cursor advanced. +- [ ] Confirm idempotency still bounds the failure direction: re-walking an + already-committed range re-upserts via + `records_connector_instance_stream_key` rather than duplicating. diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/design.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/design.md new file mode 100644 index 000000000..c8f12f8ca --- /dev/null +++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/design.md @@ -0,0 +1,116 @@ +## Context + +`runRequestedStreams` (`packages/polyfill-connectors/connectors/slack/ +index.ts:2568`) runs the messages/reactions/message_attachments unified pass +via `runMessagesUnifiedPass`, then declared coverage for all three streams +before this change. It is called from two places: + +1. `collect()`'s main flow (~line 2925), once per run, against the base + archive. +2. `mergeScopedMessageArchivePasses` (~line 1347), inside a `for (const + archive of deps.scopedArchives)` loop, once per scoped archive that + `reconcileMessageSourceCache` selected to heal a channel missing from the + base archive. + +Call site 2 only runs when `messageFamilyRequested && isUnscopedMessageBoundary +&& reconciledSourceCache.scopedArchives.length > 0` (collect(), ~line 2932). +Production's base archive drops channels over time (export-scope drift, +workspace membership changes); `reconcileMessageSourceCache` heals each +missing channel from its own existing scoped archive. Production has always +needed 3 scoped archives to heal, so call site 2's loop has always run 3 +times per run, each one calling `runRequestedStreams` again on the SAME +shared `emit`. + +`runRequestedStreams`, before this change, emitted DETAIL_COVERAGE for +`messages` (self-coverage) and for `reactions`/`message_attachments` +(family coverage, via `declareMessageFamilyCoverage`) unconditionally, +every time it ran the messages/reactions/message_attachments branch. So one +base-archive call (site 1) plus 3 scoped-archive calls (site 2's loop) = +4 emissions of each (state_stream, stream) pair through the same `emit`. +`reference-implementation/runtime/index.ts:3128`'s `trackDetailCoverage` +tracks `(state_stream, stream)` pairs it has already seen this run and +throws `Connector emitted duplicate DETAIL_COVERAGE for +state_stream=${stateStream} stream=${stream}` on any repeat. Every Slack +run touching 2+ scoped archives has hit this and failed; 8/8 recorded runs. + +## Decisions + +### (a) Move the emission out of `runRequestedStreams`, into `collect()`, after the fold + +`runRequestedStreams` returns a `MessagesPassResult` (`{ channelMaxTs, +maxMessageTs, considered }`) whether or not it's the function emitting +coverage. `mergeMessagesPassResults` (line ~849) already sums `considered` +across every result it merges — `merged.considered` (or in `collect()`, +`messageResult.considered` after the merge assignment) is the correct +summed denominator across the base archive plus every scoped archive +folded in this run. + +The fix: `runRequestedStreams` no longer emits `messages`/`reactions`/ +`message_attachments` DETAIL_COVERAGE at all. `collect()` emits it exactly +once, unconditionally, via a new `declareMergedMessageCoverage(deps, +messageResult.considered)` helper, placed after both: + +- the base-archive `runRequestedStreams` call, and +- the conditional `mergeScopedMessageArchivePasses` fold (which reassigns + `messageResult` to the folded total when it runs). + +`declareMergedMessageCoverage` internally no-ops when the message family +wasn't requested this run (mirrors the guard `runRequestedStreams` used to +apply inline), so `collect()` calls it unconditionally rather than adding +an `if` at the call site — this keeps `collect()`'s branch count from +growing (it was already at the connector's biome cognitive-complexity +ceiling). + +### (b) Why not gate `runRequestedStreams`'s emission on "am I the last call" + +An alternative considered: keep the emission inside `runRequestedStreams` +but only emit when it's the final call in a fold (e.g. thread an `isLast` +flag through the loop). Rejected: it requires the loop's caller +(`mergeScopedMessageArchivePasses`) to know about `runRequestedStreams`'s +internal emission timing, and it does nothing for the fact that the base +archive's call (site 1, outside the loop entirely) ALSO emits — the +base-archive emission and the final scoped-archive emission would still be +two separate emissions of the same pair. The loop and the base call are two +independent call sites; the only place that has ever seen the FINAL summed +`considered` for both is `collect()`, after both have run. Correctness +requires the emission to live where the merge result is observed, not +inside either producer. + +### (c) Single-archive path is unchanged + +When `reconciledSourceCache.scopedArchives.length === 0` (the common case: +no channel needs healing this run), `mergeScopedMessageArchivePasses` never +runs and `messageResult` is exactly what the base-archive +`runRequestedStreams` call returned — the same value the removed inline +emission used to read. `declareMergedMessageCoverage` still runs exactly +once, with the same `considered`/`covered` value as before this change. +This is the case `slack-collection-report.test.ts` and +`canvases-considered.test.ts`-style unit coverage exercise; the constraint +is that this change is behavior-invisible on that path. + +### Acceptance checks + +- 0 scoped archives: exactly one messages/reactions/message_attachments + DETAIL_COVERAGE emission, `considered` equal to the base archive's row + count — identical to pre-change behavior. +- 1 scoped archive: pre-fix, `runRequestedStreams` ran twice (base + the one + scoped archive), emitting each pair twice — `reference-implementation/ + runtime/index.ts`'s `trackDetailCoverage` would reject the second + occurrence. This was never actually observed failing in + `archive-reclaim.test.ts`'s existing single-scoped-archive coverage + because that harness (`runConnectorProtocolSubprocess`) spawns the + connector standalone and never runs it through the RI runtime process + that owns `trackDetailCoverage` — so a duplicate emission there is + silently accepted by the test harness, not proof the pair is duplicate + -safe in production. Post-fix: exactly one emission per pair, `considered` + equal to base + scoped row count. +- 2+ scoped archives: this is the shape production has always needed (3 + scoped archives every run) and the shape that has never once completed — + pre-fix, N+1 emissions of each pair (base + N scoped archives), and the + RI runtime throws on the 2nd. Post-fix: exactly one emission per pair, + `considered` equal to the summed row count across base + every scoped + archive folded. `scoped-archive-coverage.test.ts` drives the real + connector end-to-end (`PDPP_SLACK_SKIP_SLACKDUMP=1`, 2 distinct scoped + archives, full messages/reactions/message_attachments scope) and asserts + directly on the emitted DETAIL_COVERAGE sequence — this is the + regression test for this change. diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/proposal.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/proposal.md new file mode 100644 index 000000000..d910d5636 --- /dev/null +++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/proposal.md @@ -0,0 +1,56 @@ +## Why + +The Slack connector has 8 recorded production runs and 0 successes. Every +run that folds 2+ scoped archives (`packages/polyfill-connectors/connectors/ +slack/index.ts`, `mergeScopedMessageArchivePasses`, ~line 1364) dies with +`reference-implementation/runtime/index.ts:3135`'s `Connector emitted +duplicate DETAIL_COVERAGE for state_stream=messages stream=messages`. +Production touches 3 scoped archives per run. + +`mergeScopedMessageArchivePasses`'s `for (const archive of +deps.scopedArchives)` loop calls `runRequestedStreams` once per archive. +Before this change, `runRequestedStreams` itself emitted the messages +self-coverage `DETAIL_COVERAGE` (`state_stream=messages`, `stream=messages`) +and the message-family `DETAIL_COVERAGE` (`state_stream=messages`, +`stream=reactions|message_attachments`, via `declareMessageFamilyCoverage`) +on every call. With N scoped archives this emitted each (state_stream, +stream) pair N times through the same `emit` side-channel. `polyfill-runtime` +already requires a connector emit `DETAIL_COVERAGE` exactly once per run +(see `openspec/specs/polyfill-runtime/spec.md`, "Connectors with a detail +lane SHALL emit DETAIL_COVERAGE once per run") — Slack's scoped-archive fold +violated it. + +The summed denominator this emission needs already exists: +`mergeMessagesPassResults` (line ~849) sums `considered` across archives, +and the loop's `merged` accumulator (returned at the end of +`mergeScopedMessageArchivePasses`) carries the correct total. The emission +was simply happening at the wrong call site — inside the per-archive +function, instead of once after the fold completes. + +## What Changes + +- Remove the messages self-coverage and message-family `DETAIL_COVERAGE` + emission from inside `runRequestedStreams`. +- Add a single post-fold emission in `collect()`, using + `messageResult.considered` — the value after `mergeScopedMessageArchivePasses` + (when it runs) has folded every scoped archive into the base archive's + total. When no scoped-archive fold happens (the ordinary single-archive + run), `messageResult` is just the base archive's own result, so the + emitted value is unchanged from today. +- No change to `mergeMessagesPassResults`, `mergeScopedMessageArchivePasses`'s + fold loop, or the manifest's `state_stream`/`coverage_strategy` declarations + for `reactions`/`message_attachments`. + +## Capabilities + +Modified: +- `polyfill-runtime` + +## Impact + +- Affects `packages/polyfill-connectors/connectors/slack/index.ts` only. +- Fixes every Slack run that touches 2+ scoped archives (the only shape + production has ever exercised past the first channel-recovery run) — + these have never once completed. +- No change to the single-scoped-archive or no-scoped-archive cases: same + emission count (one), same `considered`/`covered` value as today. diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/specs/polyfill-runtime/spec.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/specs/polyfill-runtime/spec.md new file mode 100644 index 000000000..dc7622d3d --- /dev/null +++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/specs/polyfill-runtime/spec.md @@ -0,0 +1,64 @@ +## MODIFIED Requirements + +### Requirement: Connectors with a detail lane SHALL emit DETAIL_COVERAGE once per run + +A connector that runs a list+detail lane SHALL emit exactly one `DETAIL_COVERAGE` +message per run, after the detail lane completes. A list+detail lane is one that +fetches a list of records and then fetches per-record detail for at least a +subset of those records. The message SHALL carry: + +- `stream`: the detail stream name. +- `state_stream`: the list/parent stream whose cursor anchors the detail pass. +- `required_keys`: the full set of record keys the connector considered for + detail fetch in this run. +- `hydrated_keys`: the subset of `required_keys` for which detail was + successfully fetched and emitted. +- `gap_keys` (optional): keys for which a `DETAIL_GAP` was emitted. +- `optional_skip_keys` (optional): keys skipped by explicit policy (e.g. + rate-limited voluntarily, filtered by selection scope). + +Connectors that emit only flat streams with no per-record detail fetch are +exempt from this requirement. + +When a connector internally folds multiple independent sources into one +run's coverage for a `(state_stream, stream)` pair — for example, healing +missing partitions from separate archives and merging their results — the +"once per run" requirement applies to the run's single externally observed +DETAIL_COVERAGE emission for that pair, not to each internal source the +connector folds. The connector SHALL compute the merged denominator (e.g. a +summed `considered`) across every source folded, then emit exactly once +using that merged value. An internal per-source pass SHALL NOT itself emit +DETAIL_COVERAGE for a pair that the run also emits once, merged. + +#### Scenario: list+detail run emits DETAIL_COVERAGE after the detail lane + +**WHEN** a connector completes a list+detail run +**THEN** the connector SHALL emit a `DETAIL_COVERAGE` message +**AND** the message SHALL appear after the last RECORD or DETAIL_GAP emitted by +the detail lane in the same run +**AND** `required_keys` SHALL equal the set of keys the connector scanned for +detail + +#### Scenario: fully hydrated run emits DETAIL_COVERAGE with no gap_keys + +**WHEN** a list+detail run completes with no DETAIL_GAP messages +**THEN** `DETAIL_COVERAGE.hydrated_keys` SHALL equal `DETAIL_COVERAGE.required_keys` +**AND** `gap_keys` SHALL be absent or empty + +#### Scenario: partially hydrated run carries gap_keys matching emitted DETAIL_GAPs + +**WHEN** a list+detail run emits N DETAIL_GAP messages +**THEN** `DETAIL_COVERAGE.gap_keys` SHALL contain those N keys +**AND** `hydrated_keys` SHALL NOT contain keys that also appear in `gap_keys` + +#### Scenario: a run folds multiple internal sources into one coverage pair + +**WHEN** a connector's run internally reads from 2 or more independent +sources (e.g. a base archive plus one or more supplementary archives healing +missing partitions) that each contribute to the same `(state_stream, +stream)` DETAIL_COVERAGE pair +**THEN** the connector SHALL merge each source's contribution into one +denominator (e.g. summed `considered`) before emitting +**AND** the connector SHALL emit that `(state_stream, stream)` +DETAIL_COVERAGE pair exactly once for the run, using the merged denominator +**AND** no internal per-source pass SHALL emit that pair on its own diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/tasks.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/tasks.md new file mode 100644 index 000000000..cb7720b04 --- /dev/null +++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/tasks.md @@ -0,0 +1,24 @@ +## 1. Reproduction test (write first, before touching the connector) + +- [x] Add `packages/polyfill-connectors/connectors/slack/scoped-archive-coverage.test.ts`: seed a base archive missing 2 channels, each recoverable from its own separate pre-existing scoped archive, drive the real connector via `runConnectorProtocolSubprocess` (`PDPP_SLACK_SKIP_SLACKDUMP=1`) with `messages`/`reactions`/`message_attachments` in scope, and assert each `(state_stream, stream)` DETAIL_COVERAGE pair appears exactly once with the summed `considered`. +- [x] Confirm the test fails against unmodified `connectors/slack/index.ts` (3 emissions of the `messages` pair: base + 2 scoped archives), then leave it in the suite. + +## 2. Fix + +- [x] Remove the messages self-coverage and message-family DETAIL_COVERAGE emission from inside `runRequestedStreams`. +- [x] Add `declareMergedMessageCoverage(deps, considered)`, gated internally on the message family being requested (mirrors the removed inline guard), emitting the self-coverage then delegating to the existing `declareMessageFamilyCoverage`. +- [x] Call `declareMergedMessageCoverage` once from `collect()`, after both the base-archive `runRequestedStreams` call and the conditional `mergeScopedMessageArchivePasses` fold, using the (possibly folded) `messageResult.considered`. + +## 3. Existing test repair + +- [x] Update `integration.test.ts`'s `"runRequestedStreams: archive message enumeration bounds both derived streams"` test, which asserted DETAIL_COVERAGE emission from inside `runRequestedStreams` directly — that emission moved to the caller, so assert on the returned `MessagesPassResult.considered` instead, and assert no DETAIL_COVERAGE is emitted from this call in isolation. + +## 4. Validation + +- [x] Run the new test; confirm GREEN with the fix applied. +- [x] Run the full `connectors/slack/**/*.test.ts` suite; confirm no regressions (201/201 pass). +- [x] Run `reference-implementation`'s `test/slack-collection-report.test.ts` (unaffected — projection layer, not connector). +- [x] `npm --prefix reference-implementation run typecheck` — clean. +- [x] `npm --prefix packages/polyfill-connectors run typecheck` — clean. +- [x] `npx biome check` on every changed file — clean (required extracting `declareMergedMessageCoverage` and gating the message-family check inside it, rather than an `if` in `collect()`, to stay under the cognitive-complexity ceiling). +- [x] `openspec validate fix-slack-scoped-archive-coverage-duplication --strict` — passes. diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/design.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/design.md new file mode 100644 index 000000000..64292cfc0 --- /dev/null +++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/design.md @@ -0,0 +1,48 @@ +## Context + +The coverage snapshot answers one question: did the collector account for +everything the server requires? The previous gate also failed when the collector +accounted for something the server no longer asks about, which is a different +question and not one that bears on completeness. + +## Decision: the missing/unexpected asymmetry is the whole change + +**Unexpected is not fatal.** A collector reporting a store this build no longer +declares scanned *more* than was asked of it. That cannot weaken a completeness +claim. The parser already excludes unexpected entries from `rows` — it +`continue`s before pushing — so they cannot corrupt the proof either. Their only +effect was to fail the gate. + +**Missing stays fatal.** A missing store means the collector did not account for +something the server requires. The snapshot genuinely is not committed, and +saying otherwise would be the dishonesty this codebase is trying to remove. + +**Duplicate and malformed stay fatal.** Both indicate a report that cannot be +trusted to mean what it says, which is distinct from a report that means more +than needed. + +## Why not fix the device instead + +Updating the collector on the affected device resolves this instance and leaves +the class untouched. Any user whose device and server versions ever diverge — +which is every user eventually — hits the same wall, and the failure presents as +"Not measured" with no diagnostic naming the drift. The lenient gate fixes the +class and fixes this instance without touching the device. + +## Prior art + +restic exits `0` on full success and `3` on "some source files could not be +read" — an incomplete-but-usable snapshot is still created. borg distinguishes +`0` success, `1` warning, `2` error. rclone's `check` reports `--missing-on-dst`, +`--differ` and `--match` as separate named categories. None discards a result +because it is imperfect in one direction; partial coverage is a tagged status on +still-valid data. + +## Acceptance + +- Every expected store present plus one unexpected store: snapshot commits, + `unexpectedStores` still reports the drift, the unexpected entry never appears + in `rows`. +- A missing required store: snapshot does not commit. +- A duplicate store: snapshot does not commit. +- A malformed entry: snapshot does not commit. diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/proposal.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/proposal.md new file mode 100644 index 000000000..c46be7773 --- /dev/null +++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/proposal.md @@ -0,0 +1,37 @@ +## Why + +`packages/polyfill-connectors/src/local-source-inventory.ts` treated an +unexpected store as fatal to a coverage snapshot: + +```ts +const hasCommittedSnapshot = + !malformed && duplicateStores.length === 0 && + unexpectedStores.length === 0 && missingStores.length === 0; +``` + +An unexpected store is one the collector reported that the server's descriptor +table no longer declares — the normal result of a device running a build older +than the server. One such name set `hasCommittedSnapshot = false`, then +`reliable = false`, then coverage axis `unknown`, rendered as "Not measured". + +Measured on production: connection `cin_ece4bfe5096b8bf67a1468c2` ("peregrine +Codex") has **1,293,596 collected records**, summary evidence `state=fresh` with +every component `current`, a current heartbeat and a drained outbox — and +displays "Not measured". The only drift is a single legacy `logs` store. Nothing +is missing: every declared store is reported. + +## What Changes + +- An unexpected store no longer disqualifies a coverage snapshot. +- `unexpectedStores` is still returned, so the drift stays observable — it + becomes informational rather than disqualifying. +- Missing, duplicate, and malformed remain fatal, unchanged. + +## Capabilities + +- Modified: local-agent-collector-completeness + +## Impact + +- `packages/polyfill-connectors/src/local-source-inventory.ts` +- `packages/polyfill-connectors/src/local-source-inventory.test.ts` diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/specs/local-agent-collector-completeness/spec.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/specs/local-agent-collector-completeness/spec.md new file mode 100644 index 000000000..0a8f9c568 --- /dev/null +++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/specs/local-agent-collector-completeness/spec.md @@ -0,0 +1,33 @@ +## MODIFIED Requirements + +### Requirement: Coverage snapshots SHALL tolerate unexpected stores and refuse missing ones + +A local coverage snapshot SHALL commit when every store the server declares is +accounted for, even if the collector also reports stores the server does not +declare. A store the server does not declare SHALL NOT contribute to the proof and +SHALL NOT disqualify it. A snapshot SHALL NOT commit when a declared store is +missing, when a store is reported more than once, or when an entry is malformed. + +#### Scenario: Older collector reports a store the server no longer declares + +- **WHEN** a coverage snapshot contains every declared store and one additional + store the server does not declare +- **THEN** the snapshot SHALL commit +- **AND** the undeclared store SHALL be reported as unexpected +- **AND** the undeclared store SHALL NOT appear in the accounted rows + +#### Scenario: A declared store is absent + +- **WHEN** a coverage snapshot omits a store the server declares +- **THEN** the snapshot SHALL NOT commit +- **AND** the absent store SHALL be reported as missing + +#### Scenario: A store is reported twice + +- **WHEN** a coverage snapshot reports the same store more than once +- **THEN** the snapshot SHALL NOT commit + +#### Scenario: An entry is malformed + +- **WHEN** a coverage snapshot contains an entry that cannot be parsed +- **THEN** the snapshot SHALL NOT commit diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/tasks.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/tasks.md new file mode 100644 index 000000000..7e6c42f0e --- /dev/null +++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/tasks.md @@ -0,0 +1,19 @@ +## 1. Test first + +- [x] Failing test: every expected store present plus one unexpected store, asserting + the snapshot commits. Red against unmodified source. +- [x] Regression test: a missing required store still fails closed. +- [x] Existing regression coverage for duplicate and malformed entries retained. + +## 2. Implement + +- [x] Remove `unexpectedStores.length === 0` from the `hasCommittedSnapshot` gate. +- [x] Keep `unexpectedStores` in the returned result so drift stays observable. +- [x] Document the missing/unexpected asymmetry at the decision site. + +## 3. Validate + +- [x] Module tests pass (33 pass, 0 fail). +- [x] Biome clean on changed files. +- [ ] Full reference-implementation suite green. +- [ ] `openspec validate make-local-coverage-tolerate-unexpected-stores --strict` diff --git a/openspec/changes/own-run-lifecycle-state-machine/.openspec.yaml b/openspec/changes/own-run-lifecycle-state-machine/.openspec.yaml new file mode 100644 index 000000000..d160e09cf --- /dev/null +++ b/openspec/changes/own-run-lifecycle-state-machine/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-21 diff --git a/openspec/changes/own-run-lifecycle-state-machine/design.md b/openspec/changes/own-run-lifecycle-state-machine/design.md new file mode 100644 index 000000000..28d624aa6 --- /dev/null +++ b/openspec/changes/own-run-lifecycle-state-machine/design.md @@ -0,0 +1,500 @@ +## Context + +The vocabulary for run lifecycle already exists and is mostly correct. What is +missing is an owner. `run.abandoned` is a first-class terminal event in +`check-run-terminal.sql` and `run-history-writer.ts`; `run_generation` is a +textbook Kleppmann fencing token; `finalizeRunCleanup` implements a correct +compare-and-swap. Each is right on its own. None of them is *the* authority, +so each had to be invented where it was needed, and the copies drift. + +This design is therefore about consolidation, not invention. The states are +already implied by the terminal set; the transitions are already implied by the +guards; the fence already ships in `controller_identity`. The work is to state +them once, in one module, with the database — not a comment — enforcing them. + +Every claim below cites code read on `fix/sweep-fairness-and-transformer-bounds` +at `39d19704a`. Where the brief's figures did not reproduce, the measured value +is given and the discrepancy is named. + +## Goals / Non-Goals + +**Goals:** + +- One closed state set, declared once, that every consumer reads. +- One writer per transition, named in the table. +- Every durable transition is a compare-and-swap fenced by owner epoch, failing + at the database rather than by convention. +- Every historical incident cluster is expressible as an illegal transition, or + is recorded as a finding explaining why it is not. +- Property tests that fail before implementation and pass after. + +**Non-Goals:** + +- Scheduling policy of any kind (D4's guard). +- A generic "workflow engine." Six boot reconcilers should share a predicate, + not an executor — the sibling change already ruled on this and it holds here. +- Changing observable connector behavior. This is a refactor of truth-keeping + (D14). + +## (a) The closed state set + +Nine states. `run_history.status` is the durable projection of this set, and +its existing literals map one-to-one — the set is not new vocabulary, it is the +existing vocabulary closed and named. + +| State | Terminal | Durable projection | Meaning | +|---|---|---|---| +| `pending` | no | `run_history.status='pending'` | Admitted by the executor, not yet started. Exists so admission has a durable pre-state to CAS against. | +| `running` | no | `status='running'` | `run.started` emitted; the owning epoch is executing. | +| `awaiting_interaction` | no | `status='running'` + derived | The connector asked the owner for input and is blocked. Today derived in SQL by `controller-boot.ts:374-387`. | +| `cancel_requested` | no | `status='running'` + derived | `run.cancel_requested` seen; the executor has not yet stopped. | +| `succeeded` | **yes** | `status='succeeded'` | `run.completed`. | +| `failed` | **yes** | `status='failed'` | `run.failed`. An observed failure. | +| `surface_failed` | **yes** | `status='surface_failed'` | `run.browser_surface_failed`. Terminal pre-launch. | +| `cancelled` | **yes** | `status='cancelled'` | `run.cancelled`. Owner-initiated. | +| `abandoned` | **yes** | `status='abandoned'` | `run.abandoned`. No owner will ever report. Distinct from `failed` per the landed owner-epoch change. | + +`pending` is the one addition. Everything else already exists as a +`toTerminalStatus` output (`run-history-writer.ts:91-108`) or a `status` +literal. `awaiting_interaction` and `cancel_requested` are named here because +the machine must be able to *refuse* transitions out of them; they project onto +`running` so no reader changes. + +`skipped` is deliberately **not** a run state. It is written to +`run_history.status` by the scheduler's pre-run gate +(`scheduler/pre-run-gate.ts:99,119,141,157,173,193`) and by +`scheduler/run-executor.ts:555,572,592` for attempts that never started a run. +A skipped attempt has no `run.started` event and no run to transition — it is a +*dispatch outcome*, which is planner territory under D4. Recording it in the +same column as run state is how "scheduler-generated `status:"skipped"` records +fed into health classification" (the Gmail identity self-poisoning loop) became +possible. The machine does not adopt it; the migration keeps it writable by the +planner and forbids the machine from reading it as a run state. + +**The set SHALL be declared once.** Today the terminal event set is declared in +at least **thirteen** places and **six of them disagree**. `lib/spine.ts:1066` +claims authority in its own comment — "All run-status projection code must read +from this set; never hardcode subset checks" — and is then not read by the +divergent copies: + +| Declaration | Contents | Verdict | +|---|---|---| +| `lib/spine.ts:1066` (claims canonical) | 5 | correct | +| `stores/run-history-writer.ts:60`, `stores/connector-attention-store.ts:50`, `scripts/repair/adjudicate-orphaned-runs.ts:123`, `controller-boot.ts:707`, `queries/spine/check-run-terminal.sql:15`, `queries/spine/get-run-terminal-event.sql:13`, `db.ts:5992` | 5 | correct | +| `connector-summary-read-model.ts:1253` | **4 — omits `run.abandoned`** | **defect** | +| `db.ts:5437` `SPINE_TERMINAL_EVENT_TYPES_SQL` | **4 — omits `run.abandoned`** | **defect** | +| `postgres-storage.ts:2692,2716,2744,2793` | **4 — omits `run.abandoned`** | **defect** | +| `connector-summary-evidence-engine.ts:1599,1799` | **4 — omits `run.abandoned`** | **defect** | +| `lib/postgres-spine.ts:570` | **4 — omits `run.browser_surface_failed`** | **defect** | +| `postgres-storage.ts:2336` | **4 — omits `run.browser_surface_failed`** | **defect** | + +`db.ts:5433` states it is "kept in sync with" `connector-summary-read-model.ts`. +The two agree with each other and both disagree with `lib/spine.ts`. The +observable consequence: an abandoned run is invisible to the connector-summary +fold and to the `connector_instance_id` backfill the fold's partial index +serves. The 121 runs adjudicated by the sibling change are exactly the +population this omission hides. Separately, four PostgreSQL-side declarations +omit a *different* member than the SQLite-side ones, so the two backends +disagree about what "terminal" means — cluster 4's defect class, in the +terminal vocabulary itself. + +A comment asking two constants to stay in sync is not a mechanism. This is the +completeness test for D3 working as intended: writing the state set down found +six live divergences that no test catches. + +### Audit: every side-state that lives beside run state + +Verdicts are **absorb** (becomes a real state), **derive** (computed from the +machine, never stored), or **delete**. + +| Side-state | Location | Verdict | Rationale | +|---|---|---|---| +| `activeRuns` map | `controller.ts:981` | **derive** | Currently the real admission authority; `finalizeRunCleanup:3249` hand-rolls a CAS on it. Becomes a read-through cache of `state ∈ {pending, running}` fenced by epoch. Its correctness stops depending on process memory. | +| `activeRunPromises` | `controller.ts:986` | **keep, out of scope** | Holds a JS promise, not state. "Await this run" is a runtime affordance with no durable meaning. Not a second truth. | +| `settledRunIds` | `controller.ts:997` | **delete** | Exists only to answer "did finalize already run?" — which `state` in a terminal set answers exactly. The `isStale` probe at `:3312` becomes a state read. | +| `controller_active_runs` | `db.ts:1149`, `postgres-storage.ts:1945` | **absorb** | The durable flight table. Becomes the `pending`/`running` rows of the machine rather than a parallel table. `reconcileBrowserSurfaceLeasesAfterBoot` reads it and keeps working — it is being re-homed, not removed. | +| `run_generation` | `db.ts:1156`, `controller.ts:1046` | **absorb → replaced by `owner_epoch`** | A per-instance monotonic fencing token, correct in kind but reinvented and *in-process* (`runGenerations` Map, cleared only by test reset). The owner epoch is the same idea already durable in `controller_identity`. One fence, not two. Note: the run-generation column shipped to SQLite before Postgres once — the exact dual-backend gap this design forbids. | +| `runWatchdogSettlements` | `controller.ts:1031` | **derive** | Watchdog fires ⇒ attempt an `abandoned` transition. If the CAS loses, the run already terminalized. Removes the timer/completion race entirely. | +| `needsHumanAttention` | `controller.ts:1055` | **delete (not absorb)** | In-memory `Set`, lost on restart, and **not a run state** — it is a per-*connection* automation-suppression policy. Absorbing it would smuggle scheduling policy into the machine, violating D4. It moves to the scheduler's own policy input, derived from terminal runs whose reason is an unresolved interaction. | +| `cooling_off` / `blocked` | `scheduler-backoff.ts:252` | **delete from run state** | Already not run state — a `recommendedHealthState` derived from a failure streak. Named here to record it as correctly-placed policy, and to forbid it moving in. | +| `scheduler_dispatch_wedged` | `scheduler.ts:197-215` | **derive** | A synthetic `status:"failed"` `RunRecord` fabricated when the pre-launch gate misses its liveness ceiling. Under the machine this is a real `failed` transition with `terminal_reason='scheduler_dispatch_wedged'`, not a fabricated record. The word "wedged" survives as a reason, not a shadow state. | +| `run_history.status` | `db.ts:1365` | **absorb** | The durable projection. Gains `owner_epoch` (below). | +| `scheduler_managed` | `db.ts:1379` | **keep, out of scope** | Provenance, not lifecycle. Marks which writer touched the row; scheduler cadence readers filter on it. Orthogonal. | +| `manual_upload_artifacts.owner_epoch` | landed sibling | **precedent** | Not run state. Cited because it is the pattern this design generalizes: an epoch column that replaced a wall clock. | + +### F1 cluster completeness (D3's test) + +| Cluster | Expressible as illegal transition? | +|---|---| +| GroupMe 503 — dispatch probe vs. active run | **Yes.** Planner attempting any write while `state ∈ {pending, running}`. Forbidden transition F1 below. | +| Ingest 503 never retried (Aug recurrence) | **Yes**, partially. The batch-level retry is HTTP policy, but the *observable* defect — a run reporting `failed` while records were durably committed — is `running → failed` attempted by a non-owner. Forbidden F5. | +| YNAB stuck-run wedge (`activeRuns` leak) | **Yes.** A run with no live owner epoch stuck non-terminal; `abandoned` becomes reachable, so the 409 cannot be permanent. Forbidden F6. | +| Collector-runner drain boundary race | **Yes.** Two clock reads with opposite boundary semantics; under CAS the second read cannot act on the first read's stale premise. Forbidden F3. | +| Run-generation fencing / stale controller write | **Yes.** Precisely the epoch CAS. Forbidden F2. | +| Controller-restart misclassification (`failed` vs `abandoned`) | **Yes.** `running → failed` by the boot path is forbidden; only `running → abandoned` is legal there. Forbidden F4. | +| **Maintenance-sweep shared-deadline starvation (×3)** | **NO — reported as a finding.** | + +**The cluster that does not fit.** The maintenance-sweep starvation family +(2026-08-01 ×2, 2026-08-03) is not a run-lifecycle defect. No illegal run +transition occurs: every run in those incidents was in a legal state the whole +time. The defect is that a *shared budget* was divided unfairly across +page-mates, so some connections never got serviced. Forcing it into this table +would require modelling scheduler fairness as run state — which is exactly the +policy-in-the-machine error D4 forbids. Recording it as out of scope is the +honest answer, and it belongs to the sweep-fairness work already in flight on +this very branch. The state set is not missing a state on its account. + +## (b) The legal transition table + +**Single writer per transition.** "Executor" means the one owner module this +design mandates; today its responsibilities are split across `runtime/index.ts`, +`runtime/controller.ts`, and `runtime/scheduler/run-executor.ts`. + +| # | Transition | Precondition | Single writer | +|---|---|---|---| +| T1 | `∅ → pending` | No non-terminal run for this `connector_instance_id` | Executor (admission) | +| T2 | `pending → running` | Caller holds the current owner epoch; `run.started` emitted | Executor | +| T3 | `running → awaiting_interaction` | Connector emitted `run.interaction_required` / `run.assistance_requested` | Executor | +| T4 | `awaiting_interaction → running` | Interaction reached a terminal interaction event | Executor | +| T5 | `running → cancel_requested` | Owner requested cancel (`run.cancel_requested`) | Executor, on owner intent | +| T6 | `running → succeeded` | Connector reported DONE; terminal commit gate passed | Executor | +| T7 | `running → failed` | Observed failure attributable to this run | Executor | +| T8 | `running → surface_failed` | Browser surface failed pre-launch | Executor | +| T9 | `cancel_requested → cancelled` | Executor observed the request and stopped | Executor | +| T10 | `{pending, running, awaiting_interaction, cancel_requested} → abandoned` | Run's `owner_epoch` is neither the actor's nor the newest boot epoch | **Boot adjudicator** (the sole exception to "executor only"; see (f)) | +| T11 | `awaiting_interaction → abandoned` | As T10, with reason `controller_terminated_while_awaiting_owner_interaction` | Boot adjudicator | + +Terminal states have no outgoing transitions. That is the whole point of the +set being closed. + +### Forbidden transitions, each naming the incident it prevents + +| # | Forbidden | Incident prevented | +|---|---|---| +| F1 | Any run-state write by the **planner/scheduler** | **GroupMe 503.** `dispatchIfDue` probed `getForwardEvidenceDebt`'s reconcile write against an instance with a run in flight, contending on the per-instance mutex and turning committed batches into `connector_instance_busy` failures. Fixed at `scheduler.ts:636` by a guard whose own comment says it mirrors "the guard `executeRun` already applies one step later" — one rule stated twice. Under F1 the planner has no write path to guard. | +| F2 | Any transition whose actor's epoch ≠ the run's `owner_epoch` | **Stale-controller writes / run-generation fencing.** A predecessor that resumes after a successor took over must lose *at the database*. Today `run_history` has no epoch column, so this is enforced only in process memory (`controller.ts:1046`). | +| F3 | Any transition whose observed `state` ≠ the CAS expected state | **Collector-runner drain boundary race.** Two clock reads with opposite boundary semantics (`<=` on claim, `>` on `nextRetryTime`) let a deadline landing between them produce a false "empty" exit. A CAS cannot act on a premise that changed under it. | +| F4 | `{pending, running, …} → failed` by the boot/adjudication path | **Controller-restart misclassification.** 134 production runs recorded `run.failed`/`controller_restarted`; 55 had staged a cursor and 34 had durably ingested a batch. Interruption is not observed failure. Only T10/T11 are legal there. | +| F5 | A second terminal transition on an already-terminal run | **Double-terminal / ingest-503 misreporting.** The existing `AND status='running'` fence (`run-history-writer.ts:355`) already provides this for `run_history`; F5 makes it a property of the machine rather than of each writer remembering to add it. | +| F6 | A run remaining non-terminal with no live owner epoch | **YNAB stuck-run wedge.** A hung subprocess left an `activeRuns` entry forever, 409-ing every future manual run until restart. `abandoned` must always be reachable, so "wedged forever" is unrepresentable. | +| F7 | Any transition out of a terminal state | Terminal means terminal. Makes `records_emitted` revision-after-terminal — explicitly forbidden by the landed change — structurally impossible. | +| F8 | Any write to run state not routed through the owner module | **D1.** A `transitionRun()` helper that five modules import is today's distributed writes wearing a uniform. Raw `INSERT`s into `spine_events` (`controller-boot.ts:526`, `:597`) are the current instance of this. | + +### Three live defects the table would have prevented + +Found while auditing writers. Each is a real, current divergence — reported as +findings, not fixed here. + +1. **An unfenced upsert can overwrite a terminal status.** + `queries/controller/insert-run-history.sql:40` and its PostgreSQL twin + `stores/scheduler-store.ts:1018` are `ON CONFLICT (run_id, + connector_instance_id) DO UPDATE SET status = excluded.status` with **no + `status = 'running'` fence**. Every other status writer has one + (`run-history-writer.ts:355`, `:406`; `controller-boot.ts:670`, `:791`, + `:859`). The scheduler's `appendRunHistory` normally runs *after* the generic + writer has already finalized the row, so a scheduler retry can revise an + already-terminal outcome. This is forbidden transition **F5** and **F7**, + both violated by one statement. +2. **The PostgreSQL drift repair fences on `run_id` alone.** + `controller-boot.ts:829` joins `WHERE h.run_id = t.run_id AND h.status = + 'running' AND h.connector_instance_id IS NOT NULL` — an `IS NOT NULL` check, + not an equality. Its SQLite twin at `:788-790` correctly fences + `AND connector_instance_id = ?`. The codebase documents in several places + that `run_id` is not unique across connections, which is why every other + writer fences on the pair. A dual-backend asymmetry in the repair path + itself — cluster 4's defect class, inside the adjudication code. +3. **`run_generation` is written from two incompatible sources.** + `controller.ts:3139-3187` derives it from a monotonic per-instance counter; + `scheduler/run-executor.ts:819` sets it from `attempt`, a retry counter. + One column, two meanings. Absorbing the fence into `owner_epoch` (M5) + retires the ambiguity rather than arbitrating it. + +4. **Two `createReferenceSchedulerManager` definitions exist.** One is inline + at `server/index.ts:8828` and is what production calls (`:8378`). The other + is exported from `server/scheduler-manager-factory.ts:374`, carries its own + copy of all four dispatch probes — including the same + `reconcileDirtyConnectorSummaryEvidence` write at `:558` — and is imported + by **nothing outside tests**; the two importing tests take only + `createRunManagedConnectorViaController`. A parallel copy of the exact + subsystem this design governs, whose divergence no test can see. M2 must + pick one before cutting writers over. This is D5's masking-pair hazard + sitting in the tree already. + +A fifth observation, not a defect: **`drainActiveRuns` has no production +caller.** All 116 call sites are test teardown. Its own doc comments +(`controller.ts:987`, `server/index.ts:8567`) still describe it as the +graceful-shutdown path and are stale relative to `server/index.ts:9670-9691`. +See the migration section for the full verdict. + +## (c) The CAS + owner-epoch schema + +### Schema change + +`run_history` gains one column on both backends, NULL-tolerant so existing rows +migrate without a backfill: + +- SQLite (`server/db.ts`): `owner_epoch TEXT` via the existing + `addColumnIfMissing` helper — the same mechanism `run_generation` used at + `db.ts:6171`. +- PostgreSQL (`server/postgres-storage.ts`): + `ALTER TABLE run_history ADD COLUMN IF NOT EXISTS owner_epoch TEXT;` — the + same shape used for `controller_active_runs.run_generation` at + `postgres-storage.ts:1960`. + +Verified absent today: `db.ts:1358-1395` and `postgres-storage.ts:2169-2197` +list every column, and neither has an epoch. The epoch exists only inside the +spine event's `data_json` (`runtime/index.ts:2745-2749`), which no `UPDATE` can +fence on cheaply. **Both backends in the same change** — `run_generation` +shipped to SQLite without Postgres once and was caught only as a deploy +blocker; the landed owner-epoch change repeats the warning. This design treats +a single-backend schema change as a defect by construction. + +The epoch value is the boot epoch already stashed by +`emitControllerBootedAndStashEpoch` (`controller-boot.ts:202-231`), whose +identity half is durable in `controller_identity` (`controller-boot.ts:96-151`, +one row `id='singleton'`, live in production and verified by reading the file). + +### The predicate + +Every durable transition is one statement. No read-then-write. + +**SQLite** (`better-sqlite3`, synchronous; decide by `.changes`): + +```sql +UPDATE run_history + SET status = ?, -- new state + completed_at = ?, -- terminal transitions only + terminal_reason = ? + WHERE run_id = ? + AND connector_instance_id = ? -- run_id alone is not unique + AND status = ? -- expected state + AND (owner_epoch = ? OR owner_epoch IS NULL); +``` + +**PostgreSQL** (decide by `rowCount`): + +```sql +UPDATE run_history + SET status = $1, + completed_at = $2, + terminal_reason = $3 + WHERE run_id = $4 + AND connector_instance_id = $5 + AND status = $6 + AND (owner_epoch = $7 OR owner_epoch IS NULL); +``` + +Four properties of this predicate are load-bearing: + +1. **`changes`/`rowCount` = 0 means the transition was refused**, and the caller + must treat that as an ordinary outcome — someone else already moved the run. + It is never retried blindly and never escalated to the owner. +2. **`connector_instance_id` is part of the fence, not decoration.** `run_id` + alone is not unique across connections; both `run-history-writer.ts:355` and + `controller-boot.ts:670` already fence on the pair, and this preserves that. +3. **The NULL arm is spelled out explicitly, not written `IS DISTINCT FROM`.** + The landed sibling change was bitten by exactly this: on PostgreSQL, + `owner_epoch IS DISTINCT FROM NULL` reduces to `owner_epoch IS NOT NULL` and + would spare precisely the legacy rows that most need claiming. Writing + `(owner_epoch = $7 OR owner_epoch IS NULL)` is identical on both backends + and cannot silently reduce. +4. **The NULL arm is `OR`, not `AND`.** A legacy row written before the column + existed has no claimant, so any epoch may adjudicate it. A row *with* a + different epoch may not. + +The adjudication transition (T10/T11) inverts arm 4 — it must match rows whose +epoch is *not* the actor's — and additionally excludes the newest boot epoch, so +live work is never adjudicated: + +```sql + AND (owner_epoch IS NULL OR owner_epoch <> :mine) + AND (owner_epoch IS NULL OR owner_epoch <> :newest_boot_epoch) +``` + +This is the predicate the landed change already proved against production: its +dry run reported 123 before the newest-epoch exclusion and 121 after, the two +extras being runs a live container had started ninety seconds earlier. + +### Why CAS and not a lock + +An advisory lock answers "may I proceed?" at a moment; a CAS answers "was the +world still as I assumed when I wrote?" — which is the actual question after an +`await`. This repo already has at least five independently-invented admission +mechanisms (F1's count), and adding a sixth lock would extend that list. The CAS +adds no new mechanism: it is a `WHERE` clause on writes that already happen. + +## (d) Property-test skeletons + +Listed in `tasks.md` §5 with file names. Each is `.todo` or asserts against the +not-yet-existing owner module, so **none can pass before implementation**. That +is the requirement: this program has already shipped a conformance test that +asserted behavior it never exercised, and a guard that cannot fail certifies a +regression as safe. + +Each skeleton names its property, its generator, and its invariant, and each +maps to a row in the forbidden table above. Dual-backend skeletons run against +SQLite and PostgreSQL from one body, because a Postgres-only divergence is +invisible to this suite by construction — that is a named defect class here +(6,792 tests stayed green while production could not paginate). + +## (e) The D4 split, stated precisely + +- The **planner** (`runtime/scheduler.ts`, `scheduler/dispatch-governor.ts`, + `scheduler-backoff.ts`) READS run state and emits **intents**. It writes no + run state, ever. +- The **executor** is the sole writer of transitions T1-T9, and the boot + adjudicator of T10-T11. + +**The split does not hold today.** A first pass suggested it nearly did — +`runtime/scheduler.ts` emits no spine event and contains no literal `UPDATE +run_history`. That reading was wrong, and the correction matters more than the +original claim. The scheduler writes run state four ways: + +| # | Write | Site | +|---|---|---| +| P1 | `INSERT INTO run_history` for skip and back-off records | `scheduler.ts:432`, `:445` (`recordAndNotify` → `schedulerStore.appendRunHistory`), called from `dispatchIfDue:655`, `:662`; SQL at `scheduler-store.ts:570` / `:995` | +| P2 | `scheduler_last_run_times` upsert | `scheduler.ts:477`; `scheduler-store.ts:1607` | +| P3 | In-process `activeRuns` set and five announcement-dedup maps | `scheduler.ts:547`, `:576`; `dispatch-governor.ts:559`, `:563`, `:588`, `:590`, `:593` | +| P4 | **A durable repair write inside the dispatch-eligibility probe** | `server/index.ts:9057` — `getForwardEvidenceDebt` calls `reconcileDirtyConnectorSummaryEvidence([instanceId])` *before* its read at `:9058`; that reconcile takes `withConnectorInstanceWrite` and issues `withPostgresTransaction({ lockConnectorInstanceId })` upserts (`connector-summary-evidence-engine.ts:1281`, `:1866`, `:2691`) | + +**P4 is the GroupMe 503 mechanism**, and it is the clearest violation of D4: +a read-only-looking eligibility probe that takes the per-instance write mutex. +So F1 must forbid *side-effecting reads*, not merely direct writes. A planner +that "only reads" but whose read reconciles is a writer. + +P1 is the second-order version of the same error: the scheduler writes rows +that feed back into `runtime.history`, which is the next tick's own decision +input. The Gmail identity self-poisoning loop is exactly this — scheduler- +generated `status:"skipped"` records read back as run outcomes. + +Under this design: P4 moves to the executor or becomes genuinely read-only. +P1's records stay writable by the planner but are not run states (see +`skipped`, above) and the machine never reads them. P2 is planner-owned cadence +state, unaffected. P3's `activeRuns` set is replaced by an epoch-fenced read of +the machine — today it is an in-process `Set` that dies with the process, so +the `dispatchIfDue` guard does not suppress a probe against a run started by a +different process or surviving a restart. + +**The policy guard.** The machine answers *is this transition legal*. It never +answers *which connector runs next*. Concretely: `cooling_off`, `blocked`, +backoff curves, fairness rotation, admission deadlines, and per-statement +budgets stay in the planner. "Runnable" means "no legal impediment to a T1"; it +does not mean "chosen." `needsHumanAttention` is deleted from the controller +for exactly this reason — it reads like run state and is actually automation +policy. + +## (f) The D6 formalization + +`scripts/repair/adjudicate-orphaned-runs.ts` and +`reconcileOrphanedRunsAtBoot` (`controller-boot.ts:344`) implement the same +adjudication with different scopes. The script is the owner-operated backlog +tool: dry-run by default, ignores `controller_id` (the field that was broken), +excludes the newest boot epoch, snapshots pre-images into an `aor_backup` table +inside the write transaction. The boot reconciler is the steady-state path: +filters on `controller_id`, runs before HTTP routes mount, aborts boot on error. + +Under this design the boot reconciler **is** transitions T10/T11 — the same +predicate, expressed once in the owner module. Three properties must be +preserved exactly, because this is a behavior-preserving move (D14): + +1. **Idempotency stays on `caused_by_event_id`,** via the + `spine_run_abandoned_cause_unique` partial index. Repeated passes emit + exactly one terminal event per orphan. The existing code catches *only* the + named constraint and never blanket-catches `23505` / `SQLITE_CONSTRAINT_UNIQUE`; + the owner module must keep that discipline. +2. **Newest-epoch exclusion stays,** and stays load-bearing. Without it, + adjudication declares live work abandoned and frees its resource for a + competing run — reintroducing the duplicate-execution hazard the fence + exists to prevent. +3. **`records_emitted` is never revised.** Records durably ingested before the + interruption stay committed. F7 makes this structural. + +The one real change: the raw `INSERT`s at `controller-boot.ts:526` and `:597` +that bypass `emitSpineEvent` — and therefore had to hand-write their own +`run_history` projection at `:657` and `:837` to avoid stranding rows at +`running` — route through the owner module instead. The event and its +projection already commit in one transaction; that stays. The bypass is the F8 +violation this design closes. + +The repair script remains a script. It is owner-operated, dry-run-first, +`--limit`-bounded, and deliberately ignores the ownership filter — properties +of an operational tool, not of a lifecycle transition. It should call the same +predicate rather than reimplement it, which is the whole benefit. + +## (g) The D5 migration plan + +Writers cut over atomically per subsystem, old paths deleted in the same change, +readers migrate after. Never let old and new writers coexist across a deploy +boundary — every "parallel systems" period in this repo produced a masking pair. + +| Tranche | Writers cut over | Deleted in the same change | +|---|---|---| +| M1 | Schema + owner module (no callers yet) | nothing — additive only | +| M2 | Terminal writes: `runtime/index.ts` ×6, `controller.ts` ×2, `terminal-run-commit-store.ts`, `local-device-terminal-collection.ts` | the direct `emitSpineEvent` terminal calls at those sites | +| M3 | Admission + finalize: `activeRuns`, `settledRunIds`, `runWatchdogSettlements` | `settledRunIds`; hand-rolled CAS at `controller.ts:3249` | +| M4 | Boot adjudication (T10/T11) | raw `INSERT`s at `controller-boot.ts:526`, `:597` | +| M5 | Fence unification | `run_generation` column + `runGenerations` map | +| M6 | Readers: the four terminal-set declarations collapse to one | `TERMINAL_RUN_EVENT_TYPES`, `SPINE_TERMINAL_EVENT_TYPES_SQL` | + +M6 is where the `run.abandoned` omission defect is repaired — as a consequence +of single declaration, not as a separate fix. + +### The `drainActiveRuns` collision — verdict: **the brief's premise no longer holds** + +The brief asks me to confirm the Sidekiq *quiet* vs *drain* split and say which +references migrate. Measured on this branch, `39d19704a`: + +- **139 total references** — the brief's number reproduces exactly. +- **3** are in `openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/**` + (prose in the landed sibling change). +- **127** are in `reference-implementation/test/**`, of which **116** are call + expressions — overwhelmingly `await controller.drainActiveRuns(1000)` as + test teardown. +- **9** are in source: `runtime/controller.ts` ×7 (one definition at `:4144`, + one interface member at `:786`, one export at `:4394`, four comments) and + `server/index.ts` ×2 — **both of which are comments, not call sites**. + +**There is no shutdown call site left to delete. It is already gone**, removed +by the landed sibling change. `server/index.ts:8570` now reads "The shutdown +path deliberately does NOT drain," and `:9686-9689` records that +"`drainActiveRuns` itself stays on the controller: it means 'await in-flight +runs' … Sidekiq draws the same line between *quiet* and *drain*." + +So: **the split is correct and is already implemented.** My verdict confirms the +brief's reading of the semantics and corrects its state — this is not open work. +The residual figures also differ slightly from the sibling change's prose, which +says "137 references … 136 remain." Today's count is 139 including its own 3 +lines of prose, i.e. 136 outside that change plus 3 inside it. The numbers are +consistent; the sibling was counting before it wrote about itself. + +Which references migrate under this design: **none of the 116 test calls.** +`drainActiveRuns` means "await in-flight run promises" — a runtime affordance +over `activeRunPromises` (JS promises, ruled "keep, out of scope" above), not a +run-state mutation. It never transitions a run. It stays exactly where it is +and keeps its name. + +## Risks / Trade-offs + +- [Design authored against a moving branch] -> A concurrent agent owns the + scheduler-conformance helper and the store drivers. The single-writer + inventory is a point-in-time read of `39d19704a` and must be re-verified + before M2 begins. Named as a residual risk, not mitigated here. +- [`pending` is a new state with no existing projection] -> It is the only + addition, and it exists so admission has something to CAS against. If M1 + measurement shows admission can safely CAS `∅ → running` directly, `pending` + should be dropped rather than kept for symmetry — a state that no transition + needs is a second truth in waiting. +- [Absorbing `controller_active_runs` touches browser-surface leases] -> + `reconcileBrowserSurfaceLeasesAfterBoot` reads that table to decide which + leases are still held. Absorption must preserve that read or re-home it in + the same tranche; it is explicitly not a deletion. +- [Deleting `needsHumanAttention` changes automation behavior] -> It is + in-memory and already lost on every restart, so its current behavior is + "suppress until restart." Moving it to a derived scheduler policy input makes + it durable, which is a *behavior change* and must be measured on the canary + rather than assumed neutral. This is the one item in the audit that is not + behavior-preserving on its face. +- [CAS refusals become invisible] -> A transition that returns 0 rows is a + normal outcome, which means a bug that refuses *every* transition looks like + a quiet system. The canary metrics in `tasks.md` §6 therefore count + successful transitions, not just absence of anomalies. diff --git a/openspec/changes/own-run-lifecycle-state-machine/proposal.md b/openspec/changes/own-run-lifecycle-state-machine/proposal.md new file mode 100644 index 000000000..826d95672 --- /dev/null +++ b/openspec/changes/own-run-lifecycle-state-machine/proposal.md @@ -0,0 +1,142 @@ +## Why + +Run state has no owner. It is spread across an append-only spine, a durable +`run_history` projection, a durable `controller_active_runs` flight table, and +at least five in-process maps in `runtime/controller.ts` — with no single +component that decides which transitions are legal. + +The consequence is measured, not theorized. Terminal events are emitted from at +least five modules: `runtime/index.ts` (lines 4071, 5117, 5172, 5367, 5414, +5563), `runtime/controller.ts` (3707, 4083), `server/stores/terminal-run-commit-store.ts:117`, +`operations/local-device-terminal-collection.ts:211`, and — bypassing +`emitSpineEvent` entirely with raw `INSERT`s — `lib/controller-boot.ts`. Each +site re-derives for itself what a legal terminal write is. + +Two facts found while writing this proposal show the cost directly: + +1. **The terminal set disagrees with itself.** `check-run-terminal.sql:15` and + `get-run-terminal-event.sql:13` list five terminal events including + `run.abandoned`. `connector-summary-read-model.ts:1253` + (`TERMINAL_RUN_EVENT_TYPES`) and `db.ts:5437` + (`SPINE_TERMINAL_EVENT_TYPES_SQL`) list four and omit `run.abandoned` — and + `db.ts`'s own comment says the two are "kept in exact sync." They are in + sync with each other and out of sync with the spine. An abandoned run is + therefore invisible to the connector-summary fold and to the + `connector_instance_id` backfill that fold depends on. The change that made + `abandoned` a first-class terminal state could not update these because + nothing declares the set once. + +2. **The epoch cannot fence a durable write.** `run.started` stamps + `boot_epoch`, `controller_id`, and `seq` into the spine event's `data_json` + (`runtime/index.ts:2745-2749`), but `run_history` has no epoch column on + either backend (`db.ts:1358`, `postgres-storage.ts:2169`). Every durable + run-state UPDATE is fenced on `AND status = 'running'` alone + (`run-history-writer.ts:355`, `:406`; `controller-boot.ts:670`). That fence + stops a double-terminal write but cannot stop a *stale epoch's* write: a + predecessor container that comes back from a pause still sees `running` and + still wins. Fencing exists in-process instead, as a hand-rolled + compare-and-swap on a JavaScript map (`controller.ts:3249`) plus a + `run_generation` counter (`controller.ts:1046`) — correct, and invisible to + the database that actually arbitrates. + +The historical incidents are all one shape. The GroupMe 503 was two components +mutating run state through the same per-instance mutex, fixed by a guard in the +scheduler (`scheduler.ts:636`) that mirrors "the guard `executeRun` already +applies one step later" — the same rule stated twice because no component owns +it. The collector-runner drain race was two clock reads with opposite boundary +semantics. The YNAB wedge was an `activeRuns` entry that outlived its run and +409-ed the instance until restart. + +This change writes down the machine: a closed state set, a transition table +whose forbidden entries name the incident each one prevents, and a +compare-and-swap predicate fenced by the owner epoch that already ships +durably in `controller_identity` (`lib/controller-boot.ts:96-151`). + +## What Changes + +This change is **design and tests-first only**. It authors the contract and +failing property-test skeletons; it writes no implementation. That is +deliberate — the implementation is single-threaded and starts from a settled +design. + +- Define the **closed run state set** and require it be declared exactly once, + so a terminal-set divergence like `TERMINAL_RUN_EVENT_TYPES` becomes a + compile-time impossibility rather than a comment asking two constants to + agree. +- Rule on every side-state that lives beside run state today: `activeRuns`, + `controller_active_runs`, `run_generation`, `settledRunIds`, + `needsHumanAttention`, the scheduler's `cooling_off` and + `scheduler_dispatch_wedged` markers — each **absorbed**, **derived**, or + **deleted**, with the verdict recorded in `design.md`. +- Define the **legal transition table**: each transition's precondition and its + single writer. Define the forbidden transitions, each annotated with the + historical incident it prevents. +- Require every durable transition to be a **compare-and-swap fenced by owner + epoch** — `WHERE state = AND owner_epoch = ` — and specify + the predicate for SQLite and PostgreSQL together, since this repo runs both + and a Postgres-only design has already shipped silent divergence here once. +- Add an `owner_epoch` column to `run_history` on both backends, NULL-tolerant, + so the fence has a column to read. It does not exist today. +- State the **D4 split**: the planner reads the machine and emits intents; only + the executor transitions. Scheduling *policy* stays out of the machine — + "runnable" is not "chosen to run next." +- Formalize **successor adjudication** as a legal transition executed at boot, + replacing the sibling-script framing, with observable behavior unchanged. +- Author **property-test skeletons**, one per historical incident cluster, each + failing or `.todo` and marked as a skeleton. A skeleton that passes without + implementation is the hollow-test defect this program exists to kill. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `reference-implementation-architecture`: Define the closed run state set and + its single declaration, the legal-transition table and its single-writer + rule, the epoch-fenced compare-and-swap requirement and its dual-backend + predicate, the planner/executor split, and successor adjudication as a + lifecycle transition. + +## Impact + +- Design-only in this change. No runtime file is modified. +- New failing property-test skeletons under `reference-implementation/test/`, + named `run-lifecycle-*.property.test.ts`, excluded from the green gate until + the implementation change lands. +- The implementation this design governs will touch + `reference-implementation/runtime/controller.ts`, + `runtime/index.ts`, `runtime/scheduler.ts`, + `server/stores/run-history-writer.ts`, `lib/controller-boot.ts`, + `server/db.ts`, and `server/postgres-storage.ts`. +- Additive `run_history.owner_epoch` on SQLite and PostgreSQL, NULL-tolerant, + so existing databases migrate without a backfill. + +## Non-Goals + +- **Any implementation.** No writer is cut over here, no state is renamed in + code, no migration runs. +- **Scheduling policy.** Fairness, throttling, backoff curves, and dispatch + ordering stay where they are. The machine answers whether a transition is + legal, never which connector should run next. +- **Coverage-ledger and health work.** Steps 2 and 3 of this program depend on + this machine and are sequenced after it (D17). +- **Auto-resume or auto-retry of interrupted runs.** Unchanged from the landed + owner-epoch change: adjudication is silent and the normal schedule picks the + work up. +- **Committing staged cursors under interruption.** Still gated on the + bounded-`DETAIL_COVERAGE` evidence that production does not yet emit. + +## Residual risks + +- The transition table is authored against the code as read on + `fix/sweep-fairness-and-transformer-bounds` at `39d19704a`. A concurrent + agent owns the scheduler-conformance helper and the store drivers; if those + land new run-state writers, the single-writer inventory needs a re-read + before implementation begins. +- One historical cluster — the maintenance-sweep shared-deadline starvation + family — is **not** expressible as an illegal run transition. It is a + fairness defect in a different subsystem, recorded in `design.md` as a + finding rather than forced into the table. diff --git a/openspec/changes/own-run-lifecycle-state-machine/specs/reference-implementation-architecture/spec.md b/openspec/changes/own-run-lifecycle-state-machine/specs/reference-implementation-architecture/spec.md new file mode 100644 index 000000000..cbd847d95 --- /dev/null +++ b/openspec/changes/own-run-lifecycle-state-machine/specs/reference-implementation-architecture/spec.md @@ -0,0 +1,213 @@ +## ADDED Requirements + +### Requirement: Run lifecycle SHALL be a closed state set declared exactly once + +The reference implementation SHALL define the run lifecycle as a closed set of +states: `pending`, `running`, `awaiting_interaction`, `cancel_requested`, +`succeeded`, `failed`, `surface_failed`, `cancelled`, and `abandoned`. The last +five are terminal. + +The state set and its terminal subset SHALL be declared in exactly one place, +and every consumer — projection, fold, query, backfill, and health read — SHALL +derive from that declaration. No component SHALL restate the terminal set as +its own literal collection. + +A run state SHALL NOT be represented by a boolean or marker stored beside the +state. Any condition that describes where a run is in its lifecycle SHALL be a +state in this set or SHALL be derived from one. + +A dispatch outcome that never started a run SHALL NOT be a run state. The +scheduler MAY record such an attempt for cadence purposes, but the lifecycle +SHALL NOT read it as the outcome of a run. + +#### Scenario: The terminal set has one declaration + +- **WHEN** any component tests whether a run event is terminal +- **THEN** it SHALL consult the single declared terminal set +- **AND** adding a terminal state SHALL require exactly one edit + +#### Scenario: An abandoned run is visible to every terminal reader + +- **WHEN** a run reaches `abandoned` +- **THEN** every consumer of the terminal set SHALL observe that run as terminal +- **AND** no consumer SHALL classify it as non-terminal because its own copy of + the terminal set omits `abandoned` + +#### Scenario: Both backends agree on the terminal set + +- **WHEN** the same run is read through the SQLite path and the PostgreSQL path +- **THEN** both SHALL report the same terminal classification + +### Requirement: Every run-state transition SHALL be an epoch-fenced compare-and-swap + +Every durable run-state transition SHALL be expressed as a single conditional +statement predicated on both the expected current state and the acting owner +epoch. A transition SHALL NOT read the current state and then write it in a +separate statement. + +A transition whose predicate matches no row SHALL be treated as refused. A +refusal SHALL be an ordinary outcome: it SHALL NOT be retried by re-reading and +re-writing, SHALL NOT raise owner attention, and SHALL NOT be recorded as a +failure of the run. + +A run SHALL carry the identity of the owner epoch entitled to transition it, +recorded durably in the same write that admits the run. A process whose epoch +is not the run's owner epoch SHALL fail to transition that run **at the +database**, not by an in-process check. + +The predicate SHALL be expressed so that it is identical in effect on SQLite +and PostgreSQL. A null owner epoch SHALL be treated as unclaimed. The null arm +SHALL be spelled out explicitly and SHALL NOT be written in a form that reduces +to excluding null rows on either backend. + +Run identity for fencing purposes SHALL be the pair of run identifier and +connector instance identifier, because a run identifier alone is not unique +across connections. + +#### Scenario: A stale epoch's write is refused by the database + +- **WHEN** a process whose owner epoch is not the run's owner epoch attempts any + transition on that run +- **THEN** the statement SHALL match no row +- **AND** the run's state SHALL be unchanged + +#### Scenario: A transition from an unexpected state is refused + +- **WHEN** a transition expects a run to be in one state and the run is in + another +- **THEN** the statement SHALL match no row +- **AND** the caller SHALL treat the refusal as an ordinary outcome + +#### Scenario: A legacy run with no recorded owner epoch is claimable + +- **WHEN** a transition targets a run whose owner epoch is null +- **THEN** the transition SHALL be permitted +- **AND** the predicate SHALL NOT spare that run on either backend + +#### Scenario: Two connections sharing a run identifier are fenced apart + +- **WHEN** two runs on different connector instances share a run identifier +- **THEN** a transition on one SHALL NOT match the other + +### Requirement: Exactly one component SHALL write run state + +All run-state transitions SHALL pass through a single owner module. No other +component SHALL emit a run lifecycle event, update the durable run projection, +or mutate an in-process structure that other components read as run state. + +A shared helper that many components import SHALL NOT satisfy this requirement. +The requirement is one code path that owns the mutations, not one function that +many code paths call. + +A component that writes run state SHALL NOT also decide which work runs next. + +#### Scenario: A second writer cannot terminalize a run + +- **WHEN** a component other than the owner module attempts to record a run's + terminal state +- **THEN** the attempt SHALL be rejected +- **AND** the run's terminal state SHALL remain whatever the owner module wrote + +#### Scenario: A terminal run cannot be revised + +- **WHEN** any component attempts a transition on a run already in a terminal + state +- **THEN** the transition SHALL be refused +- **AND** the run's recorded record count SHALL NOT be revised + +### Requirement: The planner SHALL read run state and SHALL NOT write it + +The component that decides which connector to run SHALL read the run lifecycle +and emit intents. It SHALL NOT transition run state. + +A planner read SHALL NOT produce a durable side effect. An eligibility probe +that reconciles, repairs, or otherwise writes on the read path SHALL count as +writing run state, regardless of the name of the function that performs it. + +The lifecycle SHALL NOT encode scheduling policy. Backoff curves, cooldown +windows, fairness rotation, admission budgets, and automation suppression SHALL +remain the planner's. The lifecycle SHALL answer only whether a transition is +legal. + +A run being eligible for a transition SHALL NOT mean the run has been chosen to +execute. + +#### Scenario: The planner's eligibility probe writes nothing + +- **WHEN** the planner evaluates whether a connector instance is due +- **THEN** that evaluation SHALL perform no durable write +- **AND** it SHALL NOT acquire a write lock on the connector instance + +#### Scenario: Scheduling policy stays out of the lifecycle + +- **WHEN** a backoff, cooldown, or fairness rule changes +- **THEN** the run lifecycle's states and transitions SHALL be unchanged + +### Requirement: Illegal transitions SHALL be refused rather than reconciled later + +The reference implementation SHALL define which transitions are legal, and +SHALL refuse every transition outside that definition at the point of the +write. + +The following SHALL be illegal: a transition attempted by any component other +than the transition's declared writer; a transition whose actor epoch is not +the run's owner epoch; a second terminal transition on a terminal run; any +transition out of a terminal state; and the recording of an interrupted run as +an observed failure. + +A run SHALL NOT be able to remain non-terminal indefinitely with no live owner +epoch. A terminal state SHALL always be reachable for such a run, so that a run +whose owner has died cannot permanently block its connector instance. + +#### Scenario: An interrupted run is not recorded as a failure + +- **WHEN** a run's owner process died without reporting an outcome +- **THEN** the run SHALL transition to `abandoned` +- **AND** no transition to `failed` SHALL be permitted for that reason + +#### Scenario: An ownerless run cannot block its connection forever + +- **WHEN** a run is non-terminal and no live epoch owns it +- **THEN** a terminal transition SHALL be reachable for that run +- **AND** its connector instance SHALL become able to admit a new run + +### Requirement: Successor adjudication SHALL be a lifecycle transition + +Marking a predecessor epoch's non-terminal runs as abandoned SHALL be a legal +transition of the run lifecycle, executed by the owner module at boot. It SHALL +NOT be a separate reconciliation path with its own copy of the state +vocabulary. + +The transition SHALL preserve the adjudication behavior already established: +it SHALL be idempotent per originating run-start event so repeated passes +produce exactly one terminal event; it SHALL NOT adjudicate any run belonging +to the newest boot epoch; it SHALL decide eligibility by epoch comparison +rather than by an age threshold; and it SHALL NOT revise the record count of a +run that ingested records before being interrupted. + +The terminal event and its durable projection SHALL be written in one +transaction, so a run cannot be terminal in the event log while its projection +still claims to be running. + +An owner-operated repair tool MAY apply this transition over a historical +backlog with different scoping, but SHALL use the same transition rather than +reimplementing it, and SHALL remain distinguishable in the recorded provenance +of the events it writes. + +#### Scenario: Adjudication at boot is the same transition as everywhere else + +- **WHEN** the owner module adjudicates a predecessor epoch's non-terminal run +- **THEN** it SHALL apply the same epoch-fenced transition used for every other + run-state change + +#### Scenario: The event and its projection commit together + +- **WHEN** adjudication terminalizes a run +- **THEN** the terminal event and the durable projection SHALL commit in one + transaction +- **AND** no run SHALL be observable as terminal in one and running in the other + +#### Scenario: Repeated adjudication is idempotent + +- **WHEN** adjudication runs twice over the same interrupted run +- **THEN** exactly one terminal event SHALL exist for that run diff --git a/openspec/changes/own-run-lifecycle-state-machine/tasks.md b/openspec/changes/own-run-lifecycle-state-machine/tasks.md new file mode 100644 index 000000000..d6da49b7c --- /dev/null +++ b/openspec/changes/own-run-lifecycle-state-machine/tasks.md @@ -0,0 +1,142 @@ +## 1. Design (this change) + +- [x] 1.1 Enumerate the closed state set, including `abandoned`, and rule + `skipped` out as a dispatch outcome rather than a run state. +- [x] 1.2 Audit every boolean and side-state living beside run state and record + an absorb / derive / delete verdict for each in `design.md`. +- [x] 1.3 Express every F1 incident cluster as a forbidden transition, and + report the one cluster that cannot be expressed as a finding. +- [x] 1.4 Write the legal-transition table with each transition's precondition + and single writer, and the forbidden table annotated with the incident + each entry prevents. +- [x] 1.5 Specify the compare-and-swap predicate for SQLite and PostgreSQL + together, including the explicit null arm. +- [x] 1.6 State the planner/executor split and its policy guard. +- [x] 1.7 Specify successor adjudication as a legal transition preserving + observable behavior. +- [x] 1.8 Write the writers-first migration plan and the `drainActiveRuns` + verdict. + +## 2. Findings raised by this design (not fixed here) + +- [ ] 2.1 Six terminal-set declarations disagree with `lib/spine.ts:1066`. Four + omit `run.abandoned` (`connector-summary-read-model.ts:1253`, + `db.ts:5437`, `postgres-storage.ts:2692`, `connector-summary-evidence-engine.ts:1599`); + two omit `run.browser_surface_failed` (`lib/postgres-spine.ts:570`, + `postgres-storage.ts:2336`). Closed by M6, not before. +- [ ] 2.2 `insert-run-history.sql:40` and `scheduler-store.ts:1018` upsert + `status = excluded.status` with no `status = 'running'` fence, so a + scheduler retry can overwrite a terminal status. +- [ ] 2.3 `controller-boot.ts:829` (PostgreSQL drift repair) matches on + `run_id` alone while its SQLite twin at `:788` fences on + `connector_instance_id`. +- [ ] 2.4 `run_generation` is written from a monotonic counter + (`controller.ts:3187`) and from a retry `attempt` + (`scheduler/run-executor.ts:819`) — one column, two meanings. +- [ ] 2.5 Two `createReferenceSchedulerManager` definitions exist + (`server/index.ts:8828` live, `scheduler-manager-factory.ts:374` + test-only). Resolve before M2. + +## 3. Schema (implementation, not this change) + +- [ ] 3.1 Add NULL-tolerant `run_history.owner_epoch TEXT` on SQLite via + `addColumnIfMissing`, mirroring the `run_generation` migration. +- [ ] 3.2 Add `run_history.owner_epoch TEXT` on PostgreSQL via + `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`, **in the same change** as 3.1. +- [ ] 3.3 Stamp `owner_epoch` in the same write that admits a run. +- [ ] 3.4 Add a dual-backend test proving the column exists and is written on + both, so a single-backend migration cannot ship. + +## 4. The owner module (implementation, not this change) + +- [ ] 4.1 Declare the state set and terminal subset once; every consumer derives. +- [ ] 4.2 Implement the transition table T1-T11 as epoch-fenced compare-and-swap. +- [ ] 4.3 Return refusal as an ordinary outcome; never retry blindly. +- [ ] 4.4 Route boot adjudication (T10/T11) through the same predicate. +- [ ] 4.5 Cut writers over per the M1-M6 tranches, deleting each old path in the + same change as its replacement. + +## 5. Property-test skeletons (this change — must fail or `.todo`) + +Each skeleton is authored `.todo` or asserting against the not-yet-existing +owner module, and is marked in-file as a skeleton. **A skeleton that passes +before implementation is the defect this program exists to kill.** Each names +its property, generator, and invariant, and maps to a forbidden transition. + +- [x] 5.1 `run-lifecycle-transition-legality.property.test.ts` — **F7, F5.** + Generator: random sequences of transition attempts drawn from the full + state × transition cross-product. Invariant: the observed state after any + sequence is reachable from the initial state by legal transitions only, + and a terminal state is never left. +- [x] 5.2 `run-lifecycle-epoch-fencing.property.test.ts` — **F2.** Generator: + interleavings of two actors with distinct epochs attempting transitions on + one run. Invariant: only the run's owner epoch ever changes state; the + stale actor's statement matches zero rows. Runs on both backends. +- [x] 5.3 `run-lifecycle-single-terminal.property.test.ts` — **F5, F4.** + Generator: concurrent terminal attempts of differing kinds, including a + boot adjudicator racing an executor. Invariant: exactly one terminal state + results; an interrupted run terminalizes `abandoned`, never `failed`. +- [x] 5.4 `run-lifecycle-planner-writes-nothing.property.test.ts` — **F1.** + Generator: planner eligibility evaluations against instances in every + state, including one with a run in flight. Invariant: zero durable writes + and zero connector-instance write-lock acquisitions occur during a planner + read. This is the GroupMe 503 as a property. +- [x] 5.5 `run-lifecycle-no-permanent-wedge.property.test.ts` — **F6.** + Generator: runs abandoned mid-flight by an epoch that never returns. + Invariant: a terminal state is always reachable, and the connector + instance can admit a new run afterwards. This is the YNAB wedge. +- [x] 5.6 `run-lifecycle-cas-no-stale-premise.property.test.ts` — **F3.** + Generator: transitions whose observed state is mutated by a competing + actor between the caller's read and its write. Invariant: no transition + commits on a premise that changed under it. This is the drain/clock race. +- [x] 5.7 `run-lifecycle-terminal-set-agreement.property.test.ts` — **the + declaration requirement.** Generator: every terminal state, read through + every consumer of the terminal set. Invariant: all consumers agree, and + both backends agree. Fails today on the six divergent declarations. +- [x] 5.8 `run-lifecycle-adjudication-idempotence.property.test.ts` — **T10/T11.** + Generator: repeated adjudication passes over overlapping orphan sets, + including a run in the newest boot epoch. Invariant: exactly one terminal + event per orphan; newest-epoch runs are never adjudicated; record counts + are never revised. + +- [ ] 5.9 Register 5.1-5.8 in the suite as expected-failing until the owner + module lands, so their red state is deliberate and visible rather than + an unexplained broken build. + +## 6. Pre-registered canary metrics (D15) + +Registered **before** any Step 1 deploy. Post-hoc criteria are how "green" +claims died in this program. + +- [ ] 6.1 **Restarts.** 20 consecutive controller restarts in a replaced + container with zero adjudication anomalies: every `run.started` without a + terminal event belongs to the newest boot epoch, and no run holds two + terminal events. Measured by query, not by log reading. +- [ ] 6.2 **Race-class property tests green.** All of 5.1-5.8 pass on both + SQLite and PostgreSQL. Any skipped test counts as a failure. +- [ ] 6.3 **Transition throughput is non-zero.** Count of successful transitions + per state pair over the canary window is greater than zero for T1, T2, and + at least one terminal transition. A machine that refuses everything would + otherwise satisfy every "zero anomalies" metric. +- [ ] 6.4 **Refusal rate is bounded.** Refused transitions stay under 1% of + attempts outside adjudication. A rising refusal rate means the expected + state is being computed wrong somewhere. +- [ ] 6.5 **Behavior preservation (D14).** Per-connector run outcomes over the + canary window match the pre-change baseline in kind and count. Terminal + state distribution shifts only by the `failed` → `abandoned` + reclassification the design intends. +- [ ] 6.6 **No permanent wedge.** Zero connector instances refusing a new run + with `active_run_exists` for longer than one scheduling interval. + Baseline: the UAT instance previously held 7 of 8 `running` rows as + zombies up to two days old. +- [ ] 6.7 **Dual-backend parity.** 6.1-6.6 measured on both backends. A + PostgreSQL-only or SQLite-only pass is a failure, not a partial success. + +## 7. Validation + +- [ ] 7.1 `openspec validate own-run-lifecycle-state-machine --strict`. +- [ ] 7.2 `openspec validate --all --strict` against the recorded baseline of 11 + pre-existing failures, so this change's contribution is distinguishable. +- [ ] 7.3 Re-verify the single-writer inventory against the branch head before + M2 begins; it is a point-in-time read and concurrent agents own adjacent + files. diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/design.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/design.md new file mode 100644 index 000000000..9ff2cb9e6 --- /dev/null +++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/design.md @@ -0,0 +1,315 @@ +## Context + +The sibling change `adjudicate-interrupted-runs-by-owner-epoch` settled who +writes an interrupted run's terminal state: the successor, by owner epoch, with +no drain. It deliberately left one question open — whether a staged cursor may +be committed when a run does not reach `succeeded`. + +That question cannot be answered by policy, because the runtime has no +information to reason with. `commitState(stream, cursor)` is already a +per-stream idempotent `PUT`, and the plumbing is exercised on every successful +run. Only the *decision* is deferred to DONE, at a single gate: + +``` +if (persistState && (terminalStatus === "succeeded" || isCertifiedStreamCollectionFailure)) +``` + +`reference-implementation/runtime/index.ts` — located by content, not by line +number, since the research entry's line numbers have since drifted. + +So the missing thing is not a policy. It is a vocabulary. + +## Goals / Non-Goals + +**Goals:** + +- A connector can state a checkpoint claim the runtime can check without + knowing anything about the connector. +- A connector that cannot state a truthful claim is *rejected*, and that + rejection is the standard working, not a failure of the standard. +- The representation admits coarse-granularity truth, so a connector with a + coarse cursor is judged on whether it can prove a coarse unit closed, not + punished for lacking a finer one. +- The representation is storage-free, so the terminal form is additive. + +**Non-Goals:** + +- Fleet-wide adoption. See the proposal; this is the point, not a concession. +- A storage, compaction, or interval-merge layer. +- Rescuing a connector whose scan is unordered. + +## Decisions + +### Qualification is the product, not a side effect + +A prototype tested a four-field claim contract against `gmail`, `slack`, and +`heb`. Two of the three could not state a truthful claim, and that was first +recorded as the design failing. + +That framing is wrong and is corrected here. **A contract that admits every +connector would be worthless.** Its entire job is to separate cursors that can +express a truthful boundary from cursors that cannot. `slack` and `heb` failing +at fine granularity is the qualifier discriminating correctly — it is the +evidence that the standard has teeth, not evidence against it. + +The correct reading of the prototype: + +| Connector | Fine granularity | Day granularity | Why | +| --- | --- | --- | --- | +| `gmail` | Qualifies | n/a | IMAP UID order, and it already carries `uidvalidity` as an identifier-space epoch | +| `heb` | Fails | **Qualifies, under the closed-day rule** | Cursor is `YYYY-MM-DD`; orders sort by full timestamp, so within-day position is inexpressible | +| `slack` | Fails | Fails | No ordered scan at all; no granularity rescues it | + +### The representation is covered intervals; debt is derived + +The prototype's contract declared a `complete_through` boundary *and* a +separate `debt` list. Two fields describing one fact can disagree, and nothing +in the protocol would catch it. + +This change adopts **covered intervals** instead: the claim carries a set of +intervals over a declared space, and outstanding debt is whatever is *not* +covered. Debt is derived, so the two cannot contradict each other. A +newest-first walk states its truth directly — after page 1 `heb` covers +`[newest_day, newest_day]` and owes everything below, and that is a correct +claim at every instant rather than a promise about the future. + +This also subsumes the two-pointer shape without special-casing it. `gmail`'s +`forward_uidnext` plus `backfilled_through_uid`/`target_uid` is exactly two +covered intervals with a gap between them. + +No storage layer is specified. An implementation may keep one interval or a +thousand; the representation does not care, which is what makes a later +compaction layer additive. + +### A claim carries a granularity, and a coarse claim is still true + +The prototype tested claims at the granularity of the provider's sort key. That +is the wrong test for a connector whose cursor is coarser than its sort key, +which is precisely `heb`'s situation. + +A day-boundary claim — "every order dated D or later is accounted for" — is +truthful even when within-day ordering is inexpressible, **provided the +connector can prove the day is closed**. Coarsening the granularity does not +weaken the claim's meaning; it widens the unit over which completeness must be +proven, which makes the claim *harder* to earn, not easier. + +The claim's granularity is declared, and every interval endpoint is a position +at that granularity. A day-granularity claim covering `[D, D]` asserts that +every item in the whole of day `D` is accounted for. It does not assert +anything about ordering inside `D`, and it does not need to. + +### The `heb` day-granularity verdict: qualifies, under a closed-day rule + +**Verdict: `heb` `orders` qualifies at day granularity, and only for days +proven closed. The naive day claim is unsafe and is rejected.** + +Two independent hazards put holes *inside* a day, and each is answered: + +**Hazard 1 — the newest day is open.** `runForwardScan` +(`packages/polyfill-connectors/connectors/heb/index.ts:868-925`) walks list +pages newest-first and tracks `newestOrderDate` as a running max +(`:901-903`). At any instant mid-walk, the newest day seen may be only +partially enumerated: orders from the same day sort by full timestamp and can +straddle a page boundary, so page 1 may hold three of that day's five orders. +Claiming `[newestOrderDate, ...]` from page 1 would claim a day whose remaining +orders have not been seen. + +The rule that answers it: **a day may be claimed only once a strictly older day +has been observed.** Seeing an order dated `D-1` proves the newest-first walk +has passed every order dated `D`. This is a property of the walk order, which +`heb` genuinely has (globally reverse-chronological, per `resumeBoundary`'s +own doc comment: "H-E-B's order list is globally reverse-chronological (not +year-partitioned like Amazon)"). The newest day observed is therefore always +excluded from the claim — it is exactly the open unit. + +**Hazard 2 — `dateDropped` puts a hole in a day that cannot be located.** This +is the sharper one, and it is why the rule must be a *closed*-day rule rather +than just an *older-day-seen* rule. + +`processListOrder` (`:829-854`) parses each order's date. When +`parseOrderDate` returns null — it is a `new Date(raw)` parse of DOM free text +(`connectors/heb/parsers.ts:341-350`) — the order is pushed to +`ordersCoverage.dateDropped` (`:851`) and the function `return`s before +`emitOrderAndItems`. The order is *considered but not covered*, exactly as the +`OrdersCoverage` doc comment states. + +The hazard is that **a date-dropped order has no date**. It cannot be +attributed to any day, so it cannot be excluded from a specific day's interval. +It is a hole of unknown position, and any interval claimed on a run that +dropped a date might be the interval containing it. + +The rule that answers it: **a claim is emitted only when `dateDropped` is empty +for the run.** A single unparseable date withholds the entire claim for that +run. This is coarse and deliberately so — it fails closed, it needs no +attribution machinery, and the fallback is today's behavior. `heb` runs are +small, so withholding a whole run's claim is cheap. + +A third, lesser hazard is worth naming because it bounds the claim rather than +blocking it: `MAX_LIST_PAGES = 50` (`:64`) and the `maxPage` exhaustion at +`:919-921` can end a walk early. This does not threaten the claim, because a +truncated walk simply covers fewer days — the *upper* part of the space is +still proven, and the untraversed remainder is uncovered by construction. This +is the covered-intervals representation earning its place: truncation is +expressible as a smaller covered set, whereas under a single `complete_through` +watermark it would be indistinguishable from completion. + +Note that `CHECKPOINT_OVERLAP_DAYS = 60` (`:70`) is **not** part of the safety +argument. It re-scans a 60-day window to catch status transitions on +already-seen orders (`resumeBoundary`, `:1064-1073`). It is a freshness device, +not a completeness device, and treating it as a safety margin would be exactly +the kind of accidental-timing invariant this contract exists to replace. + +**Confidence.** High on the source reading: the scan order, the `dateDropped` +path, and the single post-scan `STATE` emission (`:1183-1186`) were each read +directly. Not verified live — `heb` sign-in costs the owner a real OTP, so no +run was triggered. Both rules fail closed, so a misreading withholds a commit +rather than losing data. + +### `slack` remains disqualified, and the emitted-watermark fix does not change it + +**Verdict: disqualified at every granularity.** + +`grep -c "ORDER BY" packages/polyfill-connectors/connectors/slack/index.ts` +returns **0**, reproduced on this branch. Slack's messages pass is one flat +interleaved query over the archive with no ordering, so any mid-run watermark +is a maximum over an arbitrary subset of rows, not the top of a contiguous +prefix. An interval claimed over that input asserts coverage of everything +below the maximum, while rows below it may not have been visited at all. That +is not a weaker claim than `heb`'s — it is a false one, and coarsening the +granularity makes it a more confident falsehood rather than a safer claim. A +day-granularity claim over an unordered scan asserts *more*. + +The sibling branch `fix/slack-emitted-watermark-0821` (head `9541a10db`) was +checked rather than assumed. It is a real and valuable fix: it advances the +durable watermarks only for rows actually emitted and accepted, removes the +`COALESCE(t.last_ts, ?)` global-floor inheritance, and parenthesizes an +operator-precedence bug in the same predicate. **It does not add an ordering.** +`grep -c "ORDER BY"` on that branch's `connectors/slack/index.ts` also returns +**0**. The fix makes the watermark honest about *which rows it counted*; it +does not make the scan a prefix. Disqualification stands, and it stands for the +same reason it did before. + +This is worth stating plainly because the two defects are easy to conflate: the +emitted-vs-iterated defect was a bug *within* an unclaimable design, and fixing +it does not make the design claimable. + +### Rejected representations + +- **`complete_through` plus a declared `debt` list.** Two fields for one fact, + free to disagree. Covered intervals derive debt instead. +- **A `covered`/`considered` ratio.** Measures detail-hydration honesty, not + position. `github` `starred` reports itself `partial` yet still advances its + watermark past dropped entries; `jellyfin` `items` writes a cursor nothing + reads. A ratio both over- and under-approximates cursor safety. +- **A `safe: true` boolean.** Unfalsifiable self-attestation. +- **A connector version or capability flag.** An allowlist with extra steps. +- **A global floor for unseen partitions.** Deliberately inexpressible. A + partition-scoped claim may move only its own partition, which makes Slack's + `COALESCE` shape unrepresentable rather than merely discouraged. + +## The ideal-compatibility rule + +Every later fix on this program obeys three constraints: + +1. **No new health logic reading evidence projections.** Health is derived from + the ledger, not from re-deriving state out of projections. +2. **No new implicit run-state flags.** Run state is explicit and durable, not + inferred from the presence or absence of a side-channel. +3. **No new cursor shapes violating the claim schema.** A new cursor either + states a checkpoint claim or stays silent; it does not invent a third + dialect. + +**Placement, and why.** The rule lives here in `design.md` as its normative +home, and is restated as a requirement in this change's `polyfill-runtime` +spec delta so it survives archival into `openspec/specs/`. + +It is deliberately **not** added to `openspec/README.md`. That file documents +the OpenSpec process — artifact kinds, lifecycle, validation commands, the +closeout checklist — and is scoped to how changes are written, not to what any +particular change may contain. A program-specific engineering constraint placed +there would be read by every contributor to every unrelated change, which is +how process docs accrete rules nobody applies. There is also no +CONTRIBUTING-adjacent surface in this repository that governs runtime +architecture. + +The spec delta is the surface that actually binds. A design.md is advisory and +is archived with its change; a requirement in `openspec/specs/polyfill-runtime/` +is the repository's standing statement of how the runtime behaves, and is what a +later contributor or agent validates against. Putting the rule anywhere that +does not survive `openspec archive` would guarantee it is forgotten by the step +that needs it most, which is Step 3 of this program. + +## Corrections to prior statements + +### The restart worst case is redo since the last COMMITTED cursor + +Earlier statements — including the research entry +`ai/research/pdpp/the-checkpoint-protocol-must-carry-a-proven-boundary-and-an-identifier-space-epoch-...md` +§5 — said the residue of an interruption is that "work done *since* the last +checkpoint is redone." + +**That is wrong, and it understates the cost.** Staged cursors are discarded on +interruption, so nothing a run staged mid-flight survives. The correct statement +is that for a `commit_on_success` connector, an interruption redoes work since +the last **committed** cursor — which is the cursor written by the last +*successful* run, effectively the run start. + +Verified by content on this branch, since line numbers have drifted: + +- `newState` is a plain in-memory `Record` + (`reference-implementation/runtime/index.ts:2762`), assigned on each `STATE` + message (`:4237`). +- `commitState` has exactly two call sites (`:5278`, `:5322`), both inside the + DONE gate at `:5266`. There is no mid-run commit path. + +So a staged cursor has no durable effect until DONE, and an interrupted run +leaves the committed cursor exactly where the previous successful run left it. + +This matters to the program's cost-benefit and is corrected because it is +accurate, not because it is convenient: it makes the pain *larger* than +previously stated, which strengthens the case for the ledger. The 465 runs that +ingested 897,916 records and advanced no cursor are the direct measurement of +this, and they are not "since the last checkpoint" losses — they are whole-run +losses. + +## Risks / Trade-offs + +- [A connector states a claim it cannot honor] -> The runtime does not trust the + claim alone. It checks the claim against a fact it wrote itself: a claim that + advances a position with zero durably ingested records for that stream this + run is staged, not committed. Flink's rule applies — the durable ingest *is* + the pre-commit. +- [The closed-day rule withholds too often on `heb`] -> It withholds a run's + claim whenever any date fails to parse. The fallback is today's behavior + (`commit_on_success`), so the failure mode is a redone run, never a lost + order. If withholding proves common in practice, the fix is to attribute + dropped orders to a day, not to weaken the rule. +- [Covered intervals grow unbounded] -> Out of scope by construction. The + representation is storage-free precisely so a compaction layer can be added + additively once a real growth pattern is measured rather than guessed. +- [Coarse granularity becomes an escape hatch] -> It is the opposite: a coarser + unit must be proven complete over a wider range, so it is harder to earn. + `slack` demonstrates the floor — coarsening does not rescue a scan that has no + order. +- [The `heb` verdict is wrong because it was not run live] -> Both rules fail + closed. The cost of a misreading is a withheld commit. + +## Migration Plan + +Sequenced cheapest-value-first. This change lands only the standard; the +numbered steps below are later tranches. + +1. Add the optional `checkpoint_claim` field to the protocol. Inert — no + connector emits it and no behavior changes. +2. Implement the runtime decision procedure. Still inert for every connector + that stays silent. +3. Claim it in `gmail` `messages` first: it already carries all the needed + structure including `uidvalidity`, and it is 203,417 of the lost records. +4. `chatgpt` and `codex` next — together 657,477 records, the largest single + win in the fleet. +5. `heb` under the closed-day rule, if and only if its interruption pain is + shown to be real. `heb` p50 run duration is 25.2s, so it may correctly never + qualify for the *investment* even though it qualifies for the *contract*. + +Rollback at any step is deletion of an optional field; a connector that stops +claiming reverts to `commit_on_success` with no data migration. diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/proposal.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/proposal.md new file mode 100644 index 000000000..374a93796 --- /dev/null +++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/proposal.md @@ -0,0 +1,106 @@ +## Why + +The connector→runtime protocol types a checkpoint payload as `cursor: unknown` +(`packages/polyfill-connectors/src/connector-runtime-protocol.ts`), and the +runtime's complete validation of it is that it is a non-array object or null. +The runtime therefore has zero information with which to tell a safe cursor +from an unsafe one, and `commit_on_success` is what that absence of information +forces — a no-information fallback, not a risk judgment. + +The cost is measured on the live spine: fleet-wide, 130,517 `run.state_staged` +against 75,513 `run.state_advanced`, so 42% of staged checkpoint work is +discarded. Restricting to terminal runs that staged at least one stream and +reported `checkpoint_commit_status = not_committed`, 465 runs across 14 +connectors durably ingested 897,916 records and advanced no cursor. + +Today, cursor safety is not a property of the cursor. It is a property of when +a connector happens to emit. `heb` `orders` sets `newestOrderDate` from page 1 +before the older pages are walked, and is safe only because its single +`emit({ type: "STATE" ... })` sits after `runForwardScan` returns. Move that +emit inside the loop and it becomes permanent data loss, with no runtime check +firing and no test failing. That is an unowned invariant held in place by code +review alone. + +This change writes down the contract that lets a connector *say* whether its +checkpoint is safe, and lets the runtime check that claim against a fact the +runtime itself wrote. + +## What Changes + +This change is a **qualification standard**, not a migration plan. It defines +what a connector must be able to express to earn incremental commit, and +accepts that most of the fleet will never express it. + +- Define an optional `checkpoint_claim` on the `STATE` message carrying a + declared identifier space with an epoch, a set of **covered intervals** over + that space, and an optional partition key. Outstanding debt is **derived from + the gaps** between covered intervals; it is never separately declared, so the + two cannot disagree. +- Define the runtime's connector-agnostic decision procedure. Absent claim, + mismatched epoch, or a claim advancing with no durably ingested records this + run ⇒ stage only. A partition-scoped claim moving any other partition's + position ⇒ protocol violation. +- Define **claim granularity**. A claim's interval endpoints are positions in a + declared space at a declared granularity. A coarse granularity (a day) is a + truthful claim when a finer one is not expressible, provided the connector + can prove every item in that coarse unit was accounted for. +- Record the qualification results for the three prototyped connectors as + evidence that the qualifier discriminates: `gmail` qualifies, `slack` is + disqualified at every granularity, and `heb` qualifies **only** at day + granularity and **only** under a closed-day rule this change specifies. +- Adopt an ideal-compatibility rule constraining every later fix on this + program: no new health logic reading evidence projections, no new implicit + run-state flags, no new cursor shapes violating the claim schema. + +Non-qualifying connectors keep `commit_on_success` unchanged and forever. That +is a correct permanent answer, not a temporary one. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `polyfill-runtime`: Define the checkpoint claim's covered-intervals + representation, the identifier-space epoch, claim granularity and the + closed-unit rule, the runtime's commit decision procedure, the + disqualification criteria, and the ideal-compatibility rule. + +## Impact + +- No production code changes in this tranche. This change is the qualification + standard; implementation is sequenced behind it. +- When implemented: `packages/polyfill-connectors/src/connector-runtime-protocol.ts` + gains an optional field, and `reference-implementation/runtime/index.ts` + gains the decision procedure at its `STATE` handler. Both are additive — a + connector that says nothing keeps today's behavior exactly. +- The representation is deliberately storage-free, so a later compaction or + interval-store layer is an additive migration rather than a rewrite. + +## Non-Goals + +- **Fleet-wide adoption.** Explicitly not a goal. A connector whose runs take + 40 seconds should redo its run; buying the contract there is cost with no + benefit. The contract is bought only where interruption pain is real, which + the live measurement localizes to a handful of connectors. +- **A storage or compaction layer for intervals.** This change specifies the + representation only. How intervals are stored, merged, or bounded is left + open so the terminal form is an additive migration. +- **Committing staged cursors under an `INTERRUPTED` terminal state.** That + remains gated on the sibling change + `adjudicate-interrupted-runs-by-owner-epoch`, whose measurement stands: zero + of 34,928 `run.detail_coverage_declared` events carry `boundary`, + `slice_start`, or `slice_end`. +- **Rescuing `slack`.** No granularity coarsening makes an unordered scan + claimable. Slack needs an ordered scan first; that is separate work. +- **Any per-connector allowlist.** If one becomes necessary, the contract has + failed — the claim is meant to travel with the data. + +## Residual risks + +- The `heb` day-granularity verdict is derived from source reading, not from a + live run. `heb` sign-in costs the owner a real OTP, so live confirmation is + deferred rather than performed. The closed-day rule is written to fail closed, + so a wrong reading withholds a commit rather than losing data. diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/specs/polyfill-runtime/spec.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/specs/polyfill-runtime/spec.md new file mode 100644 index 000000000..56ad1ee4a --- /dev/null +++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/specs/polyfill-runtime/spec.md @@ -0,0 +1,267 @@ +## ADDED Requirements + +### Requirement: A checkpoint claim SHALL be optional, and its absence SHALL mean commit_on_success + +A connector `STATE` message MAY carry a `checkpoint_claim`. A connector that +emits no claim SHALL retain `commit_on_success` behavior unchanged: its staged +cursor SHALL be committed only when the run reaches a terminal status that +already permits commit. + +The runtime SHALL NOT require a claim from any connector, and SHALL NOT +maintain a per-connector allowlist of connectors permitted to claim. Eligibility +SHALL be carried by the claim itself, so a connector qualifies or fails to +qualify by what it can express, not by enumeration. + +Fleet-wide adoption SHALL NOT be treated as a goal. A connector whose runs are +short enough that redoing one is acceptable recovery MAY correctly never emit a +claim. + +#### Scenario: A silent connector keeps today's behavior + +- **WHEN** a connector emits a `STATE` message with no `checkpoint_claim` +- **THEN** the runtime SHALL stage the cursor +- **AND** it SHALL NOT commit that cursor before the run reaches a terminal + status that already permits commit + +#### Scenario: Qualification is not an allowlist + +- **WHEN** the runtime evaluates whether a checkpoint may be committed + incrementally +- **THEN** the decision SHALL depend only on the claim and on facts the runtime + itself recorded +- **AND** it SHALL NOT depend on the connector's name or version + +### Requirement: A checkpoint claim SHALL declare an identifier space with an epoch + +A `checkpoint_claim` SHALL name the identifier space its positions belong to, +and that name SHALL incorporate an epoch that changes whenever the provider +re-seeds the space. + +The runtime SHALL compare a claim's space to the stored space by equality +alone. When they differ, the runtime SHALL stage the cursor only, SHALL discard +the prior position, and SHALL NOT interpret positions from one epoch as +comparable to positions from another. + +The runtime SHALL NOT require any provider-specific knowledge to perform this +comparison. + +#### Scenario: A re-seeded identifier space invalidates the prior position + +- **WHEN** a claim declares a space that differs from the stored space +- **THEN** the runtime SHALL NOT commit the claimed position +- **AND** the prior position SHALL be discarded rather than compared + +#### Scenario: A matching space permits comparison + +- **WHEN** a claim declares a space equal to the stored space +- **THEN** the claim's positions SHALL be treated as comparable to the stored + position + +### Requirement: A checkpoint claim SHALL represent coverage as intervals, with debt derived from the gaps + +A `checkpoint_claim` SHALL express what it has accounted for as a set of +covered intervals over its declared space. Outstanding debt SHALL be derived as +the complement of the covered set within that space, and SHALL NOT be declared +as a separate field. + +A claim SHALL NOT carry both a coverage assertion and an independently declared +debt assertion, because two fields describing one fact can disagree and the +runtime cannot adjudicate between them. + +An empty covered set SHALL be a valid claim asserting that nothing is yet +accounted for. Omitting the covered set entirely SHALL be rejected as +malformed; the two SHALL NOT be treated as equivalent. + +This requirement SHALL constrain the representation only. It SHALL NOT mandate +any storage, compaction, or interval-merge strategy, so that such a layer may +be added later without changing the claim's meaning. + +#### Scenario: A newest-first walk states a truthful partial claim + +- **WHEN** a connector walking newest-first has accounted for only the newest + part of its space +- **THEN** it MAY claim exactly that part as covered +- **AND** the remainder SHALL be treated as outstanding debt without being + separately declared + +#### Scenario: A two-pointer connector needs no special case + +- **WHEN** a connector maintains both a forward watermark and a backfill floor +- **THEN** it SHALL express that state as two covered intervals separated by a + gap + +#### Scenario: An omitted covered set is malformed + +- **WHEN** a claim omits its covered set entirely +- **THEN** the runtime SHALL reject the claim as malformed +- **AND** it SHALL NOT treat the claim as asserting empty coverage + +### Requirement: A checkpoint claim SHALL declare its granularity, and a coarse claim SHALL cover whole units + +A `checkpoint_claim` SHALL declare the granularity at which its interval +endpoints are positions. A claim MAY declare a granularity coarser than the +provider's own sort order when a finer position is not expressible from the +connector's cursor. + +A coarse-granularity interval SHALL assert that every item in every unit it +covers has been accounted for. It SHALL NOT assert anything about ordering +within a unit, and SHALL NOT be required to. + +A unit SHALL be claimed only when the connector can prove that unit closed. A +unit that may still receive items the connector has not seen SHALL NOT be +claimed, regardless of how many of its items have been accounted for. + +Coarsening granularity SHALL NOT weaken what a claim asserts. A coarser unit +SHALL require completeness to be proven over a wider range, so it SHALL be +harder to earn than a finer one, not easier. + +#### Scenario: A coarse claim is truthful when the unit is closed + +- **WHEN** a connector's cursor cannot express a position finer than a whole + unit +- **AND** it can prove every item in that unit was accounted for +- **THEN** it MAY claim that unit as covered at that granularity + +#### Scenario: The open unit at the frontier is not claimed + +- **WHEN** a connector walking in one direction has reached a unit but cannot + prove it has passed every item in that unit +- **THEN** it SHALL NOT claim that unit +- **AND** it MAY claim only units it has provably passed + +#### Scenario: An unattributable omission withholds the claim + +- **WHEN** a connector enumerated an item but could not determine which unit it + belongs to +- **THEN** it SHALL NOT emit a claim covering any unit for that run +- **AND** the run SHALL fall back to `commit_on_success` behavior + +#### Scenario: A truncated walk claims less rather than claiming falsely + +- **WHEN** a walk ends early because a pagination ceiling was reached +- **THEN** the claim SHALL cover only the units actually traversed +- **AND** the untraversed remainder SHALL remain outstanding debt + +### Requirement: The runtime SHALL check a claim against durable ingest before committing + +The runtime SHALL NOT commit a claimed position on the strength of the claim +alone. A claim that advances a stream's position SHALL be committed only when +the runtime itself recorded durable ingest for that stream during the run. + +A claim that advances a position with no durably ingested records for that +stream in that run SHALL be staged only. + +The runtime's decision procedure SHALL be connector-agnostic and SHALL depend +only on the claim's declared fields and on facts the runtime recorded. + +#### Scenario: A claim without corresponding durable ingest is not committed + +- **WHEN** a claim advances a stream's position +- **AND** the runtime recorded no durable ingest for that stream during the run +- **THEN** the runtime SHALL stage the cursor without committing it + +#### Scenario: A claim backed by durable ingest commits incrementally + +- **WHEN** a claim is well-formed, its space matches, and the runtime recorded + durable ingest for that stream during the run +- **THEN** the runtime SHALL commit that stream's cursor without waiting for the + run's terminal status + +### Requirement: A partition-scoped claim SHALL move only its own partition, and a global floor SHALL be inexpressible + +A `checkpoint_claim` MAY declare a partition key. A partition-scoped claim SHALL +move only the position of the partition it names. + +A claim that would move the position of any partition other than the one it +names SHALL be rejected as a protocol violation. + +The claim schema SHALL provide no field by which a connector can assign a +position to partitions it has not enumerated. A floor inherited by unseen +partitions SHALL therefore be unrepresentable rather than merely discouraged, +because such a floor makes every item below it permanently unreachable for a +partition that was never walked. + +#### Scenario: A claim cannot move a partition it does not name + +- **WHEN** a partition-scoped claim would advance the position of a different + partition +- **THEN** the runtime SHALL reject the claim as a protocol violation + +#### Scenario: An unenumerated partition inherits no position + +- **WHEN** a connector has not enumerated a partition +- **THEN** no claim SHALL assign that partition a position +- **AND** that partition SHALL remain fully outstanding + +### Requirement: A connector whose scan has no order SHALL NOT qualify at any granularity + +A connector SHALL NOT emit a `checkpoint_claim` for a stream whose items are +retrieved without an ordering over the claimed space. + +A maximum taken over an arbitrary subset of items SHALL NOT be treated as the +upper bound of a covered interval. Without an ordering, items below that maximum +may never have been visited, so an interval claimed up to it asserts coverage +the connector cannot demonstrate. + +Declaring a coarser granularity SHALL NOT qualify an unordered scan. A coarser +unit asserts completeness over a wider range, so coarsening an unordered scan +SHALL produce a broader false claim rather than a safer one. + +Advancing a watermark only for items actually emitted SHALL NOT by itself +qualify a stream. Restricting the watermark to emitted items makes it honest +about which items were counted; it does not establish that the counted items +form a contiguous prefix of the space. + +#### Scenario: An unordered scan is disqualified + +- **WHEN** a stream's items are retrieved by a query with no ordering over the + claimed space +- **THEN** that stream SHALL NOT emit a checkpoint claim +- **AND** it SHALL retain `commit_on_success` behavior + +#### Scenario: Coarsening does not rescue an unordered scan + +- **WHEN** an unordered stream declares a coarser claim granularity +- **THEN** it SHALL still be disqualified + +#### Scenario: An emitted-only watermark does not establish a prefix + +- **WHEN** a stream advances its watermark only over items it emitted, but still + retrieves those items without an ordering +- **THEN** it SHALL remain disqualified + +### Requirement: New work on the checkpoint program SHALL preserve ideal compatibility + +Changes made under this program SHALL observe the following constraints, so that +later tranches compose rather than accumulating incompatible mechanisms. + +New health logic SHALL NOT be derived by reading evidence projections. Health +SHALL be derived from the coverage ledger. + +New implicit run-state flags SHALL NOT be introduced. Run state SHALL be +explicit and durable, and SHALL NOT be inferred from the presence or absence of +a side-channel. + +New cursor shapes SHALL NOT violate the claim schema. A new cursor SHALL either +state a well-formed `checkpoint_claim` or emit no claim at all. + +#### Scenario: Health is not re-derived from a projection + +- **WHEN** a change introduces health logic under this program +- **THEN** that logic SHALL read the coverage ledger +- **AND** it SHALL NOT re-derive health by reading an evidence projection + +#### Scenario: Run state stays explicit + +- **WHEN** a change under this program needs to record that a run is in some + state +- **THEN** it SHALL record that state explicitly and durably +- **AND** it SHALL NOT encode it as the presence or absence of an unrelated + side-channel + +#### Scenario: A new cursor either claims well-formed or stays silent + +- **WHEN** a change under this program introduces a new cursor shape +- **THEN** that cursor SHALL either carry a well-formed `checkpoint_claim` or + carry none +- **AND** it SHALL NOT introduce a third representation of checkpoint safety diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/tasks.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/tasks.md new file mode 100644 index 000000000..aed75203e --- /dev/null +++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/tasks.md @@ -0,0 +1,80 @@ +## 1. Establish the qualification standard (this tranche) + +- [x] 1.1 Frame the checkpoint contract as a qualification standard rather than + a failed design, and state that fleet-wide adoption is not a goal. +- [x] 1.2 Adopt the covered-intervals representation, with debt derived from the + gaps, and specify no storage or compaction layer so the terminal form is + an additive migration. +- [x] 1.3 Define claim granularity and the closed-unit rule, so a connector with + a coarse cursor is judged on whether it can prove a coarse unit complete. +- [x] 1.4 Record the disqualification criteria for unordered scans, including + that coarsening does not rescue them and that an emitted-only watermark + does not establish a prefix. + +## 2. Measure the coarse-granularity question + +- [x] 2.1 Re-test `heb` `orders` at day granularity by reading + `packages/polyfill-connectors/connectors/heb/index.ts`. Verdict: + **qualifies at day granularity, under a closed-day rule**. +- [x] 2.2 Identify the frontier hazard: `runForwardScan` tracks + `newestOrderDate` as a running max (`:901-903`) while orders within a day + sort by full timestamp, so the newest day observed may be partially + enumerated. Answered by claiming a day only once a strictly older day has + been observed. +- [x] 2.3 Identify the `dateDropped` hazard: an order whose date does not parse + is recorded as considered-but-not-covered (`:849-852`) and has no day to + be attributed to, so it is a hole of unknown position. Answered by + withholding the whole run's claim when `dateDropped` is non-empty. +- [x] 2.4 Confirm truncation is expressible rather than blocking: + `MAX_LIST_PAGES = 50` (`:64`) and `maxPage` exhaustion (`:919-921`) end a + walk early, which under covered intervals is simply a smaller covered set. +- [x] 2.5 Confirm `CHECKPOINT_OVERLAP_DAYS = 60` (`:70`) is a freshness device + via `resumeBoundary` (`:1064-1073`), not part of the safety argument. +- [x] 2.6 Re-confirm `slack` disqualification: + `grep -c "ORDER BY" connectors/slack/index.ts` returns `0` on this branch. +- [x] 2.7 Check whether `fix/slack-emitted-watermark-0821` changes that. + It does not: `grep -c "ORDER BY"` on that branch's + `connectors/slack/index.ts` also returns `0`. The fix corrects + emitted-vs-iterated watermark advance and removes the global-floor + `COALESCE`, but adds no ordering. + +## 3. Correct the redo-cost statement + +- [x] 3.1 Verify by content that no mid-run commit path exists: `newState` is + in-memory (`reference-implementation/runtime/index.ts:2762`, assigned at + `:4237`) and `commitState` has exactly two call sites (`:5278`, `:5322`), + both inside the DONE gate at `:5266`. +- [x] 3.2 Correct the wording in the research entry + `ai/research/pdpp/the-checkpoint-protocol-must-carry-a-proven-boundary-...md` + §5, from "work done since the last checkpoint is redone" to redo since the + last *committed* cursor. +- [x] 3.3 Record the correction in this change's `design.md` so the corrected + cost-benefit is visible to a reviewer who does not read the corpus. +- [x] 3.4 Confirm the wording does not appear in any other OpenSpec artifact, + branch design note, or research entry. + +## 4. Record the ideal-compatibility rule + +- [x] 4.1 State the rule in `design.md` with its placement justification. +- [x] 4.2 Restate it as a requirement in the `polyfill-runtime` spec delta so it + survives archival into `openspec/specs/`. +- [x] 4.3 Decide against `openspec/README.md`: that file is scoped to OpenSpec + process, not to program-specific engineering constraints. + +## 5. Validation + +- [x] 5.1 `openspec validate qualify-connectors-for-incremental-checkpoint-commit --strict` + passes. +- [x] 5.2 `openspec validate --all --strict` shows the same 11 pre-existing + failures as the baseline captured before this change, with this change + passing. + +## 6. Deferred to later tranches + +- [ ] 6.1 Add the optional `checkpoint_claim` field to + `packages/polyfill-connectors/src/connector-runtime-protocol.ts`. +- [ ] 6.2 Implement the runtime decision procedure at the `STATE` handler. +- [ ] 6.3 Claim in `gmail` `messages` first. +- [ ] 6.4 Owner-authorized live confirmation of the `heb` day-granularity + verdict. Deferred here because a `heb` run costs the owner a real OTP, and + both rules fail closed so the cost of a misreading is a withheld commit. diff --git a/packages/cli/scripts/package-contract.ts b/packages/cli/scripts/package-contract.ts index 205222544..9b47241fe 100644 --- a/packages/cli/scripts/package-contract.ts +++ b/packages/cli/scripts/package-contract.ts @@ -3,12 +3,27 @@ import assert from "node:assert/strict"; import { existsSync, readFileSync, statSync } from "node:fs"; +import { builtinModules } from "node:module"; import { relative, resolve, sep } from "node:path"; const TEST_ARTIFACT = /(^|\/)\.?.+\.test\.(?:js|mjs|cjs|ts|mts|cts)$/; const WHITESPACE = /\s/; const NPM_PACK_OUTPUT_MAX_BYTES = 8 * 1024 * 1024; +// Node's built-in module names, with and without the `node:` prefix. +const NODE_BUILTIN_SPECIFIERS = new Set(builtinModules.flatMap((name) => [name, `node:${name}`])); + +// Static `import`/`export` are anchored to the start of a line (optionally +// indented): tsc/esbuild output always emits these as statements starting a +// line, never mid-expression, so anchoring avoids false positives on runtime +// code that merely contains the words "import"/"export" inside a string or +// property access. `export` additionally requires a trailing `from "…"` — +// the only valid syntax for a re-export naming a module specifier. +const STATIC_IMPORT_OR_EXPORT_FROM = + /^[ \t]*(?:import\s+(?:[^"'\n;]*?\s+from\s+)?["']([^"'.][^"']*)["']|export\s+[^"'\n;]*?\s+from\s+["']([^"'.][^"']*)["'])/gm; +const DYNAMIC_IMPORT = /\bimport\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g; +const REQUIRE_CALL = /\brequire\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g; + interface ExportTarget { label: string; target: string; @@ -75,9 +90,12 @@ function collectExportTargets(value: unknown, label: string, targets: ExportTarg export interface PackageManifest { bin: Record; + dependencies?: Record; exports: Record; files: string[]; name: string; + optionalDependencies?: Record; + peerDependencies?: Record; } export function assertManifestTargets(manifest: unknown, packageRoot: string): asserts manifest is PackageManifest { @@ -146,6 +164,107 @@ export function assertPackedFiles(manifest: PackageManifest, packedFiles: string } } +/** + * Resolve a bare import specifier to the npm package name it names: the + * whole specifier for an unscoped package (`zod` from `zod/v4`), or the + * first two path segments for a scoped package (`@pdpp/read-core` from + * `@pdpp/read-core/records`). + */ +function bareSpecifierPackageName(specifier: string): string { + const segments = specifier.split("/"); + if (specifier.startsWith("@")) { + return segments.slice(0, 2).join("/"); + } + return segments[0]; +} + +function isBareSpecifier(specifier: string): boolean { + return !(specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("node:")); +} + +/** + * Extract every bare (non-relative, non-absolute) import/export/require + * specifier a compiled `.js`/`.mjs`/`.d.ts` file references: static + * `import … from "x"` (including the bare side-effect form `import "x"`), + * `export … from "x"`, dynamic `import("x")`, and `require("x")`. + */ +function bareImportSpecifiers(source: string): Set { + const specifiers = new Set(); + for (const match of source.matchAll(STATIC_IMPORT_OR_EXPORT_FROM)) { + const specifier = match[1] ?? match[2]; + if (specifier && isBareSpecifier(specifier) && !specifier.startsWith("node:")) { + specifiers.add(specifier); + } + } + for (const pattern of [DYNAMIC_IMPORT, REQUIRE_CALL]) { + for (const [, specifier] of source.matchAll(pattern)) { + if (isBareSpecifier(specifier) && !specifier.startsWith("node:")) { + specifiers.add(specifier); + } + } + } + return specifiers; +} + +/** + * Every published `@pdpp/local-collector` 1.5.1-1.5.4 shipped a compiled + * `import … from "@pdpp/reference-contract/common"` that was not in + * `dependencies` and does not exist on the npm registry: it resolved for + * every developer through the pnpm workspace link and failed closed for + * every real npm install with `ERR_MODULE_NOT_FOUND`. Neither + * `assertManifestTargets` (declared dependency sections only) nor + * `assertPackedFiles` (packed file layout only) looks at what the packed + * code actually imports, so a bare specifier undeclared in package.json can + * slip through both untouched. This closes that gap: every bare import, + * export-from, dynamic import(), and require() specifier compiled into the + * packed `.js`/`.mjs`/`.d.ts` files must resolve to either a Node builtin or + * a package the manifest actually declares as a real (non-workspace, + * non-file:) dependency. `@pdpp/cli` currently declares no runtime + * dependencies at all, so today this means: no bare specifiers other than + * Node builtins may appear in the packed output. + */ +export function assertBareSpecifiersResolve( + manifest: PackageManifest, + extractedRoot: string, + packedFiles: string[] +): void { + const declaredPackages = new Set([ + ...Object.keys(manifest.dependencies ?? {}), + ...Object.keys(manifest.peerDependencies ?? {}), + ...Object.keys(manifest.optionalDependencies ?? {}), + ]); + + for (const file of packedFiles) { + if (!(file.endsWith(".js") || file.endsWith(".mjs") || file.endsWith(".d.ts"))) { + continue; + } + const source = readFileSync(resolve(extractedRoot, file), "utf8"); + for (const specifier of bareImportSpecifiers(source)) { + const packageName = bareSpecifierPackageName(specifier); + if (NODE_BUILTIN_SPECIFIERS.has(specifier) || NODE_BUILTIN_SPECIFIERS.has(packageName)) { + continue; + } + if (declaredPackages.has(packageName)) { + continue; + } + if (packageName.startsWith("@pdpp/")) { + throw new Error( + `${file} imports private workspace package "${packageName}" (specifier "${specifier}") which is not ` + + "declared in dependencies/peerDependencies/optionalDependencies. This is the exact defect that made " + + "every published @pdpp/local-collector 1.5.1-1.5.4 unrunnable (ERR_MODULE_NOT_FOUND on every " + + "install). Declare a real dependency, vendor the needed symbol, or rewrite the specifier at build " + + "time before packing." + ); + } + throw new Error( + `${file} imports "${specifier}" (package "${packageName}") which is not declared in ` + + "dependencies/peerDependencies/optionalDependencies and is not a Node builtin. A clean npm install of " + + "this package would fail to resolve this import at runtime." + ); + } + } +} + export function parseNpmPackOutput(output: string): NpmPackResult[] { assert.ok( Buffer.byteLength(output, "utf8") <= NPM_PACK_OUTPUT_MAX_BYTES, diff --git a/packages/cli/scripts/validate-package.ts b/packages/cli/scripts/validate-package.ts index a2a7794c1..9398cba62 100644 --- a/packages/cli/scripts/validate-package.ts +++ b/packages/cli/scripts/validate-package.ts @@ -8,6 +8,7 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { + assertBareSpecifiersResolve, assertManifestTargets, assertPackedFiles, type PackageManifest, @@ -51,6 +52,7 @@ try { encoding: "utf8", }); assertManifestTargets(manifest, join(extractionRoot, "package")); + assertBareSpecifiersResolve(manifest, join(extractionRoot, "package"), packedFiles); process.stdout.write(`Validated ${pack.filename} (${packedFiles.length} files).\n`); } finally { rmSync(tempRoot, { force: true, recursive: true }); diff --git a/packages/cli/test/artifact-contract.test.ts b/packages/cli/test/artifact-contract.test.ts index b2a104ebf..fe472a8f4 100644 --- a/packages/cli/test/artifact-contract.test.ts +++ b/packages/cli/test/artifact-contract.test.ts @@ -10,9 +10,12 @@ import { fileURLToPath } from "node:url"; import { assertArtifactReceipt, bindNodeEnvironment, gitHeadSha } from "../scripts/artifact-receipt.ts"; import { discoverTestFiles, needsTsx } from "../scripts/discover-tests.ts"; -import { assertManifestTargets } from "../scripts/package-contract.ts"; +import { assertBareSpecifiersResolve, assertManifestTargets } from "../scripts/package-contract.ts"; const packageRoot = fileURLToPath(new URL("..", import.meta.url)); +const UNDECLARED_PDPP_IMPORT = + /imports private workspace package "@pdpp\/reference-contract".*1\.5\.1-1\.5\.4 unrunnable/s; +const UNDECLARED_THIRD_PARTY_IMPORT = /imports "left-pad".*not declared in dependencies/s; function makeManifest(overrides = {}) { return { @@ -76,6 +79,44 @@ test("artifact contract rejects a bin that loses its shebang", () => { assert.throws(() => assertManifestTargets(makeManifest(), root), /must retain its node shebang/); }); +test("bare-specifier check rejects the exact @pdpp/local-collector 1.5.1-1.5.4 defect shape: an undeclared private-package import", () => { + const root = emittedFixture(); + writeFileSync( + join(root, "dist", "src", "index.js"), + 'import { canonicalTerminalRunCommitEnvelope } from "@pdpp/reference-contract/common";\nexport const artifact = true;\n' + ); + assert.throws( + () => assertBareSpecifiersResolve(makeManifest(), root, ["dist/src/index.js", "dist/bin/pdpp.js"]), + UNDECLARED_PDPP_IMPORT + ); +}); + +test("bare-specifier check rejects any undeclared bare import, not just @pdpp/* ones", () => { + const root = emittedFixture(); + writeFileSync( + join(root, "dist", "src", "index.js"), + 'import leftPad from "left-pad";\nexport const artifact = true;\n' + ); + assert.throws( + () => assertBareSpecifiersResolve(makeManifest(), root, ["dist/src/index.js", "dist/bin/pdpp.js"]), + UNDECLARED_THIRD_PARTY_IMPORT + ); +}); + +test("bare-specifier check accepts declared dependencies and Node builtins", () => { + const root = emittedFixture(); + writeFileSync( + join(root, "dist", "src", "index.js"), + 'import { z } from "zod";\nimport path from "node:path";\nexport const artifact = true;\n' + ); + assert.doesNotThrow(() => + assertBareSpecifiersResolve(makeManifest({ dependencies: { zod: "^4.4.3" } }), root, [ + "dist/src/index.js", + "dist/bin/pdpp.js", + ]) + ); +}); + test("extension-complete discovery and loader selection are exact", async () => { const root = mkdtempSync(join(tmpdir(), "pdpp-cli-test-discovery-")); const cases = [ diff --git a/packages/display/package.json b/packages/display/package.json index 1526d57a7..1bdf408ab 100644 --- a/packages/display/package.json +++ b/packages/display/package.json @@ -5,7 +5,8 @@ "type": "module", "description": "Framework-independent rules that turn PDPP values and metadata into human-readable display models.", "exports": { - ".": "./src/index.ts" + ".": "./src/index.ts", + "./health": "./src/health/axis-vocabulary.ts" }, "scripts": { "typecheck": "tsc --noEmit", diff --git a/packages/display/src/health/axis-vocabulary.test.ts b/packages/display/src/health/axis-vocabulary.test.ts new file mode 100644 index 000000000..608a54d89 --- /dev/null +++ b/packages/display/src/health/axis-vocabulary.test.ts @@ -0,0 +1,84 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { + formatAttentionAxis, + formatCoverageAxis, + formatFreshnessAxis, + formatOutboxAxis, +} from "./axis-vocabulary.ts"; + +test("coverage axis maps all known states to owner-facing words with correct tone", () => { + assert.equal(formatCoverageAxis("complete").value, "complete"); + assert.equal(formatCoverageAxis("complete").tone, "success"); + + assert.equal(formatCoverageAxis("deferred").value, "optional, not collected"); + assert.equal(formatCoverageAxis("deferred").tone, "neutral"); + + assert.equal(formatCoverageAxis("gaps").value, "gaps"); + assert.equal(formatCoverageAxis("gaps").tone, "warning"); + + assert.equal(formatCoverageAxis("inventory_only").value, "complete (list only, by design)"); + assert.equal(formatCoverageAxis("inventory_only").tone, "neutral"); + + assert.equal(formatCoverageAxis("partial").value, "partial"); + assert.equal(formatCoverageAxis("partial").tone, "warning"); + + assert.equal(formatCoverageAxis("retryable_gap").value, "retryable gap"); + assert.equal(formatCoverageAxis("retryable_gap").tone, "warning"); + + assert.equal(formatCoverageAxis("terminal_gap").value, "won't backfill"); + assert.equal(formatCoverageAxis("terminal_gap").tone, "danger"); + + assert.equal(formatCoverageAxis("unavailable").value, "unavailable"); + assert.equal(formatCoverageAxis("unavailable").tone, "neutral"); + + assert.equal(formatCoverageAxis("unknown").value, "not measured"); + assert.equal(formatCoverageAxis("unknown").tone, "neutral"); + + assert.equal(formatCoverageAxis("unsupported").value, "unsupported"); + assert.equal(formatCoverageAxis("unsupported").tone, "neutral"); +}); + +test("freshness axis maps known states to owner-facing words with correct tone", () => { + assert.equal(formatFreshnessAxis("fresh").value, "fresh"); + assert.equal(formatFreshnessAxis("fresh").tone, "success"); + + assert.equal(formatFreshnessAxis("stale").value, "stale"); + assert.equal(formatFreshnessAxis("stale").tone, "warning"); + + assert.equal(formatFreshnessAxis("unknown").value, "not measured"); + assert.equal(formatFreshnessAxis("unknown").tone, "neutral"); +}); + +test("outbox axis maps known states to owner-facing words with correct tone", () => { + assert.equal(formatOutboxAxis("active").value, "active"); + assert.equal(formatOutboxAxis("active").tone, "success"); + + assert.equal(formatOutboxAxis("idle").value, "idle"); + assert.equal(formatOutboxAxis("idle").tone, "success"); + + assert.equal(formatOutboxAxis("stalled").value, "stalled"); + assert.equal(formatOutboxAxis("stalled").tone, "danger"); + + assert.equal(formatOutboxAxis("unknown").value, "not measured"); + assert.equal(formatOutboxAxis("unknown").tone, "neutral"); +}); + +test("attention axis maps known states to owner-facing words with correct tone", () => { + assert.equal(formatAttentionAxis("acknowledged")?.value, "acknowledged"); + assert.equal(formatAttentionAxis("acknowledged")?.tone, "warning"); + + assert.equal(formatAttentionAxis("in_progress")?.value, "in progress"); + assert.equal(formatAttentionAxis("in_progress")?.tone, "warning"); + + assert.equal(formatAttentionAxis("none"), null); + + assert.equal(formatAttentionAxis("open")?.value, "open"); + assert.equal(formatAttentionAxis("open")?.tone, "warning"); + + assert.equal(formatAttentionAxis("unknown_state")?.value, "not measured"); + assert.equal(formatAttentionAxis("unknown_state")?.tone, "neutral"); +}); diff --git a/packages/display/src/health/axis-vocabulary.ts b/packages/display/src/health/axis-vocabulary.ts new file mode 100644 index 000000000..c1ac47122 --- /dev/null +++ b/packages/display/src/health/axis-vocabulary.ts @@ -0,0 +1,313 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The owner-facing vocabulary for the reference's connection-health axes. + * + * This module is the SINGLE source of the words an owner reads for a + * coverage / freshness / outbox / attention axis. It lives in + * `@pdpp/display` — not in the console app — because more than one surface + * must say the same thing about the same evidence: + * + * - the console `/sources` page and its source-detail subpages, and + * - `pdpp ref sources`, the headless CLI that exists so an agent and the + * owner can read the SAME rendered verdict rather than the agent reading + * raw `connector_summary_evidence` rows (the INPUTS) while the owner reads + * the rendered UI (the OUTPUT). Those two diverged badly enough to cause + * repeated miscommunication; a second copy of these strings would + * reintroduce exactly that drift. + * + * Nothing here reads the network, touches React, or depends on a framework. + * The axis KEYS are the reference server's durable wire contract; the + * `value`/`title` text is owner-facing copy this module owns. Where the two + * deliberately diverge (`deferred`, `terminal_gap`) the reason is commented + * inline — do not "fix" the mismatch by renaming a wire key. + */ + +export type EvidenceTone = "neutral" | "success" | "warning" | "danger"; + +export interface AxisChip { + /** The axis name (e.g. "Coverage", "Freshness"). Rendered muted. */ + dimension: string; + /** Short owner-facing label (e.g. "Coverage · gaps"). Kept for backward compat/tooltips. */ + label: string; + /** Long-form hover/tooltip — describes what the chip means. */ + title: string; + tone: EvidenceTone; + /** The axis state value (e.g. "gaps", "fresh"). Rendered prominent. */ + value: string; +} + +/** The reference's `connection_health.axes.coverage` domain. */ +export type CoverageAxis = + | "complete" + | "deferred" + | "gaps" + | "inventory_only" + | "partial" + | "retryable_gap" + | "terminal_gap" + | "unavailable" + | "unknown" + | "unsupported"; + +/** The reference's `connection_health.axes.freshness` domain. */ +export type FreshnessAxis = "fresh" | "stale" | "unknown"; + +/** The reference's `connection_health.axes.outbox` domain. */ +export type OutboxAxis = "active" | "idle" | "stalled" | "unknown"; + +/** The reference's `connection_health.axes.attention` domain. */ +export type AttentionAxis = "acknowledged" | "in_progress" | "none" | "open"; + +const COVERAGE_LABELS: Record = { + complete: { + dimension: "Coverage", + label: "Coverage · complete", + title: "All required streams have durable evidence of complete coverage.", + tone: "success", + value: "complete", + }, + deferred: { + dimension: "Coverage", + label: "Coverage · optional, not collected", + title: + "The manifest declares this coverage out of scope. This is an accepted, settled state — not a queued task — and does not block connection health.", + tone: "neutral", + // The underlying axis key stays "deferred" (durable manifest/runtime + // contract — see AcceptedCoveragePolicy in connector-coverage-policy.ts). + // "Deferred" read as queued/pending work to owners, contradicting the + // settled, non-degrading semantics this axis actually carries. The + // visible value/label now say plainly that this stream is optional and + // not collected; the manifest-declaration detail moves to the title. + value: "optional, not collected", + }, + gaps: { + dimension: "Coverage", + label: "Coverage · gaps", + title: "Required coverage has known retryable or terminal gaps.", + tone: "warning", + value: "gaps", + }, + inventory_only: { + dimension: "Coverage", + label: "Coverage · complete (list only, by design)", + title: + "The manifest declares that this stream only ever lists what exists, rather than downloading each item's full detail. Everything it is designed to collect has been collected, so nothing is missing and nothing is owed. This is a settled, finished state — not partial progress.", + tone: "neutral", + // B4 (owner ledger 2026-08-22): the owner asked whether the green/neutral + // tone here was intentional. It is, and it is honest: `inventory_only` is an + // AcceptedAbsencePolicy (packages/reference-contract/src/evidence/ + // coherence.ts) that the manifest declares, `hasOutstandingGap` excludes, + // and `deriveForwardDisposition` resolves to `complete` — the connection + // genuinely owes no further data. Tone therefore stays `neutral`. What was + // wrong was the WORD: "inventory only" reads as a limitation the owner + // might need to act on, so the value now states plainly that this is + // complete by design. + value: "complete (list only, by design)", + }, + partial: { + dimension: "Coverage", + label: "Coverage · partial", + title: "Some required streams collected only partial data.", + tone: "warning", + value: "partial", + }, + retryable_gap: { + dimension: "Coverage", + label: "Coverage · retryable gap", + title: + "Some required detail is missing, but the runtime expects to fill it on a later run. Records already collected stay valid; no owner action is needed yet.", + tone: "warning", + value: "retryable gap", + }, + terminal_gap: { + dimension: "Coverage", + label: "Coverage · won't backfill", + title: + "Some required detail will not backfill on its own — the connector or source cannot recover it without a change. Records already collected stay valid and usable; this is not current data loss. Open the connection's latest run to see which streams are affected and the recovery step.", + tone: "danger", + // "terminal gap" is jargon. The value stays short for the chip; the title + // carries the three things the owner actually needs (per design-notes/ + // dashboard-health-semantics-and-reliability-2026-06-01.md): what state this + // is, whether current records are safe, and what can recover coverage. The + // reference's coverage condition carries a `Review source coverage gaps` + // remediation but not the specific cause/stream/time — that contract gap is + // noted in the workstream report; the per-stream detail lives in the latest + // run's known_gaps, which the connection detail page links to. + value: "won't backfill", + }, + unavailable: { + dimension: "Coverage", + label: "Coverage · unavailable", + title: + "The manifest accepts that the source does not expose this coverage. This is a settled state, not a temporary gap awaiting a retry.", + tone: "neutral", + value: "unavailable", + }, + unknown: { + dimension: "Coverage", + label: "Coverage · not measured", + title: + "Nothing has measured how much of this stream was collected, so we can't say whether it is complete. This is missing measurement, not proven missing data — records already collected stay valid.", + tone: "neutral", + // B2 (owner ledger 2026-08-22): the chip said "unknown" while the adjacent + // forward-disposition line said "not measured" for the SAME underlying + // state, so the owner read two vocabularies for one thing. The axis key + // stays `unknown` (durable wire contract); the owner-facing word is now + // "not measured" everywhere, matching FORWARD_DISPOSITION_LABELS.unmeasured + // and SOURCE_WORK_GROUP_COPY.notMeasured. Tone stays `neutral` because + // `reference-surface-topology` requires that unknown alone SHALL NOT + // produce degraded tone — this is an absence of measurement, not a defect. + value: "not measured", + }, + unsupported: { + dimension: "Coverage", + label: "Coverage · unsupported", + title: + "The manifest accepts that the connector cannot collect this coverage. This is a settled state, not a temporary gap awaiting a retry.", + tone: "neutral", + value: "unsupported", + }, +}; + +const FRESHNESS_LABELS: Record = { + fresh: { + dimension: "Freshness", + label: "Freshness · fresh", + title: "The last successful run is within policy.", + tone: "success", + value: "fresh", + }, + stale: { + dimension: "Freshness", + label: "Freshness · stale", + title: "The last successful run is outside the configured freshness window.", + tone: "warning", + value: "stale", + }, + unknown: { + dimension: "Freshness", + label: "Freshness · not measured", + title: "Nothing has measured how recent this data is, so we can't say whether it is up to date.", + tone: "neutral", + // B2: same one-word rule as the coverage axis — "not measured" is the + // single owner-facing word for "no evidence has been taken", on every axis. + value: "not measured", + }, +}; + +const OUTBOX_LABELS: Record = { + active: { + dimension: "Outbox", + label: "Outbox · active", + title: "Outbound work is making progress.", + // `active` means the local-device outbox is draining — a healthy, + // progressing state. It previously shared `neutral` (muted grey) with + // `unknown`, so an operator could not tell a draining outbox from one + // whose evidence we could not read. `success` gives it a distinct, + // non-alarming colour (the same green as `idle`); the value text + // ("active" vs "idle") carries the finer distinction, and the row-level + // pill still escalates an actively-draining outbox to a "Syncing" badge. + tone: "success", + value: "active", + }, + idle: { + dimension: "Outbox", + label: "Outbox · idle", + title: "No retryable outbound work is pending.", + tone: "success", + value: "idle", + }, + stalled: { + dimension: "Outbox", + label: "Outbox · stalled", + title: "Retryable outbound work is stalled and not progressing.", + tone: "danger", + value: "stalled", + }, + unknown: { + dimension: "Outbox", + label: "Outbox · not measured", + title: "Nothing has measured the upload queue on this device, so we can't say whether it is keeping up.", + tone: "neutral", + // B2: one word for "no evidence taken" across every axis. + value: "not measured", + }, +}; + +const ATTENTION_LABELS: Record = { + acknowledged: { + dimension: "Attention", + label: "Attention · acknowledged", + title: "Owner action is acknowledged but not yet resolved.", + tone: "warning", + value: "acknowledged", + }, + in_progress: { + dimension: "Attention", + label: "Attention · in progress", + title: "Owner action is in progress.", + tone: "warning", + value: "in progress", + }, + none: null, + open: { + dimension: "Attention", + label: "Attention · open", + title: "Owner action is open.", + tone: "warning", + value: "open", + }, +}; + +export function formatCoverageAxis(axis: CoverageAxis | null | string | undefined): AxisChip { + return formatKnownAxis(COVERAGE_LABELS, axis, "unknown", "Coverage"); +} + +export function formatFreshnessAxis(axis: FreshnessAxis | null | string | undefined): AxisChip { + return formatKnownAxis(FRESHNESS_LABELS, axis, "unknown", "Freshness"); +} + +export function formatOutboxAxis(axis: OutboxAxis | null | string | undefined): AxisChip { + return formatKnownAxis(OUTBOX_LABELS, axis, "unknown", "Outbox"); +} + +export function formatAttentionAxis(axis: AttentionAxis | null | string | undefined): AxisChip | null { + if (axis === null) { + return null; + } + if (axis !== undefined && Object.hasOwn(ATTENTION_LABELS, axis)) { + return ATTENTION_LABELS[axis as AttentionAxis]; + } + return { + dimension: "Attention", + label: "Attention · not measured", + title: `This console does not recognize the attention state "${axis}" reported by the reference server, so it cannot say whether anything needs you.`, + tone: "neutral", + value: "not measured", + }; +} + +function formatKnownAxis( + labels: Record, + axis: T | null | string | undefined, + fallback: T, + labelPrefix: string +): AxisChip { + if (axis !== null && axis !== undefined && Object.hasOwn(labels, axis)) { + return labels[axis as T]; + } + const fallbackChip = labels[fallback]; + if (axis === null) { + return fallbackChip; + } + return { + ...fallbackChip, + dimension: labelPrefix, + title: `This console does not recognize the ${labelPrefix.toLowerCase()} state "${axis}" reported by the reference server, so it cannot say what was measured.`, + // B2: an unrecognized axis is still "we have no usable measurement", so it + // uses the same one word rather than introducing a third vocabulary. + value: "not measured", + }; +} diff --git a/packages/display/src/index.ts b/packages/display/src/index.ts index d95da5e03..0dad1a0e1 100644 --- a/packages/display/src/index.ts +++ b/packages/display/src/index.ts @@ -22,6 +22,20 @@ export type { TraceLabelInput, } from "./identity/summary-row-label.ts"; export { grantRowLabel, runRowLabel, traceRowLabel } from "./identity/summary-row-label.ts"; +export type { + AttentionAxis, + AxisChip, + CoverageAxis, + EvidenceTone, + FreshnessAxis, + OutboxAxis, +} from "./health/axis-vocabulary.ts"; +export { + formatAttentionAxis, + formatCoverageAxis, + formatFreshnessAxis, + formatOutboxAxis, +} from "./health/axis-vocabulary.ts"; export type { DeclaredFieldRoles, FieldRole } from "./record/declared-field-roles.ts"; export { EMPTY_DECLARED_FIELD_ROLES, diff --git a/packages/display/src/record/structured-value.test.ts b/packages/display/src/record/structured-value.test.ts index 782191306..727d43a51 100644 --- a/packages/display/src/record/structured-value.test.ts +++ b/packages/display/src/record/structured-value.test.ts @@ -19,11 +19,11 @@ test("an empty array reads as an explicit empty state, not a blank or raw '[]'", test("an array of objects with `name` fields renders as joined names — the Gmail `cc` case", () => { const cc = [ - { email: "mmarco@law.harvard.edu", name: "Meg Marco" }, - { email: "anna@opendatalabs.xyz", name: "Anna Kazlauskas" }, + { email: "rowan.diaz@example.edu", name: "Rowan Diaz" }, + { email: "sasha.lindqvist@example.org", name: "Sasha Lindqvist" }, ]; const result = formatStructuredCell(cc); - assert.equal(result?.text, "Meg Marco, Anna Kazlauskas"); + assert.equal(result?.text, "Rowan Diaz, Sasha Lindqvist"); assert.equal(result?.detail, undefined, "under the item cap, no separate detail is needed"); }); diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index b2ef67239..cb89e59c7 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -28,7 +28,7 @@ "test:read-surface": "node --import tsx ../../scripts/test-scratch/run-command.ts -- node ../../scripts/run-node-tests.mjs --test --import tsx \"test/*.test.ts\" test/smoke-stdio.ts", "validate:package": "pnpm build && node --import tsx scripts/package-contract.ts", "verify:artifact": "pnpm build && node --import tsx scripts/pack-install-run.ts", - "verify": "pnpm test && pnpm validate:package", + "verify": "pnpm test && pnpm validate:package && pnpm verify:artifact", "pack:dry-run": "pnpm build && npm pack --dry-run --ignore-scripts" }, "dependencies": { diff --git a/packages/mcp-server/scripts/package-contract.ts b/packages/mcp-server/scripts/package-contract.ts index f95affd27..6da4546bd 100644 --- a/packages/mcp-server/scripts/package-contract.ts +++ b/packages/mcp-server/scripts/package-contract.ts @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { existsSync, readFileSync, rmSync, statSync } from "node:fs"; +import { builtinModules } from "node:module"; import { dirname, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; @@ -11,6 +12,21 @@ const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const EXECUTABLE_PERMISSION = /[1357]/; const TEST_ARTIFACT_PATH = /(^|\/)\.?.+\.test\.(?:js|mjs|cjs|ts|mts|cts)$/; const NPM_PACK_JSON = /(\[\s*\{[\s\S]*\])\s*$/; +const NPM_PACK_JSON_OBJECT = /(\{\s*"[^"]*"\s*:\s*\{[\s\S]*\})\s*$/; + +// Node's built-in module names, with and without the `node:` prefix. +const NODE_BUILTIN_SPECIFIERS = new Set(builtinModules.flatMap((name) => [name, `node:${name}`])); + +// Static `import`/`export` are anchored to the start of a line (optionally +// indented): tsc/esbuild output always emits these as statements starting a +// line, never mid-expression, so anchoring avoids false positives on runtime +// code that merely contains the words "import"/"export" inside a string or +// property access. `export` additionally requires a trailing `from "…"` — +// the only valid syntax for a re-export naming a module specifier. +const STATIC_IMPORT_OR_EXPORT_FROM = + /^[ \t]*(?:import\s+(?:[^"'\n;]*?\s+from\s+)?["']([^"'.][^"']*)["']|export\s+[^"'\n;]*?\s+from\s+["']([^"'.][^"']*)["'])/gm; +const DYNAMIC_IMPORT = /\bimport\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g; +const REQUIRE_CALL = /\brequire\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g; // Loosely typed on purpose: this describes the runtime shape of an untrusted // `package.json` read from disk, which assertManifestTargets/assertPackedFiles @@ -150,10 +166,109 @@ export function assertPackedFiles(manifest: PackageManifest, packedFiles: string } } +/** + * Resolve a bare import specifier to the npm package name it names: the + * whole specifier for an unscoped package (`zod` from `zod/v4`), or the + * first two path segments for a scoped package (`@pdpp/read-core` from + * `@pdpp/read-core/records`). + */ +function bareSpecifierPackageName(specifier: string): string { + const segments = specifier.split("/"); + if (specifier.startsWith("@")) { + return segments.slice(0, 2).join("/"); + } + return segments[0] ?? specifier; +} + +function isBareSpecifier(specifier: string): boolean { + return !(specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("node:")); +} + +/** + * Extract every bare (non-relative, non-absolute) import/export/require + * specifier a compiled `.js`/`.mjs`/`.d.ts` file references: static + * `import … from "x"` (including the bare side-effect form `import "x"`), + * `export … from "x"`, dynamic `import("x")`, and `require("x")`. + */ +function bareImportSpecifiers(source: string): Set { + const specifiers = new Set(); + for (const match of source.matchAll(STATIC_IMPORT_OR_EXPORT_FROM)) { + const specifier = match[1] ?? match[2]; + if (specifier && isBareSpecifier(specifier) && !specifier.startsWith("node:")) { + specifiers.add(specifier); + } + } + for (const pattern of [DYNAMIC_IMPORT, REQUIRE_CALL]) { + for (const [, specifier] of source.matchAll(pattern)) { + if (specifier && isBareSpecifier(specifier) && !specifier.startsWith("node:")) { + specifiers.add(specifier); + } + } + } + return specifiers; +} + +/** + * Every published `@pdpp/local-collector` 1.5.1-1.5.4 shipped a compiled + * `import … from "@pdpp/reference-contract/common"` that was not in + * `dependencies` and does not exist on the npm registry: it resolved for + * every developer through the pnpm workspace link and failed closed for + * every real npm install with `ERR_MODULE_NOT_FOUND`. Neither + * `assertManifestTargets` (declared dependency sections only) nor + * `assertPackedFiles` (packed file layout only) looks at what the packed + * code actually imports, so a bare specifier undeclared in package.json + * can slip through both untouched. This closes that gap: every bare import, + * export-from, dynamic import(), and require() specifier compiled into the + * packed `.js`/`.mjs`/`.d.ts` files must resolve to either a Node builtin or + * a package the manifest actually declares as a real (non-workspace, + * non-file:) dependency. + */ +export function assertBareSpecifiersResolve(manifest: PackageManifest, root: string, packedFiles: string[]): void { + const declaredPackages = new Set(Object.keys(manifest.dependencies ?? {})); + + for (const file of packedFiles) { + if (!(file.endsWith(".js") || file.endsWith(".mjs") || file.endsWith(".d.ts"))) { + continue; + } + const source = readFileSync(resolve(root, file), "utf8"); + for (const specifier of bareImportSpecifiers(source)) { + const packageName = bareSpecifierPackageName(specifier); + if (NODE_BUILTIN_SPECIFIERS.has(specifier) || NODE_BUILTIN_SPECIFIERS.has(packageName)) { + continue; + } + if (declaredPackages.has(packageName)) { + continue; + } + if (packageName.startsWith("@pdpp/")) { + throw new Error( + `${file} imports private workspace package "${packageName}" (specifier "${specifier}") which is not ` + + "declared in dependencies. This is the exact defect that made every published @pdpp/local-collector " + + "1.5.1-1.5.4 unrunnable (ERR_MODULE_NOT_FOUND on every install). Declare a real dependency, vendor " + + "the needed symbol, or rewrite the specifier at build time before packing." + ); + } + throw new Error( + `${file} imports "${specifier}" (package "${packageName}") which is not declared in dependencies and is ` + + "not a Node builtin. A clean npm install of this package would fail to resolve this import at runtime." + ); + } + } +} + export function parseNpmPackOutput(output: string): NpmPackEntry[] { - const match = output.match(NPM_PACK_JSON); - assert.ok(match, "npm pack did not produce a trailing JSON payload"); - return JSON.parse(match[1] as string) as NpmPackEntry[]; + // npm's `pack --json` output shape changed across major versions: older npm + // (<=11) emits a top-level array of one record; npm 12 emits an object + // keyed by package name instead. Accept either, and tolerate `npm warn` + // lines ahead of the JSON payload (observed in this environment), rather + // than pinning this check to one npm major/config shape. + const arrayMatch = output.match(NPM_PACK_JSON); + if (arrayMatch) { + return JSON.parse(arrayMatch[1] as string) as NpmPackEntry[]; + } + const objectMatch = output.match(NPM_PACK_JSON_OBJECT); + assert.ok(objectMatch, "npm pack did not produce a trailing JSON payload"); + const parsed = JSON.parse(objectMatch[1] as string) as Record; + return Object.values(parsed); } export function packAndInspect(root: string, manifest: PackageManifest): NpmPackEntry { @@ -163,10 +278,9 @@ export function packAndInspect(root: string, manifest: PackageManifest): NpmPack }); const [pack] = parseNpmPackOutput(output); assert.ok(pack, "npm pack produced no entries"); - assertPackedFiles( - manifest, - pack.files.map((file) => file.path) - ); + const packedFiles = pack.files.map((file) => file.path); + assertPackedFiles(manifest, packedFiles); + assertBareSpecifiersResolve(manifest, root, packedFiles); return pack; } diff --git a/packages/mcp-server/test/artifact-contract.test.ts b/packages/mcp-server/test/artifact-contract.test.ts index 75c9589ae..8c7412fde 100644 --- a/packages/mcp-server/test/artifact-contract.test.ts +++ b/packages/mcp-server/test/artifact-contract.test.ts @@ -23,7 +23,12 @@ import { type SiblingCandidateEvidence, } from "../scripts/artifact-receipt.ts"; import { assertInstalledPackageMatchesTarball, resolveReceiptOutputPath } from "../scripts/pack-install-run.ts"; -import { assertManifestTargets, assertPackedFiles, type PackageManifest } from "../scripts/package-contract.ts"; +import { + assertBareSpecifiersResolve, + assertManifestTargets, + assertPackedFiles, + type PackageManifest, +} from "../scripts/package-contract.ts"; const SYMLINK = /symlink/; const STALE_OR_REPLAYED_RECEIPT = /stale or replayed receipt/; @@ -45,6 +50,9 @@ const SOURCE_TARGET = /must point into \.\/dist\//; const SOURCE_FILE = /source file leaked/; const SOURCE_FALLBACK = /resolved from source instead of the offline consumer/; const REPLAYED_RECEIPT = /stale or replayed receipt/; +const UNDECLARED_PDPP_IMPORT = + /imports private workspace package "@pdpp\/reference-contract".*1\.5\.1-1\.5\.4 unrunnable/s; +const UNDECLARED_THIRD_PARTY_IMPORT = /imports "left-pad".*not declared in dependencies/s; function manifest(overrides: Partial = {}): PackageManifest { return { @@ -133,6 +141,70 @@ test("artifact contract rejects a source fallback target and packed source files ); }); +test("bare-specifier check rejects the exact @pdpp/local-collector 1.5.1-1.5.4 defect shape: an undeclared private-package import", () => { + const root = emittedFixture(); + writeFileSync( + join(root, "dist", "src", "index.js"), + 'import { canonicalTerminalRunCommitEnvelope } from "@pdpp/reference-contract/common";\nexport const artifact = true;\n' + ); + assert.throws( + () => + assertBareSpecifiersResolve(manifest(), root, [ + "dist/src/index.js", + "dist/src/server.js", + "dist/bin/pdpp-mcp-server.js", + ]), + UNDECLARED_PDPP_IMPORT + ); +}); + +test("bare-specifier check rejects any undeclared bare import, not just @pdpp/* ones", () => { + const root = emittedFixture(); + writeFileSync( + join(root, "dist", "src", "index.js"), + 'import leftPad from "left-pad";\nexport const artifact = true;\n' + ); + assert.throws( + () => + assertBareSpecifiersResolve(manifest(), root, [ + "dist/src/index.js", + "dist/src/server.js", + "dist/bin/pdpp-mcp-server.js", + ]), + UNDECLARED_THIRD_PARTY_IMPORT + ); +}); + +test("bare-specifier check accepts declared dependencies and Node builtins", () => { + const root = emittedFixture(); + writeFileSync( + join(root, "dist", "src", "index.js"), + 'import { z } from "zod";\nimport path from "node:path";\nimport { cliThing } from "@pdpp/cli";\nexport const artifact = true;\n' + ); + assert.doesNotThrow(() => + assertBareSpecifiersResolve(manifest({ dependencies: { "@pdpp/cli": ">=0.18.11 <1.0.0", zod: "^4.4.3" } }), root, [ + "dist/src/index.js", + "dist/src/server.js", + "dist/bin/pdpp-mcp-server.js", + ]) + ); +}); + +test("bare-specifier check ignores 'import'/'export' inside string literals and property access", () => { + const root = emittedFixture(); + writeFileSync( + join(root, "dist", "src", "index.js"), + 'const assignment = line.startsWith("export ") ? line.slice("export ".length) : line;\nexport const artifact = true;\n' + ); + assert.doesNotThrow(() => + assertBareSpecifiersResolve(manifest(), root, [ + "dist/src/index.js", + "dist/src/server.js", + "dist/bin/pdpp-mcp-server.js", + ]) + ); +}); + test("consumer proof rejects an installed package symlinked to source", () => { const root = mkdtempSync(join(tmpdir(), "pdpp-mcp-source-fallback-")); const tarRoot = join(root, "tar"); diff --git a/packages/polyfill-connectors/connectors/_conformance/change-feed-is-not-inventory.test.ts b/packages/polyfill-connectors/connectors/_conformance/change-feed-is-not-inventory.test.ts new file mode 100644 index 000000000..020fd21d2 --- /dev/null +++ b/packages/polyfill-connectors/connectors/_conformance/change-feed-is-not-inventory.test.ts @@ -0,0 +1,197 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Change-feed-is-not-inventory gate. + * + * CONTRACT UNDER TEST + * ------------------- + * A stream whose manifest declares `coverage_strategy: "full_inventory"` may + * NOT prove that claim from a run whose executed code path was an incremental + * DELTA. An empty change feed and an empty inventory are different facts, and + * only one of them is evidence of anything. + * + * RFC 6578 §3.2 is normative here and makes this statically decidable: a + * sync-collection REPORT carrying an EMPTY `` MUST return every + * member of the collection, while one carrying a NON-EMPTY token returns only + * what changed since that token. So "the response listed 0 resources" means + * "0 contacts exist" in the first case and "nothing changed" in the second. + * + * THE DEFECT THIS PINS + * -------------------- + * Apple Contacts declares `full_inventory` on all three of its streams, but + * its steady-state run issues a sync-collection delta. A quiet run therefore + * observed 0 changed resources and, before the fix, reported + * `considered: 0, covered: 0` — which the coherence oracle reads as a + * measured `enumeration_boundary` proving a verified-EMPTY address book. A + * required stream holding hundreds of real contacts read Healthy with zero + * records. + * + * The connector now withholds the coverage claim entirely on an incremental + * run (`contactsBoundaryEstablished`), leaving the stream honestly unproven + * rather than falsely complete. This gate exists so that behavior cannot + * silently regress: it drives the REAL connector subprocess against a fake + * CardDAV server configured to obey RFC 6578 faithfully, with a prior sync + * token in START state, and asserts no fabricated boundary is emitted. + * + * WHY A CONFORMANCE TEST RATHER THAN A LINT OR A MANIFEST RULE + * ----------------------------------------------------------- + * A manifest-validation rule cannot see which code path executes; it can only + * read declarations, and the declaration here (`full_inventory`) is correct — + * the address book genuinely is a full inventory. A lint would have to + * pattern-match call sites and would be defeated by any indirection. + * + * The falsifiable fact is behavioral: given a prior token, does the run emit a + * coverage boundary it did not measure? Only executing the connector answers + * that, and this package already has the machinery to do it cheaply and + * credential-free (`_conformance/coverage-conformance.test.ts` drives the same + * subprocess against the same fake server). So this extends that pattern + * rather than introducing a new mechanism. + */ + +import assert from "node:assert/strict"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import type { EmittedMessage } from "@pdpp/connector-protocol"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; +import { buildVCard, startFakeCardDavServer } from "../apple_contacts/test-carddav-server.ts"; + +const PACKAGE_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const BOOK_URL_KEY_SOURCE = "/addressbooks/owner/card/"; + +interface DetailCoverageLike { + readonly considered?: number; + readonly covered?: number; + readonly stream?: string; + readonly type?: string; +} + +/** Every DETAIL_COVERAGE message the run emitted for a given stream. */ +function detailCoverageFor(messages: readonly EmittedMessage[], stream: string): DetailCoverageLike[] { + return messages.filter( + (message): message is EmittedMessage & DetailCoverageLike => + (message as DetailCoverageLike).type === "DETAIL_COVERAGE" && (message as DetailCoverageLike).stream === stream + ); +} + +function recordCountFor(messages: readonly EmittedMessage[], stream: string): number { + return messages.filter((message) => { + const shape = message as { stream?: string; type?: string }; + return shape.type === "RECORD" && shape.stream === stream; + }).length; +} + +/** + * Drive the real Apple Contacts entrypoint against a fake CardDAV server that + * obeys RFC 6578, optionally handing it a prior sync token so the run takes + * the incremental path. + */ +async function runAppleContacts(args: { readonly priorSyncToken?: string }): Promise { + const username = "owner@example.com"; + const password = "app-specific-pw"; + const server = await startFakeCardDavServer({ + enforceRfc6578IncrementalSemantics: true, + password, + username, + }); + try { + // A real, non-empty address book. The whole point is that these contacts + // exist while the change feed is empty. + for (const uid of ["contact-one", "contact-two", "contact-three"]) { + server.contacts.set(uid, { + href: `${BOOK_URL_KEY_SOURCE}${uid}.vcf`, + uid, + vcard: buildVCard({ email: `${uid}@example.com`, fn: `Fixture ${uid}`, uid }), + }); + } + + // The connector keys per-book cursor state by the book URL with trailing + // slashes stripped (`addressBookId`), so the state key must match exactly + // or the prior token is silently ignored and the run takes the full path. + const bookKeySource = `${server.origin}${BOOK_URL_KEY_SOURCE}`.replace(/\/+$/, ""); + const state = args.priorSyncToken + ? { contacts: { [bookKeySource]: { fingerprints: {}, sync_token: args.priorSyncToken } } } + : {}; + + const result = await runConnectorProtocolSubprocess({ + allowFailedDone: true, + cwd: PACKAGE_ROOT, + entrypoint: join(PACKAGE_ROOT, "connectors/apple_contacts/index.ts"), + env: { APPLE_APP_SPECIFIC_PASSWORD: password, APPLE_CARDDAV_ORIGIN: server.origin, APPLE_ID: username }, + start: { + scope: { streams: [{ name: "address_books" }, { name: "contacts" }, { name: "contact_groups" }] }, + state, + type: "START", + }, + }); + return result.messages; + } finally { + await server.close(); + } +} + +test("an initial run (empty sync token) DOES prove the full inventory", async () => { + // The control case. RFC 6578 requires an empty token to return every member, + // so this run genuinely enumerates the collection and is entitled to claim a + // measured boundary. If this stops holding, the gate below would pass + // vacuously. + const messages = await runAppleContacts({}); + + assert.equal(recordCountFor(messages, "contacts"), 3, "the initial run must emit every contact"); + + const coverage = detailCoverageFor(messages, "contacts"); + assert.equal(coverage.length, 1, "the initial run must emit exactly one contacts coverage claim"); + assert.equal(coverage[0]?.considered, 3, "considered must be the measured inventory size"); + assert.equal(coverage[0]?.covered, 3, "covered must satisfy the denominator"); +}); + +test("a quiet INCREMENTAL run must not report an empty change feed as an empty inventory", async () => { + // The regression guard for the real defect. A prior sync token puts the run + // on the delta path; the fixture's collection is unchanged, so the change + // feed is empty while three contacts still exist upstream. + const messages = await runAppleContacts({ priorSyncToken: "sync-token-1" }); + + // The delta legitimately carries no contact records — nothing changed. + assert.equal(recordCountFor(messages, "contacts"), 0, "a quiet delta emits no contact records"); + + const coverage = detailCoverageFor(messages, "contacts"); + + // THE ASSERTION THAT MATTERS. Emitting `considered: 0, covered: 0` here is + // exactly the defect: the coherence oracle reads a measured zero denominator + // as a proven-empty inventory, so a populated address book would read + // verified-empty and Healthy. Withholding the claim leaves the stream + // honestly unproven instead. + for (const claim of coverage) { + assert.notEqual( + claim.considered, + 0, + "an incremental delta must never emit considered: 0 — that is a change feed being " + + "reported as an inventory, which the coherence oracle reads as verified-empty" + ); + } + assert.equal( + coverage.length, + 0, + "a run that established no full boundary must emit NO contacts coverage claim at all; " + + "silence is the honest verdict, not a fabricated zero" + ); +}); + +test("the incremental run still commits its cursor, so withholding coverage does not stall sync", async () => { + // Withholding a coverage CLAIM must not be confused with failing the run. + // The delta is real progress: its sync token has to persist or the next run + // would re-walk from the same place forever. + const messages = await runAppleContacts({ priorSyncToken: "sync-token-1" }); + + const stateMessages = messages.filter((message) => { + const shape = message as { stream?: string; type?: string }; + return shape.type === "STATE" && shape.stream === "contacts"; + }); + assert.ok(stateMessages.length > 0, "the incremental run must still checkpoint its sync token"); + + const done = messages.find((message) => (message as { type?: string }).type === "DONE") as + | { status?: string } + | undefined; + assert.equal(done?.status, "succeeded", "a quiet incremental run is a success, not a failure"); +}); diff --git a/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts b/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts index 456e5c83f..30b61d503 100644 --- a/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts +++ b/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts @@ -50,6 +50,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import type { EmittedMessage } from "@pdpp/connector-protocol"; +import { REDDIT_JSON_ORIGIN } from "../../src/auto-login/reddit.ts"; import type { CollectContext } from "../../src/connector-runtime.ts"; import { makeRecordingEmit, type RecordingEmit } from "../../src/test-harness.ts"; @@ -169,6 +170,13 @@ function createMockRedditPage(fetchPath: (path: string) => Promise<{ status: num const { path } = args as { path: string }; return fetchPath(path); }, + // `redditFetch` calls `ensureRedditJsonOrigin` before every listing fetch, + // which reads `page.url()` and navigates when the origin is wrong. A mock + // without these reports the page as off-origin, so the fetch short-circuits + // to `status: 0` (`reddit_http_0`) and never reaches `evaluate` above. + // Same shape as reddit's own `createMockPageForFetch` oracle mock. + goto: (): Promise => Promise.resolve(null), + url: (): string => `${REDDIT_JSON_ORIGIN}/`, }; } @@ -1028,7 +1036,10 @@ export const KNOWN_UNEXERCISED_COVERAGE: ReadonlySet = new Set([ "google_calendar.events", "google_contacts.people", "google_contacts.contact_groups", - "google_maps.timeline_points", + // google_maps.timeline_points is no longer listed: it now declares + // `required: false` (matching its sibling timeline_segments), so it is not + // a required stream and this allowlist — which only tracks UNEXERCISED + // REQUIRED streams — must not claim it. "google_maps_data_portability.archive_jobs", // Google Takeout (REAL_UNLISTED_CONNECTORS): export-file snapshot-import // receipts, no driver yet. @@ -1041,10 +1052,11 @@ export const KNOWN_UNEXERCISED_COVERAGE: ReadonlySet = new Set([ "pocket.items", "heb.orders", "heb.order_items", - // iCal / iMessage (REAL_UNLISTED_CONNECTORS): file-based import receipts, no - // driver yet. + // iCal / iMessage / Signal (REAL_UNLISTED_CONNECTORS): file/subprocess-based + // import receipts, no driver yet. "ical.events", "imessage.messages", + "signal.messages", "notion.pages", "notion.databases", "oura.sleep", diff --git a/packages/polyfill-connectors/connectors/_conformance/manual-import-coverage-strategy.test.ts b/packages/polyfill-connectors/connectors/_conformance/manual-import-coverage-strategy.test.ts new file mode 100644 index 000000000..7de5f80f2 --- /dev/null +++ b/packages/polyfill-connectors/connectors/_conformance/manual-import-coverage-strategy.test.ts @@ -0,0 +1,99 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// Every manual-upload connector's data streams must declare +// `coverage_strategy: "snapshot_import_receipt"`. +// +// A manual upload is a one-time snapshot of a file the owner exported: the +// artifact is parsed once, in full, and nothing will run again. That is what +// `snapshot_import_receipt` names. `checkpoint_window` names the opposite +// shape -- a rolling cursor over a source that keeps producing -- and a +// connector whose freshness strategy is `manual_as_of` has no such window by +// construction. +// +// The two strategies happen to carry the SAME proof obligation today +// (`strategyBoundsWindowRatherThanCounting` in the shared evidence contract +// treats both as window-bounding), so this mislabel changes no verdict right +// now. It is pinned anyway because the label is the manifest's honest +// self-description of what kind of source this is, and because the two +// strategies are free to diverge later -- at which point a stale +// `checkpoint_window` on a finished import would start asking the projection +// for a window that can never close. +// +// Scoped to the manual-upload roster by `setup.modality`, so a newly added +// manual connector is covered without editing this test. + +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { test } from "node:test"; + +const MANIFESTS_DIR = new URL("../../manifests/", import.meta.url); + +interface ManifestStream { + readonly coverage_strategy?: string; + readonly freshness_strategy?: string; + readonly name: string; +} + +interface Manifest { + readonly connector_key?: string; + readonly setup?: { readonly modality?: string }; + readonly streams?: readonly ManifestStream[]; +} + +function readManifest(fileName: string): Manifest { + return JSON.parse(readFileSync(new URL(fileName, MANIFESTS_DIR), "utf8")) as Manifest; +} + +/** Every connector whose setup modality is a manual file upload. */ +function manualUploadManifests(): { manifest: Manifest; name: string }[] { + return readdirSync(new URL(MANIFESTS_DIR)) + .filter((file) => file.endsWith(".json")) + .map((file) => ({ manifest: readManifest(file), name: file })) + .filter(({ manifest }) => manifest.setup?.modality === "manual_or_upload"); +} + +test("every manual-upload connector is discoverable by setup modality", () => { + // `connector_key` is optional on the manifest type, so a missing key would + // otherwise sort as a silent `undefined` hole. Surface it as the literal + // string instead: the deepEqual below then fails loudly on the real defect + // rather than on an unexplained gap in the roster. + const found = manualUploadManifests() + .map(({ manifest }) => manifest.connector_key ?? "") + .sort((a, b) => a.localeCompare(b)); + // Guards the filter itself: if `setup.modality` were renamed, the roster + // would silently empty and every assertion below would vacuously pass. + assert.deepEqual(found, ["google-maps", "netflix-export", "whatsapp"]); +}); + +test("manual-upload data streams declare snapshot_import_receipt coverage", () => { + for (const { manifest, name } of manualUploadManifests()) { + for (const stream of manifest.streams ?? []) { + // `parent_detail_accounting` is a stricter per-item obligation (it owes + // a numerator that actually satisfies its denominator), so a stream that + // declares it is making a stronger claim, not evading this one. + if (stream.coverage_strategy === "parent_detail_accounting") { + continue; + } + assert.equal( + stream.coverage_strategy, + "snapshot_import_receipt", + `${name}: stream '${stream.name}' is a finished one-time import, so it must declare ` + + `snapshot_import_receipt (got '${String(stream.coverage_strategy)}')` + ); + } + } +}); + +test("no manual-upload stream claims a rolling checkpoint window", () => { + for (const { manifest, name } of manualUploadManifests()) { + for (const stream of manifest.streams ?? []) { + assert.notEqual( + stream.coverage_strategy, + "checkpoint_window", + `${name}: stream '${stream.name}' declares a rolling checkpoint window, but a manual ` + + "upload has no window to roll -- nothing will run again after the import" + ); + } + } +}); diff --git a/packages/polyfill-connectors/connectors/amazon/__fixtures__/orders-list-one-card-missing-order-id.html b/packages/polyfill-connectors/connectors/amazon/__fixtures__/orders-list-one-card-missing-order-id.html new file mode 100644 index 000000000..ab3592761 --- /dev/null +++ b/packages/polyfill-connectors/connectors/amazon/__fixtures__/orders-list-one-card-missing-order-id.html @@ -0,0 +1,51 @@ + + + +
+
+
+
    +
  • +
    + Order placed + January 15, 2024 +
    +
  • +
  • +
    + Total + $42.99 +
    +
  • +
+
+ Order # + 111-2222222-3333333 +
+
+
+
+ Delivered January 17 +
+
+
+
+
+
    +
  • +
    + Order placed + February 11, 2024 +
    +
  • +
+
+
+
+ Cancelled +
+
+
+
+ + diff --git a/packages/polyfill-connectors/connectors/amazon/index.ts b/packages/polyfill-connectors/connectors/amazon/index.ts index 28de4dd35..a0ff63577 100644 --- a/packages/polyfill-connectors/connectors/amazon/index.ts +++ b/packages/polyfill-connectors/connectors/amazon/index.ts @@ -36,6 +36,7 @@ import { type FingerprintCursor, openFingerprintCursor, recordFingerprint } from import { buildOrderItemRecord, buildOrderRecord, + countOrderCardsWithoutOrderId, mergeOrderItems, parseOrderDate, parseOrderDetailDom, @@ -134,6 +135,39 @@ interface EmptyListPageClassification { reason: string; } +/** + * What this connection already knows about the order history OF THE YEAR + * currently being scraped, threaded explicitly into the otherwise-pure + * empty-page classifier. + * + * `hasPriorOrders` is true when a prior run committed a `years[]` + * cursor whose `order_count` is greater than zero — the in-connector proof + * that Amazon once listed orders for this account IN THIS YEAR. + * + * PER-YEAR, not per-account, and that is the whole difference from H-E-B. + * H-E-B's list is one globally reverse-chronological feed, so any committed + * checkpoint contradicts any empty page. Amazon is year-partitioned: it + * renders a genuine, year-scoped "Looks like you didn't place an order in + * 2015." for a year the owner simply did not shop, on an account holding + * thousands of orders in other years. Account-wide evidence would abort every + * such year forever and break the incremental year sweep. Only the SAME + * year's own prior `order_count` can contradict that year's empty page. + */ +export interface PriorOrdersEvidence { + hasPriorOrders: boolean; +} + +/** Owner-facing message for the one classification whose whole point is to be + * read by a person. Says exactly what was observed and what was NOT + * concluded: neither selector drift nor a bot block is established, so + * neither is named. Stored records are untouched — this connector never + * deletes, tombstones, or overwrites on an empty page; it only declines to + * advance the year cursor. */ +export const AMAZON_EMPTY_AFTER_PRIOR_ORDERS_MESSAGE = + "Amazon reported no order history for a year in which PDPP previously collected orders. " + + "Your stored orders are retained and untouched. This run was stopped instead of recording " + + "an empty history, because a page showing no orders cannot prove those orders are gone."; + // Navigation timeouts + pacing knobs const NAV_TIMEOUT_MS = 30_000; const DEEP_PROBE_WAIT_MS = 15_000; @@ -446,12 +480,17 @@ async function fetchOrderDetail(page: Page, orderId: string): Promise { +interface PageExtractionResult { + droppedCardCount: number; + orders: ListPageOrder[]; +} + +async function extractOrdersOnPage(page: Page): Promise { try { const html = await readPageContentWithin(page); - return parseOrdersListDom(html); + return { droppedCardCount: countOrderCardsWithoutOrderId(html), orders: parseOrdersListDom(html) }; } catch { - return []; + return { droppedCardCount: 0, orders: [] }; } } @@ -1038,19 +1077,80 @@ export async function emitOrderAndItems( } } if (deps.wantsItems) { - for (const merged of mergeOrderItems(listOrder, detail)) { - await deps.emitRecord("order_items", buildOrderItemRecord(listOrder.orderId, orderDate, merged)); + const merged = mergeOrderItems(listOrder, detail); + for (const item of merged) { + await deps.emitRecord("order_items", buildOrderItemRecord(listOrder.orderId, orderDate, item)); } + await emitItemCountReconciliation(deps, listOrder, detail, merged.length); } } +/** + * Reconcile the items we emitted against the count the order page asserted, and + * report a shortfall rather than letting it pass. + * + * The denominator is measured at the parse boundary — `detail.items` is what + * the order-detail page listed, counted before the merge and independently of + * what `mergeOrderItems` decided to emit. Deduplication across the two surfaces + * means the merged list should never be SHORTER than the detail page's own + * list; if it is, an item the page showed us did not survive into a record. + * + * Only runs when a detail page was actually fetched. Without it there is no + * assertion to reconcile against — `resolveItemCount` reports null for exactly + * that case, and inventing a denominator from the list card would fabricate a + * shortfall on every 3+ item order (the list card renders no item titles once + * Amazon collapses them behind "+N more items"). + */ +async function emitItemCountReconciliation( + deps: EmitDeps, + listOrder: ListPageOrder, + detail: OrderDetail | null, + emittedItemCount: number +): Promise { + const declared = detail?.items?.length; + if (declared === undefined || emittedItemCount >= declared) { + return; + } + await deps.emit({ + type: "SKIP_RESULT", + stream: "order_items", + reason: "item_count_shortfall", + message: `order ${listOrder.orderId} listed ${declared} items on its detail page but only ${emittedItemCount} became records`, + diagnostics: { + order_id: listOrder.orderId, + declared_item_count: declared, + emitted_item_count: emittedItemCount, + }, + }); +} + /** * Run list-page extraction through the zod shape-check. Orders that fail * the shape-check become SKIP_RESULT events; the successful subset is * returned in source order. */ async function extractAndShapeCheckOrders(page: Page, emit: EmitFn): Promise { - const rawOrders = await extractOrdersOnPage(page); + const { droppedCardCount, orders: rawOrders } = await extractOrdersOnPage(page); + // Only report when the page also proved at least one real order: `.order-card`/ + // `.js-order-card` is reused by Amazon's "Buy it again" recommendation carousel on + // a genuinely empty year (see the `orders-list-empty-year-with-carousel` fixture), + // so a page with ZERO real orders and only carousel cards is not evidence of loss — + // `reportEmptyPageDiagnostics` already owns that case. Gating on `rawOrders.length + // > 0` keeps this diagnostic scoped to "some cards parsed, but not all of them." + if (droppedCardCount > 0 && rawOrders.length > 0) { + // A card matched `.order-card`/`.js-order-card` but `parseOrderCard` + // could not find a `.yohtmlc-order-id` on it, so it never became a raw + // order and never reached the shape-check below — with no fix here that + // card is lost with zero trace (no SKIP_RESULT, no coverage-considered + // id, no rejection). This is the only signal that loss ever happened. + await emit({ + type: "SKIP_RESULT", + stream: "orders", + reason: "list_page_order_id_not_found", + message: `${droppedCardCount} order card(s) on this page had no parseable .yohtmlc-order-id and were dropped before the shape-check`, + diagnostics: { dropped_card_count: droppedCardCount }, + }); + } const orders: ListPageOrder[] = []; for (const r of rawOrders) { const parsed = listPageOrderShape.safeParse(r); @@ -1077,7 +1177,13 @@ async function extractAndShapeCheckOrders(page: Page, emit: EmitFn): Promise { +async function reportEmptyPageDiagnostics( + page: Page, + year: number, + startIndex: number, + emit: EmitFn, + priorOrdersEvidence: PriorOrdersEvidence +): Promise { let diag: ListPageDiagnostics; try { diag = await page.evaluate((noOrdersTextPattern): ListPageDiagnostics => { @@ -1108,7 +1214,7 @@ async function reportEmptyPageDiagnostics(page: Page, year: number, startIndex: }); throw new Error("amazon_empty_list_page_renderer_diagnostics_failed", { cause: error }); } - const classification = classifyEmptyListPageDiagnostics(diag, startIndex); + const classification = classifyEmptyListPageDiagnostics(diag, startIndex, priorOrdersEvidence); if (classification.action === "terminal") { return; } @@ -1134,8 +1240,13 @@ async function reportEmptyPageDiagnostics(page: Page, year: number, startIndex: type: "SKIP_RESULT", stream: "orders", reason: classification.reason, - message: `Year ${year} startIndex=${startIndex}: empty Amazon list page is not a proven terminal page; refusing to advance the cursor.`, - diagnostics: diag ? redactAmazonListPageDiagnostics(diag) : { missing_diagnostics: true }, + message: + classification.reason === "amazon_empty_history_after_prior_orders" + ? AMAZON_EMPTY_AFTER_PRIOR_ORDERS_MESSAGE + : `Year ${year} startIndex=${startIndex}: empty Amazon list page is not a proven terminal page; refusing to advance the cursor.`, + diagnostics: diag + ? { ...redactAmazonListPageDiagnostics(diag), has_prior_orders: priorOrdersEvidence.hasPriorOrders, year } + : { missing_diagnostics: true }, }); } throw new Error(`amazon_empty_list_page_${classification.reason}`); @@ -1149,7 +1260,8 @@ export function redactAmazonListPageDiagnostics(diag: ListPageDiagnostics): List export function classifyEmptyListPageDiagnostics( diag: ListPageDiagnostics | null, - startIndex: number + startIndex: number, + priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false } ): EmptyListPageClassification { if (!diag) { return { action: "abort", reason: "renderer_diagnostics_failed" }; @@ -1161,6 +1273,35 @@ export function classifyEmptyListPageDiagnostics( if ((diag.any_card > 0 || diag.any_order_header > 0) && diag.order_cards === 0) { return { action: "abort", reason: "selector_drift" }; } + // A year that has already yielded orders can never prove itself empty. + // `hasPriorOrders` is this connection's own prior `years[].order_count` + // for the SAME year being scraped — durable evidence that Amazon previously + // listed orders there — threaded in explicitly by `scrapeListPage` rather + // than read from ambient state, so this branch stays pure and unit-testable. + // + // Without this check, a year holding hundreds of stored orders could finish + // as {action:"terminal", reason:"no_orders_text"}, letting the run advance + // the year cursor and report covered:0/considered:0 — replacing a measured + // coverage claim with a fabricated proven-zero. The two causes are + // indistinguishable from the page alone: Amazon may have purged the history + // upstream (making our stored copy the only copy), or the page may render + // empty for a degraded session. So the run fails loudly and lets a human + // decide, rather than guessing. + // + // Scoped to `startIndex === 0` because only the FIRST page of a year makes + // the claim "this year is empty". A later page coming back empty is ordinary + // pagination exhaustion on a year that plainly did yield orders — the rows + // preceding it were just collected on this very run — and must keep falling + // through to `pagination_exhausted` below. + // + // Ordering: BELOW the auth/challenge and selector-drift checks (an + // established block or a real markup change is the more specific and more + // actionable diagnosis, and must not be relabelled), and ABOVE the + // `no_orders_text` branch, so Amazon's own empty-state copy cannot + // short-circuit past it. + if (startIndex === 0 && diag.no_orders_text === "true" && priorOrdersEvidence.hasPriorOrders) { + return { action: "abort", reason: "amazon_empty_history_after_prior_orders" }; + } if (diag.no_orders_text === "true") { return { action: "terminal", reason: "no_orders_text" }; } @@ -1180,7 +1321,8 @@ export async function scrapeListPage( capture: CaptureDep, year: number, startIndex: number, - emit: EmitFn + emit: EmitFn, + priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false } ): Promise { const url = `https://www.amazon.com/your-orders/orders?timeFilter=year-${year}&startIndex=${startIndex}`; try { @@ -1212,7 +1354,7 @@ export async function scrapeListPage( } const orders = await extractAndShapeCheckOrders(page, emit); if (orders.length === 0) { - await reportEmptyPageDiagnostics(page, year, startIndex, emit); + await reportEmptyPageDiagnostics(page, year, startIndex, emit, priorOrdersEvidence); } return orders; } @@ -1454,19 +1596,43 @@ export function shouldEmitTrailingOrdersState( return !ordersStateEmitted && hydratedOrders.size > 0; } +/** + * Derive the prior-orders evidence for ONE year from this connection's stored + * `years` cursor. Exported and pure because `collect()` lives inside the + * `isMainModule` block and cannot be driven from a test — without this seam + * the year-state-to-evidence link would be the one untested link in the + * chain, and a mutation that hardcodes `false` here (silently disarming the + * guard for every connection) would go unnoticed. + * + * The evidence is `order_count > 0`, not the mere existence of a year cursor. + * A year that was scraped and legitimately held zero orders commits a cursor + * with `order_count: 0`; that year must stay free to report empty again on + * every later run. Only a year that actually yielded orders can contradict a + * later "no orders" page. + */ +export function priorOrdersEvidenceForYear(yearsState: YearsCursor, year: number): PriorOrdersEvidence { + return { hasPriorOrders: (yearsState[String(year)]?.order_count ?? 0) > 0 }; +} + /** * Scrape every list page for one year and emit records. Returns both the total * order count seen for the year (used for freeze-once-stable policy) and the * count of rows we could not emit because their order date was unparseable. */ -async function runYear(page: Page, deps: EmitDeps, flags: RunFlags, year: number): Promise { +async function runYear( + page: Page, + deps: EmitDeps, + flags: RunFlags, + year: number, + priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false } +): Promise { let startIndex = 0; let pageCount = 0; let yearOrderCount = 0; let unparseableDateCount = 0; while (pageCount < PAGE_LIMIT) { await deps.progress(`Amazon year ${year}: scanning page ${pageCount + 1}`, { stream: "orders" }); - const orders = await scrapeListPage(page, deps.capture, year, startIndex, deps.emit); + const orders = await scrapeListPage(page, deps.capture, year, startIndex, deps.emit, priorOrdersEvidence); if (orders.length === 0) { await deps.progress(`Amazon year ${year}: no more orders after ${yearOrderCount} seen`, { stream: "orders" }); break; @@ -1719,7 +1885,18 @@ if (isMainModule(import.meta.url)) { continue; } - const { orderCount: yearOrderCount, unparseableDateCount } = await runYear(page, deps, flags, year); + // A prior `years[].order_count > 0` is this connection's own + // record that Amazon has listed orders for this account in THIS year + // before. It is what makes a later "no orders in " page a + // contradiction to escalate rather than a result to trust. Read here, + // next to the year cursor it derives from, and passed down explicitly. + const { orderCount: yearOrderCount, unparseableDateCount } = await runYear( + page, + deps, + flags, + year, + priorOrdersEvidenceForYear(yearsState, year) + ); await applyYearCompletionState({ newYearsState, diff --git a/packages/polyfill-connectors/connectors/amazon/integration.test.ts b/packages/polyfill-connectors/connectors/amazon/integration.test.ts index 7c0bcf6c1..a10edff74 100644 --- a/packages/polyfill-connectors/connectors/amazon/integration.test.ts +++ b/packages/polyfill-connectors/connectors/amazon/integration.test.ts @@ -53,6 +53,7 @@ import { type OrderItemsCoverage, type OrdersCoverage, planIncrementalYears, + priorOrdersEvidenceForYear, processListOrder, type RunFlags, readPageContentWithin, @@ -424,13 +425,33 @@ test("collect path does not advance a year cursor after unparseable order-date d ); }); -test("amazon manifest: successful manual runs have a bounded freshness window", () => { +test("amazon manifest: successful runs have a bounded freshness window", () => { + // The bounded window is what this test is for: without + // `maximum_staleness_seconds`, freshness is `unknown` and a successful run + // cannot project `current`. + // + // The mode assertion moved from a hard-coded "manual" to the connector's + // own declared facts. Amazon declares `background_safe: true` — the + // browser session persists after the owner's first login — and mode is now + // DERIVED from that (see reference-implementation/runtime/ + // refresh-mode-derivation.ts). Pinning "manual" here contradicted the + // manifest's own background-safety claim. const manifest = JSON.parse(readFileSync(AMAZON_MANIFEST_PATH, "utf8")) as { - capabilities?: { refresh_policy?: { maximum_staleness_seconds?: number; recommended_mode?: string } }; + capabilities?: { + refresh_policy?: { + background_safe?: boolean; + interaction_posture?: string; + maximum_staleness_seconds?: number; + recommended_mode?: string; + }; + }; }; const policy = manifest.capabilities?.refresh_policy; - assert.equal(policy?.recommended_mode, "manual"); assert.equal(policy?.maximum_staleness_seconds, 86_400); + // Amazon's first login is owner-present, and the session then persists. + assert.equal(policy?.interaction_posture, "otp_likely"); + assert.equal(policy?.background_safe, true); + assert.equal(policy?.recommended_mode, "automatic"); }); // ─── Empty list-page classification ────────────────────────────────────── @@ -548,6 +569,40 @@ test("scrapeListPage: a failed list navigation cannot reuse the prior page as a assert.doesNotMatch(JSON.stringify(messages), /orders-list-minimal|B0/); }); +test("scrapeListPage: a card with no parseable order id is dropped AND reported, alongside a card that survives", async () => { + // The "prove the drop happens today, is invisible today" half of the pair. + // One card has a normal `.yohtmlc-order-id`; the other (modeling a + // never-shipped/cancelled order rendering under a variant Amazon uses for + // that order type) has none. Before this fix, the second card vanished + // from `orders` with zero SKIP_RESULT anywhere — see + // `countOrderCardsWithoutOrderId` in parsers.ts for the mechanism. + const html = readFileSync( + new URL("./__fixtures__/orders-list-one-card-missing-order-id.html", import.meta.url), + "utf8" + ); + const messages: EmittedMessage[] = []; + const page = Object.assign({} as Page, { + content: (): Promise => Promise.resolve(html), + goto: (): Promise => Promise.resolve(null), + locator: (): { first: () => { waitFor: () => Promise } } => ({ + first: () => ({ waitFor: (): Promise => Promise.resolve(null) }), + }), + }); + + const orders = await scrapeListPage(page, null, 2024, 0, (message) => { + messages.push(message); + return Promise.resolve(); + }); + + assert.equal(orders.length, 1, "only the card with a parseable order id survives"); + assert.equal(orders[0]?.orderId, "111-2222222-3333333"); + + const skip = messages.find((message) => message.type === "SKIP_RESULT"); + assert.ok(skip, "the drop must be reported, not silent"); + assert.equal(skip?.reason, "list_page_order_id_not_found"); + assert.deepEqual(skip?.diagnostics, { dropped_card_count: 1 }); +}); + test("scrapeListPage: a page-2 renderer diagnostic failure aborts without STATE or coverage", async () => { const messages: EmittedMessage[] = []; const page = Object.assign({} as Page, { @@ -579,6 +634,241 @@ test("scrapeListPage: a page-2 renderer diagnostic failure aborts without STATE assert.doesNotMatch(JSON.stringify(messages), /private renderer|owner label|amazon\.com/); }); +// ─── Proven-empty regression guard (prior-orders evidence) ──────────────── +// +// A YEAR that has already yielded orders must never be able to complete a run +// as "proven empty". Amazon's own year-scoped empty copy is trustworthy for a +// year the owner did not shop; for a year we have already measured, the same +// page is a contradiction, not a result. +// +// The evidence is PER-YEAR, unlike H-E-B's account-wide checkpoint, because +// Amazon is year-partitioned: an account with thousands of orders legitimately +// renders "Looks like you didn't place an order in 2015." for an unshopped +// year, and that year must keep reporting empty forever. + +const RESOLVED_EMPTY_YEAR_DIAG = (): ListPageDiagnostics => makeEmptyPageDiagnostics({ no_orders_text: "true" }); + +/** + * A Page stub serving Amazon's real captured empty-year list page. `content()` + * feeds the DOM parser (which finds no order cards, the trigger for the + * empty-page branch) and `evaluate()` returns diagnostics derived from the + * SAME fixture text, so `no_orders_text` is decided by Amazon's own copy + * rather than asserted by the test. + */ +function makeEmptyYearPageStub(): Page { + const html = readFileSync(AMAZON_EMPTY_YEAR_FIXTURE, "utf8"); + const visibleText = html.replace(/<[^>]+>/g, " "); + return Object.assign({} as Page, { + content: (): Promise => Promise.resolve(html), + evaluate: (): Promise => + Promise.resolve( + makeEmptyPageDiagnostics({ + no_orders_text: new RegExp(AMAZON_NO_ORDERS_TEXT_PATTERN, "i").test(visibleText).toString(), + }) + ), + goto: (): Promise => Promise.resolve(null), + locator: (): { first: () => { waitFor: () => Promise } } => ({ + first: () => ({ waitFor: (): Promise => Promise.resolve(null) }), + }), + screenshot: (): Promise => Promise.resolve(null), + }); +} + +test("classifyEmptyListPageDiagnostics: source-reported empty for a year with NO prior orders stays proven-empty", () => { + // Preserves existing behavior. A year the owner did not shop is the case + // where zero coverage is an honest measurement. + assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0, { hasPriorOrders: false }), { + action: "terminal", + reason: "no_orders_text", + }); +}); + +test("classifyEmptyListPageDiagnostics: the prior-orders argument defaults to absent, so callers cannot silently opt in", () => { + assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0), { + action: "terminal", + reason: "no_orders_text", + }); +}); + +test("classifyEmptyListPageDiagnostics: source-reported empty for a year WITH prior orders aborts instead of proving zero", () => { + // The defect this guard closes: without it, this exact input returned + // {action:"terminal", reason:"no_orders_text"}, letting a year holding + // hundreds of stored orders advance its cursor on a fabricated proven-zero. + assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0, { hasPriorOrders: true }), { + action: "abort", + reason: "amazon_empty_history_after_prior_orders", + }); +}); + +test("classifyEmptyListPageDiagnostics: a legitimately empty year on an account with orders in OTHER years stays terminal", () => { + // THE year-partition requirement, driven through Amazon's real 2015 + // empty-year capture. `priorOrdersEvidenceForYear` scopes evidence to the + // year being scraped, so 2015 carries `hasPriorOrders: false` even though + // 2024 and 2026 hold orders. Account-wide evidence would abort this year on + // every run forever and break the incremental year sweep. + const yearsState = { + "2024": { frozen: true, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 312 }, + "2026": { frozen: false, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 44 }, + }; + const html = readFileSync(AMAZON_EMPTY_YEAR_FIXTURE, "utf8"); + const visibleText = html.replace(/<[^>]+>/g, " "); + const noOrdersRe = new RegExp(AMAZON_NO_ORDERS_TEXT_PATTERN, "i"); + assert.match(visibleText, noOrdersRe, "the fixture must still carry Amazon's year-scoped empty copy"); + + assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2015), { hasPriorOrders: false }); + assert.deepEqual( + classifyEmptyListPageDiagnostics( + makeEmptyPageDiagnostics({ any_card: 1, no_orders_text: "true", order_cards: 1 }), + 0, + priorOrdersEvidenceForYear(yearsState, 2015) + ), + { action: "terminal", reason: "no_orders_text" }, + "an unshopped year must stay proven-empty on an account with orders elsewhere" + ); + + // Same account, same run, a year that DID yield orders: opposite verdict. + assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2024), { hasPriorOrders: true }); + assert.deepEqual( + classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0, priorOrdersEvidenceForYear(yearsState, 2024)), + { action: "abort", reason: "amazon_empty_history_after_prior_orders" } + ); +}); + +test("classifyEmptyListPageDiagnostics: an auth/challenge page keeps its own reason even when the year has prior orders", () => { + // Ordering guard, upper half. The auth check stays ABOVE the new branch: + // when a challenge is actually established, that is the more specific and + // more actionable diagnosis, and it must not be relabelled. + const authOverrides: Partial[] = [{ captcha: "true" }, { sign_in_form: true }]; + for (const override of authOverrides) { + assert.deepEqual( + classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics({ no_orders_text: "true", ...override }), 0, { + hasPriorOrders: true, + }), + { action: "abort", reason: "source_auth_or_challenge" } + ); + } +}); + +test("classifyEmptyListPageDiagnostics: selector drift keeps its own reason even when the year has prior orders", () => { + // Ordering guard, middle. Real markup change stays diagnosable as drift. + assert.deepEqual( + classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics({ any_order_header: 2, no_orders_text: "true" }), 0, { + hasPriorOrders: true, + }), + { action: "abort", reason: "selector_drift" } + ); +}); + +test("classifyEmptyListPageDiagnostics: prior orders do not relabel a page that never claimed to be empty", () => { + // Ordering guard, lower half. The new branch is gated on `no_orders_text`, + // so the existing terminal/abort verdicts for pages that make no empty claim + // are untouched — the guard adds a failure mode, it does not swallow others. + assert.deepEqual(classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics(), 10, { hasPriorOrders: true }), { + action: "terminal", + reason: "pagination_exhausted", + }); + assert.deepEqual(classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics(), 0, { hasPriorOrders: true }), { + action: "abort", + reason: "empty_first_page_without_terminal_signal", + }); +}); + +test("classifyEmptyListPageDiagnostics: a later empty page in a year that just yielded orders is ordinary exhaustion", () => { + // The guard is scoped to startIndex === 0 because only the FIRST page of a + // year claims the year is empty. Amazon can serve its empty-state copy on a + // trailing page; the rows before it were collected on THIS run, so this is + // pagination exhaustion, not a contradiction. Without the startIndex scope, + // every incremental run over a year with prior orders would abort on its + // last page — the guard would break the connector for exactly the accounts + // it exists to protect. + assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 10, { hasPriorOrders: true }), { + action: "terminal", + reason: "no_orders_text", + }); +}); + +test("priorOrdersEvidenceForYear: a prior year order_count > 0 is what arms the guard", () => { + // Pins the year-state-to-evidence link that collect() depends on. Without + // this test, hardcoding `hasPriorOrders: false` in collect() would disarm + // the guard for every connection while every other test still passed. + const yearsState = { + "2019": { frozen: true, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 0 }, + "2024": { frozen: true, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 312 }, + }; + assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2024), { hasPriorOrders: true }); + // A year scraped and found genuinely empty commits `order_count: 0`. It must + // stay free to report empty again — this is the year-partition case that + // makes count, not cursor existence, the right evidence. + assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2019), { hasPriorOrders: false }); + // A never-scraped year has no cursor at all. + assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2015), { hasPriorOrders: false }); +}); + +test("scrapeListPage: Amazon's empty-year page aborts when this year already yielded orders", async () => { + // Integration half, through the same live empty-year capture the + // proven-empty test uses. Same page, same markup, opposite verdict — the + // only difference is that this year has a prior non-zero order count. + const messages: EmittedMessage[] = []; + const page = makeEmptyYearPageStub(); + + await assert.rejects( + scrapeListPage( + page, + null, + 2024, + 0, + (message) => { + messages.push(message); + return Promise.resolve(); + }, + { hasPriorOrders: true } + ), + /amazon_empty_list_page_amazon_empty_history_after_prior_orders/, + "a year with prior orders cannot be proven empty by a page render" + ); + + // The failure must be legible to the owner, and must not blame anything the + // page does not establish. + const skip = messages.find( + (message) => message.type === "SKIP_RESULT" && message.reason === "amazon_empty_history_after_prior_orders" + ) as { diagnostics: Record; message: string } | undefined; + assert.ok(skip, "the abort must surface a SKIP_RESULT the owner can read"); + assert.match(skip.message, /previously collected orders/); + assert.match(skip.message, /retained and untouched/); + assert.doesNotMatch(skip.message, /selector|drift/i, "selector drift is not established and must not be blamed"); + assert.doesNotMatch(skip.message, /block|bot|captcha/i, "a bot block is not established and must not be blamed"); + assert.equal(skip.diagnostics.has_prior_orders, true); + assert.equal(skip.diagnostics.year, 2024); + + // Nothing may be recorded, no cursor advanced, no coverage claimed. The + // connector protocol has no delete or tombstone message, so proving it + // emitted no RECORD and no STATE proves the stored copy and the prior year + // cursor are both untouched. + assert.deepEqual( + [...new Set(messages.map((message) => message.type))].sort(), + ["SKIP_RESULT"], + "the abort's only durable output is the owner-facing SKIP_RESULT" + ); +}); + +test("scrapeListPage: the same empty-year page still succeeds as proven-empty for a year with no prior orders", async () => { + // The preservation half of the pair above: identical page, no prior orders, + // no throw, no diagnostic. A first-ever run on a genuinely empty year is + // unaffected by the guard. + const messages: EmittedMessage[] = []; + const orders = await scrapeListPage(makeEmptyYearPageStub(), null, 2015, 0, (message) => { + messages.push(message); + return Promise.resolve(); + }); + + assert.deepEqual(orders, [], "an empty year yields no orders and does not throw"); + assert.equal( + messages.some((message) => message.type === "SKIP_RESULT"), + false, + "a proven-empty year emits no skip diagnostic" + ); +}); + // ─── planIncrementalYears ───────────────────────────────────────────────── test("planIncrementalYears: no prior state → all discovered years planned (initial backfill)", () => { @@ -2038,3 +2328,109 @@ test("a run with zero considered orders still emits a zero-required DETAIL_COVER assert.deepEqual(cov.hydrated_keys, []); assert.equal(findDetailGaps(protocolMessages).length, 0, "zero considered orders produce no gaps"); }); + +// ─── item_count reconciliation ────────────────────────────────────────── + +/** + * `item_count` is not a provider assertion — Amazon never states a count in any + * markup this connector reads. The number is a count of item elements parsed + * from two surfaces, so the only reconcilable claim is the detail page's own + * item list: every item that page showed must survive into a record. A merged + * list SHORTER than the detail list means an item was seen and then lost. + */ +function findItemCountShortfalls(protocolMessages: readonly unknown[]): Record[] { + return protocolMessages.filter( + (m) => + (m as { type?: string; reason?: string }).type === "SKIP_RESULT" && + (m as { reason?: string }).reason === "item_count_shortfall" + ) as Record[]; +} + +test("emitOrderAndItems: every detail-page item becoming a record reports no shortfall", async () => { + const { deps, protocolMessages, emitted } = makeRecordingDeps(); + const detail = makeDetail({ + items: [ + makeDetailItem({ asin: "B000000001", name: "Widget A" }), + makeDetailItem({ asin: "B000000002", name: "Widget B" }), + ], + }); + await emitOrderAndItems(deps, makeListOrder({ items: [] }), detail, "2026-01-05"); + + assert.equal(emitted.filter((r) => r.stream === "order_items").length, 2); + assert.equal(findItemCountShortfalls(protocolMessages).length, 0, "a complete order must not report a gap"); +}); + +test("emitOrderAndItems: a detail item that never becomes a record is reported as a shortfall", async () => { + // Two detail items collapse to one record because they carry the same + // identity, so one of the items the page showed us is not in the database. + // Before this check that loss was silent. + const { deps, protocolMessages, emitted } = makeRecordingDeps(); + const detail = makeDetail({ + items: [ + makeDetailItem({ asin: "B000000001", name: "Widget A" }), + makeDetailItem({ asin: "B000000001", name: "Widget A" }), + ], + }); + await emitOrderAndItems(deps, makeListOrder({ items: [] }), detail, "2026-01-05"); + + const emittedItems = emitted.filter((r) => r.stream === "order_items").length; + const shortfalls = findItemCountShortfalls(protocolMessages); + assert.equal(emittedItems, 1, "the two detail rows collapsed to one record"); + assert.equal(shortfalls.length, 1, "the lost item must be surfaced"); + assert.deepEqual(shortfalls[0]?.diagnostics, { + order_id: makeListOrder().orderId, + declared_item_count: 2, + emitted_item_count: 1, + }); +}); + +test("emitOrderAndItems: no detail page means no denominator and no fabricated shortfall", async () => { + // A 3+ item order whose list card collapsed its titles reaches here with an + // empty list and no detail. Counting the list card as the denominator would + // invent a gap on every such order. + const { deps, protocolMessages } = makeRecordingDeps(); + await emitOrderAndItems(deps, makeListOrder({ items: [] }), null, "2026-01-05"); + + assert.equal( + findItemCountShortfalls(protocolMessages).length, + 0, + "an unfetched detail page is an unknown, not a proven shortfall" + ); +}); + +test("emitOrderAndItems: the denominator is the detail page, never the list card", async () => { + // Pins WHICH surface supplies the claim. The list card here shows two items + // that dedupe to one record — a merge outcome, not a loss, because the list + // card is not an authority on how many items an order has. Only the detail + // page's own item list is a claim worth holding the connector to; falling + // back to the list card manufactures a gap out of ordinary deduplication. + const { deps, protocolMessages, emitted } = makeRecordingDeps(); + const listOrder = makeListOrder({ + items: [ + { asin: "B000000001", name: "Widget A", url: null }, + { asin: "B000000001", name: "Widget A", url: null }, + ], + }); + await emitOrderAndItems(deps, listOrder, null, "2026-01-05"); + + assert.equal(emitted.filter((r) => r.stream === "order_items").length, 1, "the duplicate list rows collapse"); + assert.equal( + findItemCountShortfalls(protocolMessages).length, + 0, + "the list card is not a provider claim, so its count must not become a denominator" + ); +}); + +test("emitOrderAndItems: items out of scope suppress the reconciliation entirely", async () => { + const { deps, protocolMessages } = makeRecordingDeps({ wantsItems: false }); + const detail = makeDetail({ + items: [makeDetailItem({ asin: "B000000001" }), makeDetailItem({ asin: "B000000001" })], + }); + await emitOrderAndItems(deps, makeListOrder({ items: [] }), detail, "2026-01-05"); + + assert.equal( + findItemCountShortfalls(protocolMessages).length, + 0, + "a stream nobody asked for cannot report a gap against records it never tried to emit" + ); +}); diff --git a/packages/polyfill-connectors/connectors/amazon/item-count-honesty.test.ts b/packages/polyfill-connectors/connectors/amazon/item-count-honesty.test.ts new file mode 100644 index 000000000..0aa59f788 --- /dev/null +++ b/packages/polyfill-connectors/connectors/amazon/item-count-honesty.test.ts @@ -0,0 +1,153 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * `item_count` had no independent source. It was + * `Math.max(list.items.length, detail.items.length)` against a non-nullable + * `min(0)` schema, so an order whose detail page was never fetched asserted a + * confident `item_count: 0` — indistinguishable from a genuinely empty order. + * + * That mattered because the list card only renders item titles for SMALL + * orders; Amazon collapses 3+ item orders behind a "+N more items" affordance + * that carries no per-item markup. For those orders the list contributes 0 and + * the count rests entirely on the detail page, which the connector defers under + * its per-run attempt budget, its temporary-failure cap, or a latched session + * repair. + * + * The live signature is unambiguous: across 1,183 collected orders, + * `item_count === 0` never occurs on an order holding 1 or 2 item records, and + * occurs on 53 orders holding 3-7 — every one of which also has a null + * `shipping_address_summary`, proving the detail page (not the parse) was + * missing. Those records claimed an empty order while the database held their + * items. + * + * These tests pin the distinction the old shape could not express: a count of + * zero must mean "we looked and there were none", never "we did not look". + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { buildOrderRecord } from "./parsers.ts"; +import { validateRecord } from "./schemas.ts"; +import type { DetailItem, ListPageOrder, OrderDetail } from "./types.ts"; + +function makeListOrder(overrides: Partial = {}): ListPageOrder { + return { + orderId: "111-2222222-3333333", + orderDateRaw: "January 15, 2024", + orderTotal: "$10.00", + deliveryStatus: "Delivered Jan 17", + items: [], + ...overrides, + }; +} + +function makeDetailItem(overrides: Partial = {}): DetailItem { + return { + asin: null, + name: "", + url: null, + unit_price: null, + quantity: 1, + seller: null, + item_image_url: null, + refund_status: null, + ...overrides, + }; +} + +function makeDetail(overrides: Partial = {}): OrderDetail { + return { + status_detail: null, + recipient_name: null, + shipping_address_summary: null, + payment_method_summary: null, + grand_total: null, + gift_order: false, + digital_order: false, + items: [], + ...overrides, + }; +} + +function build(listOrder: ListPageOrder, detail: OrderDetail | null) { + return buildOrderRecord(listOrder, detail, "2024-01-15", "2024-01-20T00:00:00Z"); +} + +test("amazon item_count: an unfetched detail page reports unknown, not an empty order", () => { + // The exact live shape: a 3+ item order whose list card collapsed its titles + // and whose detail fetch was deferred. Before this, the record asserted 0. + const rec = build(makeListOrder({ items: [] }), null); + + assert.equal(rec.item_count, null, "no surface saw an item and none was fetched — the count is unknown"); + assert.notEqual(rec.item_count, 0, "0 would claim a proven-empty order we never actually looked at"); +}); + +test("amazon item_count: a fetched detail page with no items is a proven-empty order", () => { + // We DID look at the surface that carries items, and it carried none. That is + // a fact, and it must stay distinguishable from the case above. + const rec = build(makeListOrder({ items: [] }), makeDetail({ items: [] })); + + assert.equal(rec.item_count, 0, "a successful detail fetch that found nothing proves zero"); +}); + +test("amazon item_count: the detail page count wins when the list card collapsed its titles", () => { + // The 3+ item case with the detail page present: list contributes 0, detail + // carries the real count. + const rec = build( + makeListOrder({ items: [] }), + makeDetail({ + items: [ + makeDetailItem({ asin: "B01", name: "A" }), + makeDetailItem({ asin: "B02", name: "B" }), + makeDetailItem({ asin: "B03", name: "C" }), + ], + }) + ); + + assert.equal(rec.item_count, 3); +}); + +test("amazon item_count: the list card alone still counts when no detail was fetched", () => { + // A small order whose titles the list card did render. We saw real items, so + // the count is known even without a detail page — this must not regress to + // null just because `detail` is absent. + const rec = build( + makeListOrder({ + items: [ + { asin: "B01", name: "A", url: null }, + { asin: "B02", name: "B", url: null }, + ], + }), + null + ); + + assert.equal(rec.item_count, 2, "observed items are observed regardless of which surface showed them"); +}); + +test("amazon item_count: the larger of the two surfaces is kept", () => { + const rec = build( + makeListOrder({ items: [{ asin: "B01", name: "A", url: null }] }), + makeDetail({ items: [makeDetailItem({ asin: "B01" }), makeDetailItem({ asin: "B02" })] }) + ); + + assert.equal(rec.item_count, 2, "the detail page saw more than the list card"); +}); + +test("amazon item_count: a null count is accepted by the orders schema", () => { + // The schema was non-nullable, which is what forced the unknown to be encoded + // as 0 in the first place. If this regresses, the honest value cannot be + // emitted at all and the connector silently falls back to the old lie. + const rec = build(makeListOrder({ items: [] }), null); + const result = validateRecord("orders", rec as unknown as Record); + + assert.equal(result.ok, true, `a null item_count must validate: ${JSON.stringify(result.ok ? [] : result.issues)}`); +}); + +test("amazon item_count: a real count still validates", () => { + const rec = build(makeListOrder({ items: [{ asin: "B01", name: "A", url: null }] }), makeDetail()); + const result = validateRecord("orders", rec as unknown as Record); + + assert.equal(result.ok, true); + assert.equal(rec.item_count, 1); +}); diff --git a/packages/polyfill-connectors/connectors/amazon/parsers.test.ts b/packages/polyfill-connectors/connectors/amazon/parsers.test.ts index e874fc61f..ffe8d9468 100644 --- a/packages/polyfill-connectors/connectors/amazon/parsers.test.ts +++ b/packages/polyfill-connectors/connectors/amazon/parsers.test.ts @@ -9,6 +9,7 @@ import { fileURLToPath } from "node:url"; import { buildOrderItemRecord, buildOrderRecord, + countOrderCardsWithoutOrderId, itemId, mergeDetailByKey, mergeOrderItems, @@ -152,6 +153,62 @@ test("parseOrdersListDom: empty page returns []", () => { assert.deepEqual(parseOrdersListDom(""), []); }); +// A card that matches `.order-card`/`.js-order-card` but has no +// `.yohtmlc-order-id` (e.g. a never-shipped/cancelled Subscribe & Save order +// rendering under a variant Amazon uses for that order type) is dropped by +// `parseOrdersListDom` with no signal anywhere else in the connector: it +// never reaches the shape-check, so it produces no SKIP_RESULT, no +// coverage-considered id, and no rejection. `countOrderCardsWithoutOrderId` +// is the one place that gap becomes visible. +const NORMAL_CARD_HTML = ` +
+
+
    +
  • + Order placed + October 17, 2023 +
  • +
  • + Order # +
    + Order # + 114-0000000-0000000 +
    +
  • +
+
+
`; +const CARD_WITHOUT_ORDER_ID_HTML = ` +
+
+
    +
  • + Order placed + October 17, 2023 +
  • +
+
+
+
Cancelled
+
+
`; + +test("countOrderCardsWithoutOrderId: 0 when every card has a .yohtmlc-order-id", () => { + const html = `
${NORMAL_CARD_HTML}
`; + assert.equal(countOrderCardsWithoutOrderId(html), 0); + assert.equal(parseOrdersListDom(html).length, 1); +}); + +test("countOrderCardsWithoutOrderId: counts a card with no .yohtmlc-order-id that parseOrdersListDom silently drops", () => { + const html = `
${NORMAL_CARD_HTML}${CARD_WITHOUT_ORDER_ID_HTML}
`; + // The card without an order id vanishes from parseOrdersListDom's output — + // this is the pre-existing, unfixed behavior of parseOrderCard/findOrderId. + const orders = parseOrdersListDom(html); + assert.equal(orders.length, 1, "only the normal card survives parseOrdersListDom"); + // countOrderCardsWithoutOrderId is what makes that loss visible. + assert.equal(countOrderCardsWithoutOrderId(html), 1); +}); + test("parseOrdersListDom: local real fixture parses ≥5 orders with ids + dates", { skip: !existsSync(LOCAL_RAW_DIR), }, () => { @@ -523,7 +580,9 @@ test("buildOrderRecord: both list + detail present — detail wins for enrichmen assert.equal(rec.delivery_status, "Arriving tomorrow"); assert.equal(rec.recipient_name, "Fictional Person"); assert.equal(rec.payment_method_summary, "Visa ending in 1234"); - // item_count = max(list.items.length, detail.items.length) = max(1, 2) = 2. + // Both surfaces saw items, so the larger count wins: max(1, 2) = 2. The + // zero case is where the two surfaces stop being interchangeable — see + // item-count-honesty.test.ts. assert.equal(rec.item_count, 2); assert.equal(rec.fetched_at, "2024-01-20T00:00:00Z"); }); diff --git a/packages/polyfill-connectors/connectors/amazon/parsers.ts b/packages/polyfill-connectors/connectors/amazon/parsers.ts index 1be526cb5..c50c22b29 100644 --- a/packages/polyfill-connectors/connectors/amazon/parsers.ts +++ b/packages/polyfill-connectors/connectors/amazon/parsers.ts @@ -568,6 +568,29 @@ export function parseOrdersListDom(html: string): ListPageOrder[] { return results; } +/** + * Count `.order-card`/`.js-order-card` elements that `parseOrdersListDom` + * silently drops because `findOrderId` found no `.yohtmlc-order-id` span + * matching the canonical order-id shape. `parseOrderCard`'s `if (!orderId) + * return null` (and `parseOrdersListDom`'s discard of that `null`) leaves no + * other trace: the card never reaches the shape-check, so it never produces + * a SKIP_RESULT, a coverage-considered id, or a rejection — a page can lose + * cards here with zero connector-visible evidence. Called alongside + * `parseOrdersListDom` so a caller can detect and report the gap the parse + * step itself cannot see (it only ever sees what survived). + */ +export function countOrderCardsWithoutOrderId(html: string): number { + const { document } = parseHTML(html); + const cards = document.querySelectorAll(".order-card, .js-order-card"); + let dropped = 0; + for (const card of cards) { + if (!findOrderId(card)) { + dropped += 1; + } + } + return dropped; +} + export function parseOrderDate(raw: string | null | undefined): string | null { if (!raw) { return null; @@ -698,6 +721,43 @@ export function buildOrderItemRecord(orderId: string, orderDate: string, merged: * with the detail-page fetch. Prefers detail-page grand total (includes tax) * over the list-page total. */ +/** + * Resolve the order's item count, or null when neither surface can prove one. + * + * Amazon asserts no item count anywhere we read; the number is the count of + * item elements we managed to parse, from two independent surfaces. That makes + * a zero ambiguous, and the ambiguity was being resolved the wrong way. + * + * The list card only renders item titles for SMALL orders. Live data is + * unambiguous about this: across 1,183 collected orders, `item_count === 0` + * NEVER occurs on an order holding 1 or 2 items, and occurs on 53 orders + * holding 3-7 — Amazon collapses 3+ item orders behind a "+N more items" + * affordance that renders no per-item titles. For those orders the list + * contributes 0 and the count rests entirely on the order-detail page. + * + * When that detail fetch is deferred (per-run attempt budget, temporary-failure + * cap, or a latched session repair) `detail` is null, and the previous + * `Math.max(list, detail ?? 0)` collapsed to a confident `0`. Every one of + * those 53 orders also has a null `shipping_address_summary`, confirming the + * detail page — not the parse — is what was missing. The record then claimed an + * empty order while the database held 3-7 item rows for it. + * + * A non-nullable count cannot express "not fetched", so a fetch failure was + * indistinguishable from a genuinely empty order. Returning null when no + * surface saw an item keeps a real empty order (which reaches here with a + * successful detail) reporting 0, while an unfetched one reports unknown. + */ +function resolveItemCount(listOrder: ListPageOrder, detail: OrderDetail | null): number | null { + const counted = Math.max(listOrder.items.length, detail?.items?.length ?? 0); + if (counted > 0) { + return counted; + } + // Nothing counted. Only the detail page can distinguish an empty order from + // an unobserved one: with it we looked and found nothing, without it we never + // looked at the surface that carries items for anything but a tiny order. + return detail ? 0 : null; +} + export function buildOrderRecord( listOrder: ListPageOrder, detail: OrderDetail | null, @@ -717,7 +777,7 @@ export function buildOrderRecord( payment_method_summary: detail?.payment_method_summary || null, gift_order: detail?.gift_order ?? false, digital_order: detail?.digital_order ?? false, - item_count: Math.max(listOrder.items.length, detail?.items?.length ?? 0), + item_count: resolveItemCount(listOrder, detail), fetched_at: emittedAt, }; } diff --git a/packages/polyfill-connectors/connectors/amazon/schemas.ts b/packages/polyfill-connectors/connectors/amazon/schemas.ts index cc8166b77..54de75d92 100644 --- a/packages/polyfill-connectors/connectors/amazon/schemas.ts +++ b/packages/polyfill-connectors/connectors/amazon/schemas.ts @@ -96,7 +96,10 @@ export const orderSchema = z.object({ payment_method_summary: paymentMethodSchema, gift_order: z.boolean(), digital_order: z.boolean(), - item_count: z.number().int().min(0), + // Nullable because a deferred order-detail fetch leaves the count genuinely + // unknown, and a non-nullable count can only express that as `0` — which + // reads as a proven-empty order. See `resolveItemCount`. + item_count: z.number().int().min(0).nullable(), fetched_at: z.string(), }); diff --git a/packages/polyfill-connectors/connectors/amazon/types.ts b/packages/polyfill-connectors/connectors/amazon/types.ts index 77ea0799e..4593fdaa4 100644 --- a/packages/polyfill-connectors/connectors/amazon/types.ts +++ b/packages/polyfill-connectors/connectors/amazon/types.ts @@ -61,7 +61,7 @@ export interface OrdersRecord { fetched_at: string; gift_order: boolean; id: string; - item_count: number; + item_count: number | null; order_date: string; order_total: string | null; order_total_cents: number | null; diff --git a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts index fffedffcf..a6ae99be2 100644 --- a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts +++ b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + addressbookMultiget, addressbookQueryAll, CardDavStructuralError, listAddressBooks, @@ -475,3 +476,83 @@ test("syncCollectionReport: a genuinely transient HTTP-status failure is a plain await server.close(); } }); + +test("syncCollectionReport: reports enumerated members that carry no inlined address-data", async () => { + // Verbatim shape of iCloud's sync-collection multistatus (probe against + // p196-contacts.icloud.com, 2026-08-19): getetag only, no address-data, and + // the collection's own href (no trailing slash) listed beside its members. + const body = ` + + + /18913754167/carddavhome/card + mfhpss66HTTP/1.1 200 OK + + + /18913754167/carddavhome/card/3FF77740-4879-41BB-8267-0429C3EB15A8.vcf + "msndeisq"HTTP/1.1 200 OK + +HwoQEgwAABJ++QlJcAACAAEYAhgAIhYI +`; + + const result = await syncCollectionReport({ + bookUrl: "https://p196-contacts.icloud.com/18913754167/carddavhome/card/", + authHeader: AUTH_HEADER, + fetchImpl: async () => syntheticResponse(207, {}, body), + trustedOrigins: ["https://p196-contacts.icloud.com"], + priorSyncToken: "", + }); + + assert.equal(result.supportsSyncCollection, true); + // The member must be surfaced for a multiget follow-up, not silently dropped. + assert.deepEqual(result.hrefsMissingBodies, [ + "https://p196-contacts.icloud.com/18913754167/carddavhome/card/3FF77740-4879-41BB-8267-0429C3EB15A8.vcf", + ]); + // The collection's own href is not a member resource. + assert.equal(result.resources.length, 0); + assert.equal(result.deletedHrefs.length, 0); +}); + +test("addressbookMultiget: fetches vCard bodies for explicitly requested hrefs", async () => { + const server = await startFakeCardDavServer({ + username: USERNAME, + password: PASSWORD, + syncCollectionOmitsAddressData: true, + }); + try { + server.contacts.set("gina", { + uid: "gina", + href: "/addressbooks/owner/card/gina.vcf", + vcard: buildVCard({ uid: "gina", fn: "Gina Example" }), + }); + const discovery = await discover(server.origin); + const books = await listAddressBooks({ + homeUrl: discovery.addressBookHomeUrl, + authHeader: AUTH_HEADER, + fetchImpl, + trustedOrigins: discovery.visitedOrigins, + }); + const bookUrl = books[0]?.url ?? ""; + + const sync = await syncCollectionReport({ + bookUrl, + authHeader: AUTH_HEADER, + fetchImpl, + trustedOrigins: discovery.visitedOrigins, + priorSyncToken: "", + }); + assert.equal(sync.resources.length, 0, "server inlines nothing"); + assert.equal(sync.hrefsMissingBodies.length, 1); + + const fetched = await addressbookMultiget({ + bookUrl, + authHeader: AUTH_HEADER, + fetchImpl, + trustedOrigins: discovery.visitedOrigins, + hrefs: sync.hrefsMissingBodies, + }); + assert.equal(fetched.length, 1); + assert.equal(parseVCards(fetched[0]?.vcardText ?? "")[0]?.fn, "Gina Example"); + } finally { + await server.close(); + } +}); diff --git a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts index e3decb917..9c6597c92 100644 --- a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts +++ b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts @@ -34,6 +34,17 @@ export interface VCardResource { export interface SyncCollectionResult { deletedHrefs: string[]; + /** + * Hrefs the REPORT enumerated as present-and-current but for which the + * server returned NO `address-data` body, even though the request asked for + * it. RFC 6578 §3.2 does not oblige a server to inline arbitrary properties + * in a sync-collection response, and iCloud in fact does not: its + * sync-collection multistatus carries `getetag` only. These hrefs are real + * members whose bodies must be fetched in a follow-up + * `addressbook-multiget` (RFC 6352 §8.7) — dropping them silently is how a + * populated address book reports as empty. + */ + hrefsMissingBodies: string[]; resources: VCardResource[]; supportsSyncCollection: boolean; syncToken?: string; @@ -207,6 +218,48 @@ const ADDRESSBOOK_QUERY_ALL_BODY = ` `; +/** Hrefs per addressbook-multiget request. Bounds both the request body and + * the response size so one large change set can't produce an unbounded + * round-trip (davRequest's MAX_RESPONSE_BYTES would reject it outright). */ +const MULTIGET_CHUNK_SIZE = 50; + +function xmlEscapeText(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +/** + * Reduce an href to the path form (`/path/to/card.vcf`) a multiget body must + * carry. `syncCollectionReport` resolves member hrefs to absolute URLs so the + * rest of the connector can key records by a stable absolute id, but iCloud + * answers `400 Bad Request` to an absolute `` inside an + * addressbook-multiget and `207` to the path form (probe against + * p196-contacts.icloud.com, 2026-08-19). RFC 6352 §8.7's own examples use the + * path form, so this is the interoperable shape, not an Apple workaround. + * A value that does not parse as a URL is passed through unchanged. + */ +function hrefPathOnly(href: string): string { + try { + const url = new URL(href); + return `${url.pathname}${url.search}`; + } catch { + return href; + } +} + +const MULTIGET_BODY = (hrefs: readonly string[]): string => ` + + + + + +${hrefs.map((href) => ` ${xmlEscapeText(hrefPathOnly(href))}`).join("\n")} +`; + /** * Attempt RFC 6578 `sync-collection` REPORT. `priorSyncToken` empty string * means "initial sync" per RFC 6578 §3.2. Returns @@ -232,22 +285,42 @@ export async function syncCollectionReport(args: { trustedOrigins ); if (res.status === 405 || res.status === 501 || res.status === 415) { - return { resources: [], deletedHrefs: [], supportsSyncCollection: false, truncated: false }; + return { + resources: [], + deletedHrefs: [], + hrefsMissingBodies: [], + supportsSyncCollection: false, + truncated: false, + }; } if (res.status === 507) { // Insufficient storage / token too old (RFC 6578 §3.6): server wants a // full resync. Signal via empty sync token so the caller re-derives. - return { resources: [], deletedHrefs: [], supportsSyncCollection: true, truncated: false, syncToken: "" }; + return { + resources: [], + deletedHrefs: [], + hrefsMissingBodies: [], + supportsSyncCollection: true, + truncated: false, + syncToken: "", + }; } if (res.status < 200 || res.status >= 300) { throw new Error(`carddav_sync_collection_failed: status=${String(res.status)}`); } const newSyncToken = extractTag(res.text, "sync-token"); if (!newSyncToken) { - return { resources: [], deletedHrefs: [], supportsSyncCollection: false, truncated: false }; + return { + resources: [], + deletedHrefs: [], + hrefsMissingBodies: [], + supportsSyncCollection: false, + truncated: false, + }; } const resources: VCardResource[] = []; const deletedHrefs: string[] = []; + const hrefsMissingBodies: string[] = []; for (const block of extractAllHrefBlocks(res.text)) { const href = extractTag(block, "href"); if (!href) { @@ -258,18 +331,97 @@ export async function syncCollectionReport(args: { deletedHrefs.push(new URL(href, res.finalUrl).toString()); continue; } + const absoluteHref = new URL(href, res.finalUrl).toString(); + // The collection's own href appears in the multistatus alongside its + // members (iCloud reports the collection with its own getetag). It is not + // a contact resource, so it must not be queued for a body fetch. + if (isSameCollection(absoluteHref, res.finalUrl)) { + continue; + } const vcardText = extractTag(block, "address-data"); if (!vcardText) { + // Enumerated member with no inlined body: record it for the multiget + // follow-up rather than dropping it. + hrefsMissingBodies.push(absoluteHref); continue; } const etag = extractTag(block, "getetag"); resources.push({ - href: new URL(href, res.finalUrl).toString(), + href: absoluteHref, ...(etag ? { etag } : {}), vcardText: decodeXmlEntities(vcardText), }); } - return { resources, deletedHrefs, supportsSyncCollection: true, syncToken: newSyncToken, truncated: false }; + return { + resources, + deletedHrefs, + hrefsMissingBodies, + supportsSyncCollection: true, + syncToken: newSyncToken, + truncated: false, + }; +} + +/** True when two URLs name the same collection, ignoring a trailing slash. + * iCloud reports the collection itself as `.../card` while the request URL is + * `.../card/`, so a bare string compare would miss it. */ +function isSameCollection(candidate: string, collectionUrl: string): boolean { + const strip = (u: string): string => u.replace(TRAILING_SLASH_RE, ""); + return strip(candidate) === strip(collectionUrl); +} + +const TRAILING_SLASH_RE = /\/+$/; + +/** + * Fetch vCard bodies for an explicit set of member hrefs + * (RFC 6352 §8.7 `addressbook-multiget`). This is the companion to + * `syncCollectionReport` for servers — iCloud among them — that enumerate + * members in a sync-collection response without inlining `address-data`. + * + * Requests are chunked so a large change set cannot produce a single + * unbounded request or response body. + */ +export async function addressbookMultiget(args: { + authHeader: string; + bookUrl: string; + fetchImpl: DiscoveryFetch; + hrefs: readonly string[]; + trustedOrigins: string[]; +}): Promise { + const { authHeader, bookUrl, fetchImpl, hrefs, trustedOrigins } = args; + if (hrefs.length === 0) { + return []; + } + const resources: VCardResource[] = []; + for (let start = 0; start < hrefs.length; start += MULTIGET_CHUNK_SIZE) { + const chunk = hrefs.slice(start, start + MULTIGET_CHUNK_SIZE); + const res = await davRequest( + fetchImpl, + bookUrl, + "REPORT", + authHeader, + { Depth: "1" }, + MULTIGET_BODY(chunk), + trustedOrigins + ); + if (res.status < 200 || res.status >= 300) { + throw new Error(`carddav_addressbook_multiget_failed: status=${String(res.status)}`); + } + for (const block of extractAllHrefBlocks(res.text)) { + const href = extractTag(block, "href"); + const vcardText = extractTag(block, "address-data"); + if (!(href && vcardText)) { + continue; + } + const etag = extractTag(block, "getetag"); + resources.push({ + href: new URL(href, res.finalUrl).toString(), + ...(etag ? { etag } : {}), + vcardText: decodeXmlEntities(vcardText), + }); + } + } + return resources; } /** Bounded full snapshot via `addressbook-query` (RFC 6352 §8.6) — the diff --git a/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.test.ts b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.test.ts new file mode 100644 index 000000000..1477c5be3 --- /dev/null +++ b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.test.ts @@ -0,0 +1,207 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Completeness-anchor tests for the `contact_groups` stream. + * + * `contact_groups` is manifest-REQUIRED and had emitted zero records for + * this owner, ever — including zero tombstones. The connector read only the + * vCard-standard `CATEGORIES` property, but iCloud stores each group as its + * own vCard resource marked `X-ADDRESSBOOKSERVER-KIND:group`. So for an + * iCloud account the stream could not emit a record no matter what the + * account contained, and the resulting zero was indistinguishable from a + * genuinely empty address book. + * + * The vCard bodies below use Apple's real wire shape. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + type GroupAnchor, + groupAnchorVerdict, + groupMemberUids, + isGroupVCard, + partitionVCards, +} from "./group-vcards.ts"; +import { parseVCards } from "./vcard.ts"; + +/** Apple's real group-vCard wire shape, as iCloud serves it. */ +const APPLE_GROUP_VCARD = [ + "BEGIN:VCARD", + "VERSION:3.0", + "N:Family;;;;", + "FN:Family", + "UID:11111111-2222-3333-4444-555555555555", + "X-ADDRESSBOOKSERVER-KIND:group", + "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:AAAAAAAA-0000-0000-0000-000000000001", + "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:BBBBBBBB-0000-0000-0000-000000000002", + "END:VCARD", +].join("\r\n"); + +const PERSON_VCARD = [ + "BEGIN:VCARD", + "VERSION:3.0", + "FN:Tim Nunamaker", + "UID:AAAAAAAA-0000-0000-0000-000000000001", + "END:VCARD", +].join("\r\n"); + +const PERSON_WITH_CATEGORIES = [ + "BEGIN:VCARD", + "VERSION:3.0", + "FN:Someone Else", + "UID:BBBBBBBB-0000-0000-0000-000000000002", + "CATEGORIES:Work,Friends", + "END:VCARD", +].join("\r\n"); + +function card(text: string) { + const [parsed] = parseVCards(text); + assert.ok(parsed, "fixture failed to parse"); + return parsed; +} + +// ─── isGroupVCard: see the groups iCloud actually ships ────────────────── + +test("isGroupVCard recognises Apple's real group vCard", () => { + // Before this, the connector was blind to exactly this resource. + assert.equal(isGroupVCard(card(APPLE_GROUP_VCARD)), true); +}); + +test("isGroupVCard recognises the RFC 6350 standard KIND:group", () => { + const standard = ["BEGIN:VCARD", "VERSION:4.0", "FN:Team", "KIND:group", "END:VCARD"].join("\r\n"); + assert.equal(isGroupVCard(card(standard)), true); +}); + +test("isGroupVCard is case-insensitive on the value", () => { + const upper = ["BEGIN:VCARD", "VERSION:3.0", "FN:Team", "X-ADDRESSBOOKSERVER-KIND:GROUP", "END:VCARD"].join("\r\n"); + assert.equal(isGroupVCard(card(upper)), true); +}); + +test("isGroupVCard treats an ordinary person as a contact", () => { + assert.equal(isGroupVCard(card(PERSON_VCARD)), false); +}); + +test("isGroupVCard fails safe toward contact on an unknown KIND", () => { + // A resource is a group only when the server explicitly says so, so this + // predicate can never silently drop a real person from `contacts`. + const org = ["BEGIN:VCARD", "VERSION:4.0", "FN:ACME", "KIND:org", "END:VCARD"].join("\r\n"); + assert.equal(isGroupVCard(card(org)), false); +}); + +// ─── groupMemberUids ───────────────────────────────────────────────────── + +test("groupMemberUids strips the urn:uuid: prefix Apple ships", () => { + assert.deepEqual(groupMemberUids(card(APPLE_GROUP_VCARD)), [ + "AAAAAAAA-0000-0000-0000-000000000001", + "BBBBBBBB-0000-0000-0000-000000000002", + ]); +}); + +test("groupMemberUids returns empty for a group with no members", () => { + const empty = ["BEGIN:VCARD", "VERSION:3.0", "FN:Empty", "X-ADDRESSBOOKSERVER-KIND:group", "END:VCARD"].join("\r\n"); + assert.deepEqual(groupMemberUids(card(empty)), []); +}); + +test("groupMemberUids de-duplicates while preserving order", () => { + const dupes = [ + "BEGIN:VCARD", + "VERSION:3.0", + "FN:Dupes", + "X-ADDRESSBOOKSERVER-KIND:group", + "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:B", + "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:A", + "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:B", + "END:VCARD", + ].join("\r\n"); + assert.deepEqual(groupMemberUids(card(dupes)), ["B", "A"]); +}); + +// ─── partitionVCards: the phantom-contact fix ──────────────────────────── + +test("partitionVCards keeps a group vCard out of the contact set", () => { + // The phantom-contact defect: a group emitted as a contact whose + // display_name is the group's name, counted as a covered contact. + const resources = [ + { card: card(PERSON_VCARD) }, + { card: card(APPLE_GROUP_VCARD) }, + { card: card(PERSON_WITH_CATEGORIES) }, + ]; + const { contacts, groups } = partitionVCards(resources); + assert.equal(contacts.length, 2); + assert.equal(groups.length, 1); + assert.equal(groups[0]?.card.fn, "Family"); + assert.equal( + contacts.some((c) => c.card.fn === "Family"), + false, + "group leaked into the contact set" + ); +}); + +test("partitionVCards handles a collection with no groups", () => { + const { contacts, groups } = partitionVCards([{ card: card(PERSON_VCARD) }]); + assert.equal(contacts.length, 1); + assert.equal(groups.length, 0); +}); + +// ─── groupAnchorVerdict: the anchor ────────────────────────────────────── + +function anchor(overrides: Partial): GroupAnchor { + return { + serverGroupVCards: 0, + derivedCategoryGroups: 0, + emitted: 0, + boundaryEstablished: true, + ...overrides, + }; +} + +test("groupAnchorVerdict refuses to claim anything without a boundary", () => { + // An incomplete enumeration cannot prove or disprove completeness. + const verdict = groupAnchorVerdict(anchor({ boundaryEstablished: false, serverGroupVCards: 3 })); + assert.equal(verdict.status, "unproven"); +}); + +test("groupAnchorVerdict turns the live zero into a CHECKED zero", () => { + // This is the outcome that resolves the original question: the server + // enumerated the whole collection and it genuinely holds no groups. + assert.equal(groupAnchorVerdict(anchor({})).status, "empty_confirmed"); +}); + +test("groupAnchorVerdict reports SHORT when the server holds groups we did not emit", () => { + // The blindness case: iCloud has 3 group vCards, the connector emitted none. + const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 3, emitted: 0 })); + assert.equal(verdict.status, "short"); + assert.equal(verdict.status === "short" && verdict.missing, 3); + assert.equal(verdict.status === "short" && verdict.considered, 3); +}); + +test("groupAnchorVerdict reports COMPLETE when every server group was emitted", () => { + const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 2, emitted: 2 })); + assert.equal(verdict.status, "complete"); + assert.equal(verdict.status === "complete" && verdict.covered, 2); +}); + +test("groupAnchorVerdict does NOT flag CATEGORIES groups as an overage", () => { + // CATEGORIES groups have no server-side resource, so emitting more than + // the measured denominator is correct behaviour, not a defect. A two-way + // equality here would flag correct behaviour as failure. + const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 1, derivedCategoryGroups: 2, emitted: 3 })); + assert.equal(verdict.status, "complete"); +}); + +test("groupAnchorVerdict keeps a partial shortfall visible", () => { + // 4 server groups, only 1 emitted: still short by 3 even though something + // was emitted. + const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 4, emitted: 1 })); + assert.equal(verdict.status, "short"); + assert.equal(verdict.status === "short" && verdict.missing, 3); +}); + +test("groupAnchorVerdict does not confirm empty when CATEGORIES groups exist", () => { + // A CATEGORIES-only account is not an empty one; claiming + // `empty_confirmed` there would be a false clean bill. + const verdict = groupAnchorVerdict(anchor({ derivedCategoryGroups: 2, emitted: 2 })); + assert.notEqual(verdict.status, "empty_confirmed"); +}); diff --git a/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.ts b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.ts new file mode 100644 index 000000000..dbd65da7f --- /dev/null +++ b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.ts @@ -0,0 +1,190 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// Recognition of Apple's group vCards, and the `contact_groups` completeness +// anchor built on it. +// +// THE PROBLEM THIS SOLVES +// ----------------------- +// `contact_groups` is a manifest-REQUIRED stream that has emitted zero +// records for this owner, ever — including zero tombstones. Two very +// different worlds produce that same zero: +// +// (a) the address book genuinely has no groups, or +// (b) the connector cannot see the groups it has. +// +// Until now the connector could not tell them apart, and a required stream +// sitting at zero for an unfalsifiable reason is exactly the defect this +// work exists to eliminate. +// +// The connector derives groups from the standards-based vCard `CATEGORIES` +// property (RFC 6350 §6.7.1). That derivation is sound as far as it goes. +// But Apple/iCloud does not represent groups with CATEGORIES: it stores each +// group as its OWN vCard resource in the same collection, marked +// `X-ADDRESSBOOKSERVER-KIND:group`, whose members are listed as +// `X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:` lines. +// +// So for an iCloud account, (b) was the live behaviour and the connector +// reported it as (a). +// +// THE SECOND, WORSE CONSEQUENCE +// ----------------------------- +// A group vCard is a real resource in the collection, so the enumeration +// fetches it and the contact emitter emits it AS A CONTACT — a phantom whose +// `display_name` is the group's name, counted as a covered contact. Nothing +// in the emit path checked `KIND`. `isGroupVCard` is what lets the caller +// keep groups out of `contacts`. +// +// THE ANCHOR +// ---------- +// CardDAV enumerates the collection fully, so the set of resources is +// measured at the SOURCE boundary. Partitioning that enumerated set by KIND +// yields a real denominator for `contact_groups`: the number of group vCards +// the server actually holds. Comparing it against what was emitted turns the +// unfalsifiable zero into a checkable claim — and a zero that survives the +// check is now genuine evidence of an empty account rather than an absence +// of evidence. +// +// CEILING, stated honestly +// ------------------------ +// This anchors groups as iCloud REPRESENTS them. Two group sources are +// unified here — Apple group vCards and CATEGORIES — and a group expressed +// only through CATEGORIES on contacts still has no independent server-side +// resource to count, so its denominator contribution is derived, not +// measured. `groupAnchor` reports those two populations separately for that +// reason; it never merges a derived count into the measured one. + +import type { ParsedVCard } from "./vcard.ts"; + +/** RFC 6350 §6.1.4 defines `KIND`; Apple ships the pre-standard + * `X-ADDRESSBOOKSERVER-KIND` on iCloud. Both are accepted so a standards + * -compliant server and iCloud are read the same way. */ +const KIND_PROPERTY_NAMES = ["X-ADDRESSBOOKSERVER-KIND", "KIND"]; + +/** Apple's member property. RFC 6350 §6.6.5 standardises `MEMBER`; iCloud + * ships the `X-ADDRESSBOOKSERVER-` prefixed form. */ +const MEMBER_PROPERTY_NAMES = ["X-ADDRESSBOOKSERVER-MEMBER", "MEMBER"]; + +/** Members are listed as `urn:uuid:`. The prefix is stripped so a + * member uid compares equal to the `UID` the same server reports on the + * member's own vCard, which `vcard.ts` already strips identically. */ +const URN_UUID_PREFIX_RE = /^urn:uuid:/i; + +function rawValues(card: ParsedVCard, names: readonly string[]): string[] { + const wanted = new Set(names.map((n) => n.toUpperCase())); + return card.rawProperties.filter((p) => wanted.has(p.name.toUpperCase())).map((p) => p.value.trim()); +} + +/** + * True when this vCard is a GROUP rather than a person. + * + * Fails SAFE toward "contact": a resource is only treated as a group when + * the server explicitly said so. An unrecognised or absent KIND means the + * resource keeps its existing treatment as a contact, so this predicate can + * never silently remove a real person from `contacts`. + */ +export function isGroupVCard(card: ParsedVCard): boolean { + return rawValues(card, KIND_PROPERTY_NAMES).some((v) => v.toLowerCase() === "group"); +} + +/** + * The member UIDs a group vCard lists, `urn:uuid:` prefix stripped and + * blanks dropped. Order is preserved and duplicates are removed, so the + * result is a stable set-like list suitable for a record body. + */ +export function groupMemberUids(card: ParsedVCard): string[] { + const out: string[] = []; + const seen = new Set(); + for (const value of rawValues(card, MEMBER_PROPERTY_NAMES)) { + const uid = value.replace(URN_UUID_PREFIX_RE, "").trim(); + if (uid && !seen.has(uid)) { + seen.add(uid); + out.push(uid); + } + } + return out; +} + +/** The enumerated collection, partitioned by what each resource actually is. */ +export interface VCardPartition { + contacts: T[]; + groups: T[]; +} + +/** + * Split an enumerated collection into person vCards and group vCards. + * + * This is the single place that decides what a resource IS, so the contact + * emitter and the group anchor cannot disagree about it — the phantom-contact + * defect was precisely such a disagreement, with no partition at all. + */ +export function partitionVCards(resources: readonly T[]): VCardPartition { + const contacts: T[] = []; + const groups: T[] = []; + for (const resource of resources) { + if (isGroupVCard(resource.card)) { + groups.push(resource); + } else { + contacts.push(resource); + } + } + return { contacts, groups }; +} + +/** + * The `contact_groups` completeness anchor for one address book. + * + * `serverGroupVCards` is the MEASURED denominator: group resources the + * server itself enumerated. `derivedCategoryGroups` is reported alongside it + * but deliberately kept separate — it is derived from contact bodies, not + * measured at the boundary, and merging the two would manufacture a + * denominator partly out of the data it is meant to verify. + */ +export interface GroupAnchor { + /** True only when the enumeration reached a complete boundary. */ + boundaryEstablished: boolean; + /** Distinct group names derived from contacts' CATEGORIES. Derived, not measured. */ + derivedCategoryGroups: number; + /** Group records actually emitted this run. */ + emitted: number; + /** Group vCards the server enumerated. Measured at the source boundary. */ + serverGroupVCards: number; +} + +export type GroupAnchorVerdict = + /** Enumeration was incomplete; no claim about completeness is possible. */ + | { status: "unproven"; reason: "boundary_not_established" } + /** The server enumerated no groups and no CATEGORIES groups were derived. + * This is a CHECKED zero — genuine evidence of an empty account. */ + | { status: "empty_confirmed" } + /** Every group the server holds is accounted for in what was emitted. */ + | { status: "complete"; considered: number; covered: number } + /** The server holds groups that were not emitted. */ + | { status: "short"; considered: number; covered: number; missing: number }; + +/** + * Turn the measured anchor into a verdict. + * + * The comparison is one-directional on purpose. Emitting MORE groups than + * the server enumerated as group vCards is normal and correct: CATEGORIES + * groups have no server-side resource, so they legitimately add to the + * emitted count without adding to the measured denominator. Only "the server + * holds groups we did not emit" is a gap. + * + * This mirrors the deletion-safe reasoning used elsewhere in the fleet: a + * two-way equality would flag correct behaviour as failure. + */ +export function groupAnchorVerdict(anchor: GroupAnchor): GroupAnchorVerdict { + if (!anchor.boundaryEstablished) { + return { status: "unproven", reason: "boundary_not_established" }; + } + if (anchor.serverGroupVCards === 0 && anchor.derivedCategoryGroups === 0 && anchor.emitted === 0) { + return { status: "empty_confirmed" }; + } + const considered = anchor.serverGroupVCards; + const covered = Math.min(anchor.emitted, considered); + if (covered < considered) { + return { status: "short", considered, covered, missing: considered - covered }; + } + return { status: "complete", considered, covered }; +} diff --git a/packages/polyfill-connectors/connectors/apple_contacts/index.ts b/packages/polyfill-connectors/connectors/apple_contacts/index.ts index 676c5c6c8..83b541978 100644 --- a/packages/polyfill-connectors/connectors/apple_contacts/index.ts +++ b/packages/polyfill-connectors/connectors/apple_contacts/index.ts @@ -50,6 +50,7 @@ import { } from "../../src/connector-runtime.ts"; import { type FingerprintCursor, openFingerprintCursor } from "../../src/fingerprint-cursor.ts"; import { + addressbookMultiget, addressbookQueryAll, CardDavStructuralError, listAddressBooks, @@ -63,6 +64,13 @@ import { discoverCardDav, nativeFetchAdapter, } from "./discovery.ts"; +import { + type GroupAnchor, + groupAnchorVerdict, + groupMemberUids, + isGroupVCard, + partitionVCards, +} from "./group-vcards.ts"; import { validateRecord } from "./schemas.ts"; import { categoriesOf, type ParsedVCard, parseVCards } from "./vcard.ts"; @@ -234,18 +242,47 @@ export function groupRecord(bookUrl: string, name: string, memberUids: string[]) }; } -/** Derive group-membership records from every contact's CATEGORIES field. - * This is CardDAV/vCard-standard (RFC 6350 §6.7.1), unlike Apple's - * proprietary group-vCard mechanism, which is unconfirmed for iCloud. */ +/** + * Derive group-membership records from BOTH mechanisms a CardDAV server may + * use. + * + * 1. Apple's group vCards — a resource in the collection carrying + * `X-ADDRESSBOOKSERVER-KIND:group` and `X-ADDRESSBOOKSERVER-MEMBER` + * lines. This is how iCloud actually stores groups. + * 2. The vCard-standard `CATEGORIES` property on each contact + * (RFC 6350 §6.7.1). + * + * Only (2) was previously read. For an iCloud account that meant + * `contact_groups` — a manifest-REQUIRED stream — could never emit a record + * no matter how many groups the account had, and the resulting zero was + * indistinguishable from a genuinely empty address book. + * + * A group vCard wins over a same-named CATEGORIES group: the server's own + * membership list is authoritative over one inferred from contact bodies. + */ export function deriveGroups(bookUrl: string, cards: ReadonlyArray<{ card: ParsedVCard; uid: string }>): RecordData[] { + const { contacts, groups } = partitionVCards(cards); + const membersByGroup = new Map(); - for (const { card, uid } of cards) { + for (const { card, uid } of contacts) { for (const category of categoriesOf(card)) { const members = membersByGroup.get(category) ?? []; members.push(uid); membersByGroup.set(category, members); } } + + // Apple group vCards are authoritative; they overwrite a CATEGORIES-derived + // entry of the same name rather than merging into it, so a group's + // membership is never half server-stated and half inferred. + for (const { card } of groups) { + const name = card.fn?.trim(); + if (!name) { + continue; + } + membersByGroup.set(name, groupMemberUids(card)); + } + return [...membersByGroup.entries()].map(([name, members]) => groupRecord(bookUrl, name, members)); } @@ -253,6 +290,10 @@ interface AddressBookCollectionCtx { authHeader: string; book: { url: string; displayName?: string }; bookCursor: FingerprintCursor; + /** See `BaseCollectContext.collectionMode`. `"full_refresh"` makes this book + * ignore its stored `sync_token` for this run. `undefined` means + * `"incremental"`, matching that contract. */ + collectionMode: "full_refresh" | "incremental" | undefined; emit: (msg: { type: "STATE"; stream: string; cursor: unknown }) => Promise; emitRecord: (stream: string, data: RecordData) => Promise; fetchImpl: DiscoveryFetch; @@ -319,10 +360,18 @@ async function emitContactGroupsIfRequested(args: { emitRecord: (stream: string, data: RecordData) => Promise; requested: Map; seenCards: Array<{ card: ParsedVCard; uid: string }>; -}): Promise { +}): Promise<{ emitted: number; anchor: GroupAnchor }> { const { bookUrl, boundaryEstablished, emit, emitRecord, requested, seenCards } = args; + const { contacts, groups } = partitionVCards(seenCards); + const derivedCategoryGroups = new Set(contacts.flatMap(({ card }) => categoriesOf(card))).size; + const anchor: GroupAnchor = { + serverGroupVCards: groups.length, + derivedCategoryGroups, + emitted: 0, + boundaryEstablished, + }; if (!(requested.has("contact_groups") && boundaryEstablished)) { - return 0; + return { emitted: 0, anchor }; } let groupsEmitted = 0; @@ -331,7 +380,50 @@ async function emitContactGroupsIfRequested(args: { groupsEmitted += 1; } await emit({ type: "STATE", stream: "contact_groups", cursor: { fetched_at: nowIso() } }); - return groupsEmitted; + return { emitted: groupsEmitted, anchor: { ...anchor, emitted: groupsEmitted } }; +} + +/** + * Fetch and emit vCard bodies for members a sync-collection response + * enumerated without inlining `address-data` (RFC 6352 §8.7 + * addressbook-multiget). Returns the count of enumerated members whose body + * the server never handed over, so the caller can keep them in the coverage + * denominator instead of dropping them. + */ +async function hydrateMissingBodies(args: { + authHeader: string; + bookUrl: string; + emitContactRecord: (resource: VCardResource) => Promise; + fetchImpl: DiscoveryFetch; + hrefs: readonly string[]; + progress: (message: string, extra?: { count?: number; stream?: string; total?: number }) => Promise; + trustedOrigins: string[]; +}): Promise { + const { authHeader, bookUrl, emitContactRecord, fetchImpl, hrefs, progress, trustedOrigins } = args; + if (hrefs.length === 0) { + return 0; + } + await progress("Fetching contact bodies the sync response did not inline", { + stream: "contacts", + count: 0, + total: hrefs.length, + }); + const fetched = await addressbookMultiget({ bookUrl, authHeader, fetchImpl, trustedOrigins, hrefs }).catch( + (err: unknown) => { + throw classifyCardDavRequestFailure(err, { retryableByDefault: true }); + } + ); + const fetchedByHref = new Map(fetched.map((resource) => [resource.href, resource])); + let unfetched = 0; + for (const href of hrefs) { + const resource = fetchedByHref.get(href); + if (resource) { + await emitContactRecord(resource); + continue; + } + unfetched += 1; + } + return unfetched; } async function loadFullGroupSnapshot(args: { @@ -359,6 +451,44 @@ async function loadFullGroupSnapshot(args: { return { cards, complete }; } +/** + * No token to resume from: `resolveSyncResult` treats `undefined` as "initial + * sync". Named so the full_refresh branch below states what it is doing rather + * than falling off the end of the function. + */ +const WITHHOLD_SYNC_TOKEN = undefined; + +/** + * Decide which stored sync token (if any) this run may resume from. + * + * `full_refresh` is the owner/operator bypass BaseCollectContext.collectionMode + * defines: a connector with its own incremental bookkeeping MUST ignore that + * bookkeeping and walk each stream to its natural end. Withholding the stored + * token here (rather than deleting it) sends `resolveSyncResult` down its + * initial-sync path, which sets `fullBoundary` and so re-establishes the + * contacts enumeration boundary this run — the ONLY way the contacts coverage + * claim can be emitted for a book that already has a token. The stored token is + * left untouched on disk: the run rewrites it from its own sync-collection + * response, and a failed run persists nothing, so an incremental resume stays + * available either way. `undefined` collectionMode means `"incremental"`, per + * that same contract. + */ +async function resolveResumableSyncToken( + collectionMode: "full_refresh" | "incremental" | undefined, + storedSyncToken: string | undefined, + progress: AddressBookCollectionCtx["progress"] +): Promise { + if (collectionMode !== "full_refresh") { + return storedSyncToken; + } + if (storedSyncToken) { + await progress("Full refresh requested: re-enumerating this address book", { stream: "contacts" }); + } + // Withhold the stored token so `resolveSyncResult` takes its initial-sync + // path and re-establishes the enumeration boundary. The token stays on disk. + return WITHHOLD_SYNC_TOKEN; +} + /** * Collect one address book: probe sync capability, fetch (sync-collection or * bounded full snapshot), emit the address-book entity record plus contact + @@ -367,9 +497,11 @@ async function loadFullGroupSnapshot(args: { * is the whole per-book unit of work in one place. */ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ + contactsBoundaryEstablished: boolean; contactsConsidered: number; contactsCovered: number; covered: boolean; + groupAnchor: GroupAnchor; groupsEmitted: number; groupsBoundaryEstablished: boolean; hadUnparseableResource: boolean; @@ -378,6 +510,7 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ book, bookCursor, authHeader, + collectionMode, fetchImpl, trustedOrigins, state, @@ -392,13 +525,17 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ state.contacts as Record }> )?.[bookKey]; + // See `resolveResumableSyncToken`: a full_refresh run withholds the stored + // token so this book re-establishes its enumeration boundary. + const priorSyncToken = await resolveResumableSyncToken(collectionMode, priorSync?.sync_token, progress); + await progress("Probing sync capability", { stream: "contacts" }); const syncResult = await resolveSyncResult({ bookUrl: book.url, authHeader, fetchImpl, trustedOrigins, - priorSyncToken: priorSync?.sync_token, + priorSyncToken, }).catch((err: unknown) => { // resolveSyncResult throws either the HTTP-status-shaped // carddav_sync_collection_failed (transient — retryable, matching the @@ -414,7 +551,18 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ // An incremental sync reports only changes, so its empty resource list is // not an empty inventory. Only the initial sync (no prior token) or the // non-incremental fallback establishes the full contact boundary. - let groupsBoundaryEstablished = !supportsSync || fullBoundary; + // + // This is a CONTACTS boundary fact as much as a groups one: on an + // incremental run `resourcesEnumerated` counts changed resources, not the + // address book's inventory, so a no-change run measures 0 without having + // enumerated anything. Emitting that as `considered: 0` would hand the + // coherence contract a fabricated `enumeration_boundary` proof + // (`packages/reference-contract/src/evidence/coherence.ts` rule 2 reads a + // measured `considered: 0` as "I enumerated the boundary and it held + // nothing"). Track the boundary for contacts explicitly and let the caller + // withhold the claim rather than overstate it. + const contactsBoundaryEstablished = !supportsSync || fullBoundary; + let groupsBoundaryEstablished = contactsBoundaryEstablished; const bookCovered = await emitAddressBookRecordIfRequested({ book, bookCursor, requested, emitRecord, supportsSync }); const fingerprintState = @@ -439,10 +587,20 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ return; } const record = contactRecord(book.url, resource, card); - if (requested.has("contacts") && entityCursor.shouldEmit(record)) { + // A group vCard is a real resource in this collection, so the + // enumeration returns it alongside people. Emitting it as a contact + // creates a phantom whose `display_name` is the group's name and which + // counts as a covered contact. It is still SEEN (it belongs in + // `seenCards`, where the group derivation and the group anchor both + // read it), and it still counts as enumerated — it simply is not a + // person, so it must not enter the `contacts` stream. + const isGroup = isGroupVCard(card); + if (!isGroup && requested.has("contacts") && entityCursor.shouldEmit(record)) { await emitRecord("contacts", record); } - contactCount += 1; + if (!isGroup) { + contactCount += 1; + } seenCards.push({ card, uid: String(record.id) }); }; @@ -450,6 +608,26 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ for (const resource of resolvedSyncResult.resources) { await emitContactRecord(resource); } + // A sync-collection response is only obliged to enumerate members; RFC + // 6578 §3.2 does not require the server to inline the `address-data` the + // request asked for, and iCloud returns `getetag` only. Fetch those + // bodies explicitly (RFC 6352 §8.7 addressbook-multiget) — treating an + // un-inlined member as absent is what made a populated address book + // report zero contacts. + const unfetched = await hydrateMissingBodies({ + authHeader, + bookUrl: book.url, + emitContactRecord, + fetchImpl, + hrefs: resolvedSyncResult.hrefsMissingBodies, + progress, + trustedOrigins, + }); + // A member the server enumerated but whose body it never returned must + // stay in the denominator: considered-but-not-covered is an honest + // partial, where dropping it would fabricate a complete claim. + resourcesEnumerated += unfetched; + unparseableResources += unfetched; for (const deletedHref of resolvedSyncResult.deletedHrefs) { if (requested.has("contacts")) { await emitRecord("contacts", contactTombstone(book.url, deletedHref)); @@ -506,7 +684,7 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ // after the source enumeration and derivation complete successfully; // this lets a genuine zero-group result prove coverage without turning a // failed or unattempted scan into proof. - const groupsEmitted = await emitContactGroupsIfRequested({ + const { emitted: groupsEmitted, anchor: groupAnchor } = await emitContactGroupsIfRequested({ bookUrl: book.url, boundaryEstablished: groupsBoundaryEstablished, emit, @@ -515,6 +693,16 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ seenCards: groupCards, }); + // The `contact_groups` completeness anchor. The verdict is RETURNED rather + // than emitted here: this scope's `emit` is deliberately narrowed to STATE + // messages, and widening it just to report a finding would erode a + // boundary that is doing useful work. The caller owns the full emit. + await progress("Group inventory checked against the enumerated collection", { + stream: "contact_groups", + count: groupsEmitted, + total: Math.max(groupAnchor.serverGroupVCards, groupsEmitted), + }); + const contactsState = (newState.contacts as Record }>) ?? {}; contactsState[bookKey] = { @@ -537,13 +725,21 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{ // address book still proves considered === covered === 0). When one or // more resources failed to parse, considered > covered, so the caller // reads a real partial instead of a fabricated complete. + // + // Group vCards are enumerated resources that are deliberately NOT + // contacts, so they must leave the contacts denominator too — otherwise + // excluding them from `contactsCovered` alone would manufacture a + // permanent considered > covered shortfall out of correct behaviour. They + // are accounted for by the `contact_groups` anchor above instead. return { - contactsConsidered: resourcesEnumerated, + contactsBoundaryEstablished, + contactsConsidered: resourcesEnumerated - groupAnchor.serverGroupVCards, contactsCovered: contactCount, hadUnparseableResource: unparseableResources > 0, covered: bookCovered, groupsEmitted, groupsBoundaryEstablished, + groupAnchor, }; } @@ -564,7 +760,7 @@ if (isMainModule(import.meta.url)) { kind: "env", required: [["APPLE_ID", "APPLE_ID_EMAIL"], "APPLE_APP_SPECIFIC_PASSWORD"], }, - async collect({ state, requested, credentials, emit, emitRecord, progress }) { + async collect({ state, requested, credentials, emit, emitRecord, progress, collectionMode }) { const accountEmail = credentials.APPLE_ID || credentials.APPLE_ID_EMAIL; const appPassword = credentials.APPLE_APP_SPECIFIC_PASSWORD; if (!(accountEmail && appPassword)) { @@ -614,14 +810,17 @@ if (isMainModule(import.meta.url)) { let groupsBoundaryEstablished = true; let contactsConsidered = 0; let contactsCovered = 0; + let contactsBoundaryEstablished = true; let anyUnparseableResource = false; for (const book of books) { considered += 1; const { + contactsBoundaryEstablished: bookContactsBoundaryEstablished, contactsConsidered: bookContactsConsidered, contactsCovered: bookContactsCovered, covered: bookCovered, + groupAnchor, groupsEmitted, groupsBoundaryEstablished: bookGroupsBoundaryEstablished, hadUnparseableResource, @@ -629,6 +828,7 @@ if (isMainModule(import.meta.url)) { book, bookCursor, authHeader, + collectionMode, fetchImpl, trustedOrigins, state, @@ -641,17 +841,48 @@ if (isMainModule(import.meta.url)) { if (bookCovered) { covered += 1; } + // The completeness anchor for this book's groups. `short` is the + // case the connector was previously blind to: the server enumerated + // group vCards that never became records. It is reported as a + // SKIP_RESULT here, where the full `emit` is in scope. + const groupVerdict = groupAnchorVerdict(groupAnchor); + if (groupVerdict.status === "short") { + await emit({ + type: "SKIP_RESULT", + stream: "contact_groups", + reason: "group_inventory_short", + message: "The address book holds groups this run did not record", + diagnostics: { + considered: groupVerdict.considered, + covered: groupVerdict.covered, + missing: groupVerdict.missing, + }, + }); + } // deriveGroups has no drop/filter path: every derived group is // unconditionally emitted, so considered === covered === the exact // count emitted for this book (including a genuine zero-group book). + // The anchor above is what checks that claim against the server's + // own enumeration rather than trusting it. groupsConsidered += groupsEmitted; groupsBoundaryEstablished = groupsBoundaryEstablished && bookGroupsBoundaryEstablished; + contactsBoundaryEstablished = contactsBoundaryEstablished && bookContactsBoundaryEstablished; contactsConsidered += bookContactsConsidered; contactsCovered += bookContactsCovered; anyUnparseableResource = anyUnparseableResource || hadUnparseableResource; } - if (requested.has("contacts")) { + // Withhold the contacts coverage claim entirely when no book established + // a full boundary this run. An incremental sync-collection delta is a + // change feed, not an inventory: its `considered` is the number of + // CHANGED resources, so a quiet run would otherwise emit + // `considered: 0, covered: 0` and be read as a proven-empty address + // book. Emitting nothing leaves the stream honestly unproven (the + // coherence contract's `checkpoint_only`/`no_proof_strategy` -> axis + // `unknown`) instead of falsely complete. A run that DOES establish the + // boundary — initial sync, stale-token resync, or the non-incremental + // fallback — still proves a genuine zero exactly as before. + if (requested.has("contacts") && contactsBoundaryEstablished) { // `considered` counts every resource the server enumerated; // `covered` counts only the ones this run successfully parsed and // accounted for (emitted, or suppressed as unchanged by the diff --git a/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts b/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts index 2d30d045a..807f8904f 100644 --- a/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts +++ b/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts @@ -231,6 +231,38 @@ test("apple_contacts integration: a second run emits a tombstone for a server-si } }); +test("apple_contacts integration: an initial sync of a genuinely empty address book proves verified-empty coverage", async () => { + const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD }); + try { + // No contacts at all, and no prior cursor: this run establishes the full + // boundary, so `considered: 0` is a real measurement ("I enumerated the + // address book and it held nothing") rather than a quiet change feed. + // Withholding the claim here would turn a legitimately-empty required + // stream into a permanent `unknown`, which is its own failure. + const only = await runConnectorProtocolSubprocess({ + cwd: CWD, + entrypoint: ENTRYPOINT, + start: startMessage(), + env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin }, + }); + + const done = only.messages.findLast((m) => m.type === "DONE"); + assert.ok(done && done.type === "DONE"); + assert.equal(done.status, "succeeded"); + assert.equal(recordsOf(only.messages, "contacts").length, 0); + + const contactsCoverage = only.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts"); + assert.ok( + contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE", + "a boundary-establishing run must still emit contacts coverage for a genuine zero" + ); + assert.equal(contactsCoverage.considered, 0); + assert.equal(contactsCoverage.covered, 0); + } finally { + await server.close(); + } +}); + test("apple_contacts integration: an unchanged incremental sync re-enumerates the group inventory", async () => { const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD }); try { @@ -268,6 +300,76 @@ test("apple_contacts integration: an unchanged incremental sync re-enumerates th assert.ok(groupsCoverage && groupsCoverage.type === "DETAIL_COVERAGE"); assert.equal(groupsCoverage.considered, 1); assert.equal(groupsCoverage.covered, 1); + + // The quiet incremental run enumerated nothing, so it must not claim a + // contacts boundary. This is the shape observed in production on + // cin_d344ba53d6d95c7dd343393d: a stored sync_token with an empty + // fingerprint map, replaying as `considered: 0` forever. + const contactsCoverage = second.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts"); + assert.equal( + contactsCoverage, + undefined, + "an unchanged incremental sync must not claim a proven contacts boundary" + ); + } finally { + await server.close(); + } +}); + +test("apple_contacts integration: a full_refresh run re-enumerates and proves contacts coverage despite a stored sync token", async () => { + const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD }); + try { + server.contacts.set("erin", { + uid: "erin", + href: "/addressbooks/owner/card/erin.vcf", + vcard: buildVCard({ uid: "erin", fn: "Erin Example", categories: ["Friends"] }), + }); + server.contacts.set("frank", { + uid: "frank", + href: "/addressbooks/owner/card/frank.vcf", + vcard: buildVCard({ uid: "frank", fn: "Frank Example" }), + }); + + const first = await runConnectorProtocolSubprocess({ + cwd: CWD, + entrypoint: ENTRYPOINT, + start: startMessage(), + env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin }, + }); + const firstState = first.messages.findLast( + (m): m is Extract => m.type === "STATE" && m.stream === "contacts" + ); + assert.ok(firstState); + + // Same stored-token starting point as the quiet-incremental test above — + // which asserts this run emits NO contacts coverage. The only difference + // here is the owner-requested `collection_mode: "full_refresh"`, so this + // pins the bypass itself rather than any change in the source data. + const refreshed = await runConnectorProtocolSubprocess({ + cwd: CWD, + entrypoint: ENTRYPOINT, + start: { ...startMessage({ contacts: firstState.cursor }), collection_mode: "full_refresh" }, + env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin }, + }); + + const contactsCoverage = refreshed.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts"); + assert.ok( + contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE", + "a full_refresh run must re-establish the contacts boundary and claim coverage" + ); + // The claim must be the real inventory (both contacts), not a `considered: 0` + // artifact of a change feed — that distinction is the whole point. + assert.equal(contactsCoverage.considered, 2); + assert.equal(contactsCoverage.covered, 2); + + // The refreshed run must still write a usable token forward, so the next + // ordinary run can resume incrementally rather than re-walking forever. + const refreshedState = refreshed.messages.findLast( + (m): m is Extract => m.type === "STATE" && m.stream === "contacts" + ); + assert.ok(refreshedState); + const [bookState] = Object.values(refreshedState.cursor as Record); + assert.ok(bookState?.sync_token, "a full_refresh run must still persist a sync token for the next run"); } finally { await server.close(); } @@ -315,10 +417,20 @@ test("apple_contacts integration: incremental group requests re-enumerate a genu assert.equal(contactRecords.filter((message) => message.op !== "delete").length, 0); assert.equal(recordsOf(second.messages, "contact_groups").length, 0); + // The incremental contact delta is not a CONTACT boundary either. This run + // saw one deletion and zero live resources, which is a change feed, not an + // enumeration of the address book. Emitting `considered: 0, covered: 0` + // here would hand the coherence contract a fabricated + // `enumeration_boundary` proof and paint a required stream green off a + // quiet sync. The honest move is to emit no contacts coverage at all and + // let the stream read `unknown` until a run actually establishes the + // boundary. const contactsCoverage = second.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts"); - assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE"); - assert.equal(contactsCoverage.considered, 0); - assert.equal(contactsCoverage.covered, 0); + assert.equal( + contactsCoverage, + undefined, + "an incremental contact delta must not claim a proven-empty contacts inventory" + ); // The incremental contact delta is not a group boundary. The connector // must obtain a full snapshot before proving that the now-empty group @@ -508,3 +620,183 @@ test("apple_contacts integration: never logs credentials or vCard field values i await server.close(); } }); + +/** + * Live-shape regression (probe against p196-contacts.icloud.com, 2026-08-19). + * + * iCloud answers a `sync-collection` REPORT by enumerating each member with + * `getetag` ONLY — it does not inline the `address-data` the request asked + * for — and it also lists the collection's own href beside its members. That + * is RFC-legal (RFC 6578 §3.2 requires enumeration, not inlining), so the + * client must fetch bodies via `addressbook-multiget` (RFC 6352 §8.7). + * + * Before this was handled, the connector dropped every member that arrived + * without an inlined body, and a populated address book reported a *measured* + * zero — the most dangerous possible failure, because a proven zero is + * indistinguishable downstream from a genuinely empty account. + */ +test("apple_contacts integration: collects contacts when sync-collection enumerates members without inlining address-data", async () => { + const server = await startFakeCardDavServer({ + username: USERNAME, + password: PASSWORD, + syncCollectionOmitsAddressData: true, + }); + try { + server.contacts.set("frank", { + uid: "frank", + href: "/addressbooks/owner/card/frank.vcf", + vcard: buildVCard({ uid: "frank", fn: "Frank Example", email: "frank@example.com", categories: ["Family"] }), + }); + + const result = await runConnectorProtocolSubprocess({ + cwd: CWD, + entrypoint: ENTRYPOINT, + start: startMessage(), + env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin }, + }); + + const done = result.messages.findLast((m) => m.type === "DONE"); + assert.ok(done && done.type === "DONE"); + assert.equal(done.status, "succeeded"); + + const contacts = recordsOf(result.messages, "contacts"); + assert.equal(contacts.length, 1, "the enumerated member's body must be fetched, not dropped"); + assert.equal(contacts[0]?.display_name, "Frank Example"); + + // The collection's own href is enumerated alongside its members; it must + // not be mistaken for a contact resource. + assert.equal( + contacts.some((c) => String(c.id).endsWith("/addressbooks/owner/card")), + false, + "the collection itself must not be emitted as a contact" + ); + + const contactsCoverage = result.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts"); + assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE"); + assert.equal(contactsCoverage.considered, 1); + assert.equal(contactsCoverage.covered, 1); + + // Groups derive from CATEGORIES, which only exist once bodies are hydrated. + const groups = recordsOf(result.messages, "contact_groups"); + assert.deepEqual( + groups.map((g) => g.name), + ["Family"] + ); + } finally { + await server.close(); + } +}); + +test("apple_contacts integration: an enumerated member whose body never arrives is an honest partial, not a proven zero", async () => { + const server = await startFakeCardDavServer({ + username: USERNAME, + password: PASSWORD, + syncCollectionOmitsAddressData: true, + multigetReturnsNoBodies: true, + }); + try { + server.contacts.set("ghost", { + uid: "ghost", + href: "/addressbooks/owner/card/ghost.vcf", + vcard: buildVCard({ uid: "ghost", fn: "Ghost Example" }), + }); + const result = await runConnectorProtocolSubprocess({ + cwd: CWD, + entrypoint: ENTRYPOINT, + start: startMessage(), + env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin }, + }); + + const done = result.messages.findLast((m) => m.type === "DONE"); + assert.ok(done && done.type === "DONE"); + assert.equal(done.status, "succeeded"); + + assert.equal(recordsOf(result.messages, "contacts").length, 0); + + const contactsCoverage = result.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts"); + assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE"); + assert.equal(contactsCoverage.considered, 1, "the member the server enumerated must stay in the denominator"); + assert.equal(contactsCoverage.covered, 0); + assert.equal( + contactsCoverage.considered > contactsCoverage.covered, + true, + "an unfetchable member must read as an honest partial, never a proven zero" + ); + } finally { + await server.close(); + } +}); + +test("apple_contacts integration: an iCloud group vCard becomes a group, not a phantom contact", async () => { + // The defect this closes: iCloud stores each group as its own vCard + // resource marked `X-ADDRESSBOOKSERVER-KIND:group`, but the connector read + // only the vCard-standard CATEGORIES property. So the group's own resource + // was emitted AS A CONTACT — a phantom whose display_name is the group's + // name, counted as a covered contact — while `contact_groups`, a REQUIRED + // stream, stayed empty. The fake server could not synthesize this shape + // until now, which is why the whole suite passed while the connector was + // blind. + const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD }); + try { + server.contacts.set("alice", { + uid: "alice", + href: "/addressbooks/owner/card/alice.vcf", + vcard: buildVCard({ uid: "alice", fn: "Alice Example" }), + }); + server.contacts.set("bob", { + uid: "bob", + href: "/addressbooks/owner/card/bob.vcf", + vcard: buildVCard({ uid: "bob", fn: "Bob Example" }), + }); + server.contacts.set("family-group", { + uid: "family-group", + href: "/addressbooks/owner/card/family-group.vcf", + vcard: buildVCard({ uid: "family-group", fn: "Family", groupMemberUids: ["alice", "bob"] }), + }); + + const result = await runConnectorProtocolSubprocess({ + cwd: CWD, + entrypoint: ENTRYPOINT, + start: startMessage(), + env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin }, + }); + + const done = result.messages.findLast((m) => m.type === "DONE"); + assert.ok(done && done.type === "DONE"); + assert.equal(done.status, "succeeded"); + + const contacts = recordsOf(result.messages, "contacts"); + const contactNames = contacts.map((c) => String(c.display_name)).sort((a, b) => a.localeCompare(b)); + assert.deepEqual(contactNames, ["Alice Example", "Bob Example"]); + assert.equal( + contacts.some((c) => c.display_name === "Family"), + false, + "the group vCard leaked into the contacts stream as a phantom contact" + ); + + // The group the connector was previously blind to must now be a record, + // with the membership the SERVER stated rather than one inferred from + // contact bodies. + const groups = recordsOf(result.messages, "contact_groups"); + assert.deepEqual( + groups.map((g) => g.name), + ["Family"] + ); + const memberUids = groups[0]?.member_uids as string[] | undefined; + assert.ok(memberUids); + assert.deepEqual( + [...memberUids].sort((a, b) => a.localeCompare(b)), + ["alice", "bob"] + ); + + // The group resource must leave the contacts denominator too: counting + // it as considered-but-not-covered would manufacture a permanent + // shortfall out of correct behaviour. + const contactsCoverage = result.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts"); + assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE"); + assert.equal(contactsCoverage.considered, 2); + assert.equal(contactsCoverage.covered, 2); + } finally { + await server.close(); + } +}); diff --git a/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts b/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts index 94bbb97be..afda47a1f 100644 --- a/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts +++ b/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts @@ -23,11 +23,42 @@ export interface FakeServerOptions { /** When true, REPORT sync-collection returns 501 (unsupported); the * server still answers addressbook-query so the fallback path works. */ disableSyncCollection?: boolean; + /** + * When true, sync-collection obeys RFC 6578 §3.2 faithfully: a request + * carrying a NON-EMPTY `` returns only the members that + * changed since that token — for a quiet collection, an EMPTY response with + * a fresh token — while a request with an empty token still returns every + * member. The default (false) returns all contacts for any token, which is + * a permissive fiction convenient for most tests. + * + * Set this to reproduce the class of defect where an empty CHANGE FEED is + * mistaken for an empty INVENTORY. + */ + enforceRfc6578IncrementalSemantics?: boolean; + /** When true, `addressbook-multiget` answers 404 for every requested href. + * Models a member the server enumerated but whose body it will not hand + * over, so the client must report considered-but-not-covered rather than a + * proven zero. */ + multigetReturnsNoBodies?: boolean; password: string; /** When true, /.well-known/carddav redirects to a second listener * simulating iCloud's regional-host resolution, instead of a * same-origin redirect. */ regionalHost?: boolean; + /** + * When true, the sync-collection REPORT enumerates members with `getetag` + * ONLY — it does not inline the requested `address-data` — and additionally + * reports the collection's own href alongside its members. This is the + * behavior observed live against iCloud (p196-contacts.icloud.com, probe + * 2026-08-19) and it is RFC-legal: RFC 6578 §3.2 obliges the server to + * enumerate changed members, not to inline arbitrary properties. Bodies are + * then only obtainable via `addressbook-multiget` (RFC 6352 §8.7). + * + * Set this to reproduce the class of defect where a client drops every + * enumerated member that arrived without an inlined body, turning a + * populated address book into a measured zero. + */ + syncCollectionOmitsAddressData?: boolean; /** When true, the sync-collection REPORT (and addressbook-query REPORT) * respond with a 302 to an untrusted, unrelated origin instead of the * normal multistatus body — models a compromised/misconfigured server @@ -57,6 +88,8 @@ export interface FakeCardDavServer { url: (path: string) => string; } +const TRAILING_SLASH_RE = /\/$/; +const ABSOLUTE_HREF_RE = /^https?:\/\//i; const NS_D = "DAV:"; const NS_CS = "http://calendarserver.org/ns/"; const PRINCIPAL_PATH = "/principals/owner/"; @@ -71,6 +104,19 @@ function multistatus(inner: string): string { return `${inner}`; } +/** + * Read the `` value out of a sync-collection REPORT body. + * RFC 6578 §3.2 distinguishes an EMPTY token (send me the whole collection) + * from a non-empty one (send me only what changed since then), so the fake + * server needs the actual value, not merely its presence. + */ +const SYNC_TOKEN_RE = /<(?:[A-Za-z0-9]+:)?sync-token>([\s\S]*?)<\/(?:[A-Za-z0-9]+:)?sync-token>/; + +function extractSyncTokenFromRequest(body: string): string { + const match = SYNC_TOKEN_RE.exec(body); + return match?.[1]?.trim() ?? ""; +} + function readBody(req: IncomingMessage): Promise { return new Promise((resolve, reject) => { let data = ""; @@ -101,9 +147,12 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis username, password, disableSyncCollection = false, + enforceRfc6578IncrementalSemantics = false, regionalHost = false, wellKnownAnswersInlineWithoutPrincipal = false, syncReportRedirectsToUnsafeOrigin = false, + syncCollectionOmitsAddressData = false, + multigetReturnsNoBodies = false, } = options; const contacts = new Map(); const deletedHrefs = new Set(); @@ -165,7 +214,51 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis ) .join(""); - const respondSyncCollection = (res: ServerResponse): void => { + /** Members enumerated with `getetag` only — no inlined `address-data`. */ + const etagOnlyResponseBlocks = (): string => + [...contacts.values()] + .map( + (c) => + `${c.href}"${c.uid}-${String(changeCounter)}"HTTP/1.1 200 OK` + ) + .join(""); + + const HREF_RE = /<[^:>]*:?href[^>]*>([\s\S]*?)<\/[^:>]*:?href>/gi; + + /** RFC 6352 §8.7 addressbook-multiget: return bodies for exactly the + * requested hrefs, and 404 any href the collection does not hold. */ + const respondAddressbookMultiget = (res: ServerResponse, body: string): void => { + const requestedHrefs = [...body.matchAll(HREF_RE)].map((m) => (m[1] ?? "").trim()); + // iCloud answers 400 to an absolute inside a multiget and 207 to + // the path form (probe against p196-contacts.icloud.com, 2026-08-19). + // Enforce that here so a regression to absolute hrefs fails in tests + // instead of only against the real provider. + if (requestedHrefs.some((href) => ABSOLUTE_HREF_RE.test(href))) { + res.writeHead(400, { "Content-Type": "text/plain" }); + res.end("Bad Request"); + return; + } + const blocks = requestedHrefs + .map((href) => { + const path = (() => { + try { + return new URL(href, "http://placeholder.invalid").pathname; + } catch { + return href; + } + })(); + const contact = multigetReturnsNoBodies ? undefined : [...contacts.values()].find((c) => c.href === path); + if (!contact) { + return `${href}HTTP/1.1 404 Not Found`; + } + return `${contact.href}"${contact.uid}-${String(changeCounter)}"${xmlEscape(contact.vcard)}HTTP/1.1 200 OK`; + }) + .join(""); + res.writeHead(207, { "Content-Type": "application/xml" }); + res.end(multistatus(blocks)); + }; + + const respondSyncCollection = (res: ServerResponse, requestedSyncToken: string): void => { if (syncReportRedirectsToUnsafeOrigin) { res.writeHead(302, { Location: "https://attacker.example/steal-carddav-creds" }); res.end(); @@ -180,7 +273,19 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis const deleted = [...deletedHrefs] .map((href) => `${href}HTTP/1.1 404 Not Found`) .join(""); - const responseBody = multistatus(`${contactResponseBlocks()}${deleted}${newToken}`); + // RFC 6578 §3.2: a non-empty sync-token requests only what changed since + // that token. Nothing has changed in this fixture, so the change feed is + // empty — while the collection itself still holds every contact. + const quiet = enforceRfc6578IncrementalSemantics && requestedSyncToken.length > 0; + const populatedMembers = syncCollectionOmitsAddressData ? etagOnlyResponseBlocks() : contactResponseBlocks(); + const members = quiet ? "" : populatedMembers; + // iCloud reports the collection's own href (without a trailing slash) + // beside its members. A client must not mistake it for a contact. + const collectionBlock = + syncCollectionOmitsAddressData && !quiet + ? `${BOOK_PATH.replace(TRAILING_SLASH_RE, "")}collection-${String(changeCounter)}HTTP/1.1 200 OK` + : ""; + const responseBody = multistatus(`${collectionBlock}${members}${deleted}${newToken}`); res.writeHead(207, { "Content-Type": "application/xml" }); res.end(responseBody); }; @@ -197,7 +302,7 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis interface Route { match: (req: IncomingMessage, url: string, body: string) => boolean; - respond: (req: IncomingMessage, res: ServerResponse, thisOrigin: () => string) => void; + respond: (req: IncomingMessage, res: ServerResponse, thisOrigin: () => string, body: string) => void; } const routes: Route[] = [ @@ -228,7 +333,11 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis }, { match: (req, url, body) => req.method === "REPORT" && url === BOOK_PATH && body.includes("sync-collection"), - respond: (_req, res) => respondSyncCollection(res), + respond: (_req, res, _origin, body) => respondSyncCollection(res, extractSyncTokenFromRequest(body ?? "")), + }, + { + match: (req, url, body) => req.method === "REPORT" && url === BOOK_PATH && body.includes("addressbook-multiget"), + respond: (_req, res, _origin, body) => respondAddressbookMultiget(res, body ?? ""), }, { match: (req, url, body) => req.method === "REPORT" && url === BOOK_PATH && body.includes("addressbook-query"), @@ -252,7 +361,7 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis const body = await readBody(req); const route = routes.find((r) => r.match(req, url, body)); if (route) { - route.respond(req, res, thisOrigin); + route.respond(req, res, thisOrigin, body); return; } @@ -326,6 +435,12 @@ export function buildVCard(fields: { categories?: string[]; email?: string; fn: string; + /** Member UIDs. Presence makes this a group vCard in Apple's wire shape + * (`X-ADDRESSBOOKSERVER-KIND:group`), which is how iCloud actually stores + * groups. Without this the fixture could only synthesize CATEGORIES, so a + * connector blind to group vCards still passed every test — the fixture + * gap that let the `contact_groups` defect hide. */ + groupMemberUids?: string[]; photo?: { base64: string; mediaType: string }; uid: string; }): string { @@ -336,6 +451,12 @@ export function buildVCard(fields: { if (fields.categories?.length) { lines.push(`CATEGORIES:${fields.categories.join(",")}`); } + if (fields.groupMemberUids) { + lines.push("X-ADDRESSBOOKSERVER-KIND:group"); + for (const member of fields.groupMemberUids) { + lines.push(`X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:${member}`); + } + } if (fields.photo) { lines.push(`PHOTO;ENCODING=b;TYPE=${fields.photo.mediaType.toUpperCase()}:${fields.photo.base64}`); } diff --git a/packages/polyfill-connectors/connectors/chase/cursor-staleness.test.ts b/packages/polyfill-connectors/connectors/chase/cursor-staleness.test.ts new file mode 100644 index 000000000..bb847849d --- /dev/null +++ b/packages/polyfill-connectors/connectors/chase/cursor-staleness.test.ts @@ -0,0 +1,137 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Tests for the two cursor defects that let Chase collection stall silently. + * + * 1. `chooseActivity` picked "Since last statement" whenever a cursor + * existed, with no check that the cursor was recent enough for that + * option's window to reach it. Chase's "Since last statement" covers the + * current cycle only (~30 days), so once a cursor fell behind, every + * scheduled run downloaded a QFX, saw nothing older, and reported + * success — no run could ever reach back across the gap. + * + * 2. `per_account` cursors were carried forward wholesale, so an account + * that disappeared from `discoverAccounts()` kept its cursor forever and + * was skipped silently with no gap emitted. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { chooseActivity, cursorAgeInDays, prunePerAccountCursors } from "./parsers.ts"; +import type { TransactionCursor, TransactionsStateShape } from "./types.ts"; + +const ACCOUNT = "acct-1"; +const RUN_DATE = "2026-08-20"; + +function stateWithCursor(maxSeenDate: string): TransactionsStateShape { + return { per_account: { [ACCOUNT]: { max_seen_date: maxSeenDate } as TransactionCursor } }; +} + +// ─── cursorAgeInDays ───────────────────────────────────────────────────── + +test("cursorAgeInDays measures whole days between cursor and run date", () => { + assert.equal(cursorAgeInDays("2026-08-10", RUN_DATE), 10); + assert.equal(cursorAgeInDays("2026-08-20", RUN_DATE), 0); +}); + +test("cursorAgeInDays tolerates a full ISO timestamp", () => { + assert.equal(cursorAgeInDays("2026-08-10T12:34:56Z", RUN_DATE), 10); +}); + +test("cursorAgeInDays returns null for an unparseable date", () => { + // Must NOT read as fresh — the caller takes the safe path on null. + assert.equal(cursorAgeInDays("not-a-date", RUN_DATE), null); + assert.equal(cursorAgeInDays("", RUN_DATE), null); +}); + +// ─── chooseActivity: an explicit scope still wins ──────────────────────── + +test("chooseActivity honours an explicit time_range over any cursor", () => { + const requested = new Map([["transactions", { time_range: { since: "2026-05-01", until: "2026-07-31" } }]]); + const choice = chooseActivity(requested, stateWithCursor("2026-08-19"), "transactions", ACCOUNT, RUN_DATE); + assert.equal(choice.activity, "date_range"); + assert.deepEqual(choice.dateRange, { from: "2026-05-01", to: "2026-07-31" }); +}); + +test("chooseActivity bootstraps with 'all' when there is no cursor", () => { + const choice = chooseActivity(new Map(), {}, "transactions", ACCOUNT, RUN_DATE); + assert.equal(choice.activity, "all"); +}); + +// ─── chooseActivity: the staleness fix ─────────────────────────────────── + +test("chooseActivity uses 'since_last_statement' for a FRESH cursor", () => { + // The incremental fast path must still work — this is the common case. + const choice = chooseActivity(new Map(), stateWithCursor("2026-08-19"), "transactions", ACCOUNT, RUN_DATE); + assert.equal(choice.activity, "since_last_statement"); +}); + +test("chooseActivity still uses 'since_last_statement' at the freshness edge", () => { + // 25 days: the last age that is still considered fresh. + const choice = chooseActivity(new Map(), stateWithCursor("2026-07-26"), "transactions", ACCOUNT, RUN_DATE); + assert.equal(choice.activity, "since_last_statement"); +}); + +test("chooseActivity falls back to a bounded date_range for a STALE cursor", () => { + // This is the defect: a 90-day-old cursor previously still produced + // "since_last_statement", whose window could not reach it, so the gap + // between them was never re-downloaded by any scheduled run. + const choice = chooseActivity(new Map(), stateWithCursor("2026-05-22"), "transactions", ACCOUNT, RUN_DATE); + assert.equal(choice.activity, "date_range"); + assert.deepEqual(choice.dateRange, { from: "2026-05-22", to: RUN_DATE }); +}); + +test("chooseActivity date_range starts AT the cursor, keeping a safe overlap", () => { + const choice = chooseActivity(new Map(), stateWithCursor("2026-01-15"), "transactions", ACCOUNT, RUN_DATE); + // Starting at (not after) the cursor re-sees one day; stable transaction + // ids make that a suppression, not a duplicate. + assert.equal(choice.dateRange?.from, "2026-01-15"); +}); + +test("chooseActivity treats an unparseable cursor as unprovable, not fresh", () => { + const choice = chooseActivity(new Map(), stateWithCursor("garbage"), "transactions", ACCOUNT, RUN_DATE); + assert.equal(choice.activity, "date_range"); +}); + +test("chooseActivity treats a future-dated cursor as unprovable", () => { + // Clock skew must not be read as maximally fresh. + const choice = chooseActivity(new Map(), stateWithCursor("2027-01-01"), "transactions", ACCOUNT, RUN_DATE); + assert.equal(choice.activity, "date_range"); +}); + +// ─── prunePerAccountCursors ────────────────────────────────────────────── + +const CURSORS: Record = { + "acct-1": { max_seen_date: "2026-08-19" } as TransactionCursor, + "acct-2": { max_seen_date: "2026-08-18" } as TransactionCursor, + "acct-gone": { max_seen_date: "2025-01-01" } as TransactionCursor, +}; + +test("prunePerAccountCursors drops cursors for undiscovered accounts", () => { + const { kept, dropped } = prunePerAccountCursors(CURSORS, new Set(["acct-1", "acct-2"])); + assert.deepEqual(Object.keys(kept).sort(), ["acct-1", "acct-2"]); + assert.deepEqual(dropped, ["acct-gone"]); +}); + +test("prunePerAccountCursors keeps every cursor when all accounts are present", () => { + const { kept, dropped } = prunePerAccountCursors(CURSORS, new Set(["acct-1", "acct-2", "acct-gone"])); + assert.deepEqual(Object.keys(kept).sort(), ["acct-1", "acct-2", "acct-gone"]); + assert.deepEqual(dropped, []); +}); + +test("prunePerAccountCursors reports the drop rather than swallowing it", () => { + // The orphaned cursor must be VISIBLE: a vanished account may be a closed + // account or a discovery regression, and silently dropping it hides both. + const { dropped } = prunePerAccountCursors(CURSORS, new Set(["acct-1"])); + assert.deepEqual( + [...dropped].sort((a, b) => a.localeCompare(b)), + ["acct-2", "acct-gone"] + ); +}); + +test("prunePerAccountCursors handles an empty cursor map", () => { + const { kept, dropped } = prunePerAccountCursors({}, new Set(["acct-1"])); + assert.deepEqual(kept, {}); + assert.deepEqual(dropped, []); +}); diff --git a/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts b/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts index 6987562a6..67a394708 100644 --- a/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts +++ b/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts @@ -2,31 +2,43 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Chase served-gap recovery: DETAIL_GAP_RECOVERED for reached account gaps. + * Chase served-gap recovery: DETAIL_GAP_RECOVERED for reached account gaps + * and for statement gaps whose PDF is hydrated again. * * When the runtime serves the Chase connector a pending per-account * `DETAIL_GAP` at START (`ctx.detailGaps`), Chase re-enumerates and re-downloads * every in-scope account anyway, so the served account is hydrated by the normal - * QFX pass. The missing step this suite pins is the acknowledgement: on a - * successful (or source-limited no-activity) outcome for a served account, the - * connector emits `DETAIL_GAP_RECOVERED` with the served `gap_id`, so the - * durable `connector_detail_gaps` row moves to `recovered` instead of being - * reset to `pending` by runtime cleanup. + * QFX pass. On a successful (or source-limited no-activity) outcome for a + * served account, the connector emits `DETAIL_GAP_RECOVERED` with the served + * `gap_id`, so the durable `connector_detail_gaps` row moves to `recovered` + * instead of being reset to `pending` by runtime cleanup. + * + * Chase `statements` has the identical shape: a failed PDF download opens a + * retryable `DETAIL_GAP` (see `processStatementRow`), and only a later run + * that hydrates that statement's PDF again can close it — via + * `recoverServedStatementGaps`, mirroring the account-gap path above. * * These exercise the exported recovery helpers directly through the recording * harness (the same pattern detail-coverage.test.ts uses) so they validate the - * emitted protocol messages without driving Playwright. The account is treated + * emitted protocol messages without driving Playwright. An account is treated * as reached exactly when `emitTransactionsDetailCoverage` would count it as a - * `hydrated_key`, keeping recovery and coverage in lockstep. + * `hydrated_key`; a statement is treated as recoverable exactly when + * `emitStatementDetailCoverage` counts it in `hydratedKeys` — keeping recovery + * and coverage in lockstep for both streams. * * Safety pinned here (lose-no-data): * - only a served gap whose account was reached is recovered; + * - only a served statement gap whose PDF is hydrated again this run is + * recovered — a considered-but-still-missing statement never is, even + * though it appears in the same run's `requiredKeys`; * - a served gap whose account still FAILS this run is never recovered * (it stays on the DETAIL_GAP re-emit path → runtime resets to pending); * - a served gap whose account is not enumerated this run is never recovered; * - the recovery gap_id is always the served gap_id, never synthesized; - * - only account-level chase.account transaction gaps are recovered — a - * foreign or malformed served gap is ignored. + * - only account-level chase.account transaction gaps are recovered by + * `buildServedAccountGapLookup`, and only chase.statement statements gaps + * by `buildServedStatementGapLookup` — a foreign, wrong-stream, or + * malformed served gap is ignored by both. */ import assert from "node:assert/strict"; import { test } from "node:test"; @@ -35,10 +47,14 @@ import { makeRecordingEmit } from "../../src/test-harness.ts"; import { type AccountDetailOutcome, buildServedAccountGapLookup, + buildServedStatementGapLookup, classifyNoActivityOutcome, type EmitDeps, emitNoActivityProgress, + emitStatementDetailCoverage, recoverServedAccountGaps, + recoverServedStatementGaps, + type StatementDetailOutcome, } from "./index.ts"; import { validateRecord } from "./schemas.ts"; import type { TransactionCursor, TransactionsStateShape } from "./types.ts"; @@ -50,8 +66,10 @@ interface HarnessOverrides { requestedStreams?: readonly StreamScope[]; resFilters?: Map | null>; servedAccountGaps?: ReadonlyMap; + servedStatementGaps?: ReadonlyMap; txState?: TransactionsStateShape; wantsAccounts?: boolean; + wantsStatements?: boolean; wantsTransactions?: boolean; } @@ -78,12 +96,13 @@ function makeHarness(overrides: HarnessOverrides = {}): Harness { requested, resFilters: overrides.resFilters ?? new Map(), servedAccountGaps: overrides.servedAccountGaps, + servedStatementGaps: overrides.servedStatementGaps, tmpDir: "/tmp/pdpp-chase-test-noop", txState: overrides.txState ?? {}, wantsAccounts: overrides.wantsAccounts ?? true, wantsBalances: true, wantsCurrentActivity: false, - wantsStatements: false, + wantsStatements: overrides.wantsStatements ?? false, wantsTransactions: overrides.wantsTransactions ?? true, }; return { deps, messages: harness.protocolMessages }; @@ -146,7 +165,8 @@ test("buildServedAccountGapLookup: ignores foreign, non-transactions, or malform status: "pending", detail_locator: { kind: "amazon.order_detail", order_id: "O1" }, }, - // wrong stream + // wrong stream for THIS lookup — a statements gap must go through + // buildServedStatementGapLookup instead, never this one { gap_id: "g2", stream: "statements", @@ -248,3 +268,171 @@ test("recoverServedAccountGaps: undefined servedAccountGaps is a no-op (legacy/o await recoverServedAccountGaps(deps, outcomes); assert.deepEqual(recoveriesOf(messages), []); }); + +// ─── buildServedStatementGapLookup: only chase.statement gaps ──────────── + +function servedStatementGap(statementId: string, gapId: string): DetailGapStartEntry { + return { + gap_id: gapId, + stream: "statements", + status: "pending", + reference_only: true, + record_key: statementId, + detail_locator: { kind: "chase.statement", statement_id: statementId }, + }; +} + +test("buildServedStatementGapLookup: maps served chase.statement gaps by statement id", () => { + const lookup = buildServedStatementGapLookup([ + servedStatementGap("stmt-1", "gap-1"), + servedStatementGap("stmt-2", "gap-2"), + ]); + assert.equal(lookup.get("stmt-1"), "gap-1"); + assert.equal(lookup.get("stmt-2"), "gap-2"); + assert.equal(lookup.size, 2); +}); + +test("buildServedStatementGapLookup: ignores foreign, non-statements, or malformed served gaps", () => { + const lookup = buildServedStatementGapLookup([ + // foreign connector locator kind + { + gap_id: "g1", + stream: "statements", + status: "pending", + detail_locator: { kind: "amazon.order_detail", order_id: "O1" }, + }, + // wrong stream (a transactions gap, never recoverable by the statements path) + { + gap_id: "g2", + stream: "transactions", + status: "pending", + detail_locator: { kind: "chase.statement", statement_id: "stmt-9" }, + }, + // missing statement_id + { gap_id: "g3", stream: "statements", status: "pending", detail_locator: { kind: "chase.statement" } }, + // null locator + { gap_id: "g4", stream: "statements", status: "pending", detail_locator: null }, + // not pending (already recovered) + { ...servedStatementGap("stmt-done", "g6"), status: "recovered" as never }, + // a valid one survives the filter + servedStatementGap("stmt-ok", "g5"), + ] as readonly DetailGapStartEntry[]); + assert.deepEqual([...lookup.entries()], [["stmt-ok", "g5"]]); +}); + +// ─── recoverServedStatementGaps / emitStatementDetailCoverage: statements close on hydration ─── + +test("recoverServedStatementGaps: a served statement gap whose PDF is hydrated again this run is recovered", async () => { + const { deps, messages } = makeHarness({ + servedStatementGaps: new Map([["stmt-1", "gap-1"]]), + }); + await recoverServedStatementGaps(deps, ["stmt-1"]); + + assert.deepEqual(recoveriesOf(messages), [ + { + type: "DETAIL_GAP_RECOVERED", + reference_only: true, + gap_id: "gap-1", + stream: "statements", + record_key: "stmt-1", + }, + ]); +}); + +test("recoverServedStatementGaps: a served gap whose statement is not in hydratedKeys is never recovered", async () => { + // Mirrors USAA's mutation-killing case: feeding requiredKeys (every + // considered id) instead of hydratedKeys (only proven-hydrated ids) would + // recover a still-missing statement's gap. hydratedKeys must be the only + // acceptable input. + const { deps, messages } = makeHarness({ + servedStatementGaps: new Map([ + ["stmt-have", "gap-have"], + ["stmt-missing", "gap-missing"], + ]), + }); + // Only stmt-have is passed as hydrated; stmt-missing was considered + // (it would appear in requiredKeys) but never proven hydrated. + await recoverServedStatementGaps(deps, ["stmt-have"]); + + assert.deepEqual( + recoveriesOf(messages).map((r) => r.gap_id), + ["gap-have"], + "recovery must follow hydration proof, never merely a considered/required key" + ); +}); + +test("recoverServedStatementGaps: an unmatched or unserved statement id is never recovered", async () => { + const { deps, messages } = makeHarness({ servedStatementGaps: new Map() }); + await recoverServedStatementGaps(deps, ["stmt-unserved"]); + assert.deepEqual( + recoveriesOf(messages), + [], + "a gap id the runtime did not serve must never be synthesized — that could close unrelated work" + ); +}); + +test("recoverServedStatementGaps: undefined servedStatementGaps is a no-op (legacy/ordinary run)", async () => { + const { deps, messages } = makeHarness({}); + await recoverServedStatementGaps(deps, ["stmt-1"]); + assert.deepEqual(recoveriesOf(messages), []); +}); + +const STATEMENT_OUTCOME_HYDRATED = (id: string): StatementDetailOutcome => ({ kind: "hydrated", id }); +const STATEMENT_OUTCOME_GAP = (id: string): StatementDetailOutcome => ({ + kind: "gap", + id, + reason: "temporary_unavailable", + errorClass: "pdf_download_failed", +}); + +test("emitStatementDetailCoverage: recovers a served statement gap once its PDF is hydrated again", async () => { + // The exact defect this suite pins: a statement PDF download failure opens + // a DETAIL_GAP (see processStatementRow), and until this recovery path + // existed, NOTHING ever closed it — buildServedAccountGapLookup only ever + // recovers `stream: "transactions"`, so a served `statements` gap was + // filtered out as wrong-stream and stayed pending forever, even after the + // statement was collected on a later run. + const { deps, messages } = makeHarness({ + wantsStatements: true, + servedStatementGaps: new Map([["stmt-recovered", "gap-recovered"]]), + }); + await emitStatementDetailCoverage(deps, [STATEMENT_OUTCOME_HYDRATED("stmt-recovered")]); + + assert.deepEqual( + recoveriesOf(messages), + [ + { + type: "DETAIL_GAP_RECOVERED", + reference_only: true, + gap_id: "gap-recovered", + stream: "statements", + record_key: "stmt-recovered", + }, + ], + "a statement whose PDF is hydrated again must close its served gap, or the gap stays pending forever" + ); +}); + +test("emitStatementDetailCoverage: never closes a gap for a statement still missing its PDF", async () => { + const { deps, messages } = makeHarness({ + wantsStatements: true, + servedStatementGaps: new Map([ + ["stmt-have", "gap-have"], + ["stmt-missing", "gap-missing"], + ]), + }); + await emitStatementDetailCoverage(deps, [ + STATEMENT_OUTCOME_HYDRATED("stmt-have"), + STATEMENT_OUTCOME_GAP("stmt-missing"), + ]); + + assert.deepEqual( + recoveriesOf(messages).map((r) => r.gap_id), + ["gap-have"], + "recovery must follow the hydration proof, never merely the fact that a gap was served" + ); + assert.ok( + messages.some((m) => m.type === "DETAIL_GAP" && m.record_key === "stmt-missing"), + "the still-missing statement must keep a pending gap" + ); +}); diff --git a/packages/polyfill-connectors/connectors/chase/index.ts b/packages/polyfill-connectors/connectors/chase/index.ts index 55eb096eb..7b01ff717 100644 --- a/packages/polyfill-connectors/connectors/chase/index.ts +++ b/packages/polyfill-connectors/connectors/chase/index.ts @@ -90,6 +90,7 @@ import { parseDashboardAccountsDom, parseDateDelivered, parseStatementsListDom, + prunePerAccountCursors, resolveAccountIdForRow, sha256Hex, shortHash, @@ -1313,6 +1314,13 @@ export interface EmitDeps { * durable gap moves to `recovered` instead of being reset to `pending` by * runtime cleanup. Empty on an ordinary run with no served gaps. */ servedAccountGaps?: ReadonlyMap | undefined; + /** Pending `statements` PDF gaps the runtime served this run at START, + * keyed by statement `id` → served `gap_id`. When a served statement's + * PDF is hydrated again this run, the connector emits `DETAIL_GAP_RECOVERED` + * with the served `gap_id` so the durable gap moves to `recovered` instead + * of staying `pending` forever. Empty on an ordinary run with no served + * statement gaps. */ + servedStatementGaps?: ReadonlyMap | undefined; tmpDir: string; /** Per-transaction fingerprint cursor (excludes the run-clock * `fetched_at`). Shared across all accounts for the whole transactions @@ -1760,6 +1768,8 @@ export async function emitStatementDetailCoverage( } } + await recoverServedStatementGaps(deps, hydratedKeys); + await deps.emit( buildDetailCoverageMessage({ stream: "statements", @@ -1773,6 +1783,82 @@ export async function emitStatementDetailCoverage( ); } +/** + * Build the `statement_id → served gap_id` lookup from the pending detail + * gaps the runtime served this run at START (`ctx.detailGaps`). Filtered to + * Chase statement-PDF gaps — the exact shape `emitStatementDetailCoverage` + * writes via `buildDetailGap`: `stream === "statements"`, + * `detail_locator.kind === "chase.statement"`, and a non-empty + * `statement_id`. Any other served gap (a different connector's locator, a + * non-statements stream, a malformed locator) is ignored so the connector can + * only ever recover a gap it actually understands. Sourcing the `gap_id` from + * the served gap — never synthesizing one — is what guarantees the connector + * cannot mark an unrelated or unserved gap recovered. + */ +export function buildServedStatementGapLookup( + detailGaps: readonly BrowserCollectContext["detailGaps"][number][] +): Map { + const lookup = new Map(); + for (const gap of detailGaps) { + if (gap.stream !== "statements" || gap.status !== "pending") { + continue; + } + const locator = gap.detail_locator; + if (locator?.kind !== "chase.statement") { + continue; + } + const statementId = locator.statement_id; + if ( + typeof statementId !== "string" || + statementId.length === 0 || + typeof gap.gap_id !== "string" || + !gap.gap_id + ) { + continue; + } + // First served gap per statement wins; the runtime serves at most one + // pending gap per (instance, stream, record_key) so a duplicate would be + // a store anomaly, not an expected state. + if (!lookup.has(statementId)) { + lookup.set(statementId, gap.gap_id); + } + } + return lookup; +} + +/** + * Emit `DETAIL_GAP_RECOVERED` for each served statement gap whose PDF is + * hydrated again this run. `hydratedKeys` is the honest input: it is exactly + * the set `emitStatementDetailCoverage` already proved hydrated + * (`outcome.kind === "hydrated"`, driven by `isHydrated` at the download + * site), the same predicate feeding the coverage numerator, so recovery and + * coverage cannot drift apart. A statement whose PDF is still missing this + * run is left on the `DETAIL_GAP` re-emit path and is never recovered here — + * lose-no-data preserved. + */ +export async function recoverServedStatementGaps( + deps: EmitDeps, + hydratedKeys: readonly string[] +): Promise { + const served = deps.servedStatementGaps; + if (!served || served.size === 0) { + return; + } + for (const statementId of hydratedKeys) { + const gapId = served.get(statementId); + if (!gapId) { + continue; + } + await deps.emit({ + type: "DETAIL_GAP_RECOVERED", + reference_only: true, + gap_id: gapId, + stream: "statements", + record_key: statementId, + }); + } +} + /** * Emit the transactions STATE cursor iff we actually emitted * transactions this run. Skipping the emit on empty runs keeps @@ -2606,9 +2692,10 @@ if (isMainModule(import.meta.url)) { // `design-notes/chase-anti-bot.md`. Isolated-per-connector profile works. browser: { profileName: "chase" }, timeRangeField: chaseTimeRangeField, - async ensureSession({ context, onCredentialSubmit, page, sendInteraction }): Promise { + async ensureSession({ context, credentials, onCredentialSubmit, page, sendInteraction }): Promise { await ensureChaseSession({ context, + credentials, onCredentialSubmit, page, sendInteraction, @@ -2687,6 +2774,7 @@ if (isMainModule(import.meta.url)) { requested, resFilters, servedAccountGaps: buildServedAccountGapLookup(ctx.detailGaps), + servedStatementGaps: buildServedStatementGapLookup(ctx.detailGaps), tmpDir, txState, transactionsFingerprintCursor, @@ -2702,9 +2790,38 @@ if (isMainModule(import.meta.url)) { const accounts = await collectChaseAccountInventory({ capture, emit, page }); if (accounts.length === 0) { + // Discovery found nothing. That is how a failed or blocked + // dashboard scrape presents, so the per-account cursors are + // deliberately left untouched — pruning here would erase every + // saved position and force a full re-download of every account. return; // runtime emits DONE succeeded } + // Prune per-account cursors for accounts discovery no longer + // returns. An orphaned cursor is indistinguishable from a live one, + // so without this an account that vanished from discovery is + // skipped silently, forever, with no gap emitted. Safe here because + // discovery is known non-empty. + const { dropped: droppedAccountCursors } = prunePerAccountCursors( + deps.maxSeenByAccount, + new Set(accounts.map((a) => a.internal_id)) + ); + if (droppedAccountCursors.length > 0) { + // Surfaced, not swallowed: this may be a closed account (benign) + // or a discovery regression (not benign), and the connector does + // not guess which. + await emit({ + type: "SKIP_RESULT", + stream: "transactions", + reason: "account_no_longer_discovered", + message: "An account with a saved position is no longer listed on the dashboard", + diagnostics: { dropped_account_count: droppedAccountCursors.length }, + }); + for (const accountId of droppedAccountCursors) { + delete deps.maxSeenByAccount[accountId]; + } + } + // Snapshot the dashboard overview DOM now while the page is still on // it — the MDS recent-activity table (tr.mds-activity-table__row // [data-values]) is only present here, NOT on the QFX download form @@ -2796,8 +2913,10 @@ if (isMainModule(import.meta.url)) { } // Emit STATE for incremental resumption. The per_account cursor drives - // the next run's chooseActivity() — when max_seen_date is present we'll - // use "since_last_statement" instead of re-downloading all transactions. + // the next run's chooseActivity() — when max_seen_date is present and + // still FRESH we'll use "since_last_statement" instead of + // re-downloading all transactions; a stale cursor falls back to a + // bounded date_range export that actually reaches back to it. await emitTransactionsStateIfAny(deps); }, }); diff --git a/packages/polyfill-connectors/connectors/chase/parsers.test.ts b/packages/polyfill-connectors/connectors/chase/parsers.test.ts index 814300a1a..0924b28e7 100644 --- a/packages/polyfill-connectors/connectors/chase/parsers.test.ts +++ b/packages/polyfill-connectors/connectors/chase/parsers.test.ts @@ -668,14 +668,27 @@ test("chooseActivity: a since-only scope closes at the deterministic run date", assert.deepEqual(choice.dateRange, { from: "2026-05-01", to: "2026-08-13" }); }); -test("chooseActivity: cursor max_seen_date → since_last_statement", () => { +test("chooseActivity: FRESH cursor max_seen_date → since_last_statement", () => { const state: TransactionsStateShape = { - per_account: { ID: { max_seen_date: "2026-03-01" } }, + per_account: { ID: { max_seen_date: "2026-08-01" } }, }; const choice = chooseActivity(new Map(), state, "transactions", "ID", "2026-08-13T12:00:00Z"); assert.equal(choice.activity, "since_last_statement"); }); +test("chooseActivity: STALE cursor max_seen_date → bounded date_range", () => { + // This case previously asserted `since_last_statement`, which was the + // defect: a cursor 165 days behind the run date cannot be reached by + // Chase's "Since last statement" window (~30 days), so every scheduled + // run skipped the intervening period and still reported success. + const state: TransactionsStateShape = { + per_account: { ID: { max_seen_date: "2026-03-01" } }, + }; + const choice = chooseActivity(new Map(), state, "transactions", "ID", "2026-08-13T12:00:00Z"); + assert.equal(choice.activity, "date_range"); + assert.deepEqual(choice.dateRange, { from: "2026-03-01", to: "2026-08-13" }); +}); + test("chooseActivity: no hints → all (bootstrap)", () => { assert.equal(chooseActivity(new Map(), {}, "transactions", "ID", "2026-08-13T12:00:00Z").activity, "all"); }); diff --git a/packages/polyfill-connectors/connectors/chase/parsers.ts b/packages/polyfill-connectors/connectors/chase/parsers.ts index dfab0bcb7..121437591 100644 --- a/packages/polyfill-connectors/connectors/chase/parsers.ts +++ b/packages/polyfill-connectors/connectors/chase/parsers.ts @@ -21,6 +21,7 @@ import type { QfxExtracted, QfxTransaction, StatementRow, + TransactionCursor, TransactionsStateShape, } from "./types.ts"; @@ -786,10 +787,54 @@ export interface StreamScopeLike { time_range?: TimeRange; } +/** + * How stale a cursor may be before "Since last statement" stops being a safe + * incremental choice. + * + * Chase's "Since last statement" option covers the current statement cycle + * only — roughly 30 days. If a cursor is older than that, the option's window + * no longer reaches back to where collection left off, and every scheduled + * run silently skips the gap between them: the run succeeds, downloads a + * QFX, and never sees the missing period. Nothing in the run reports a + * problem, because from the connector's point of view it asked for + * everything new and got it. + * + * 25 days is deliberately shorter than the ~30-day cycle so a cursor that is + * merely at the edge of the window still falls back to a bounded + * `date_range` export rather than gambling on the cycle boundary. Being + * early costs one wider export; being late costs a silent hole. + */ +const SINCE_LAST_STATEMENT_MAX_CURSOR_AGE_DAYS = 25; + +const MS_PER_DAY = 86_400_000; + +/** + * Whole days between two YYYY-MM-DD dates, or null when either is not a + * usable date. Returning null (rather than 0 or Infinity) keeps an + * unparseable cursor from being read as "fresh" — the caller treats null as + * "cannot prove freshness" and takes the safe path. + */ +export function cursorAgeInDays(maxSeenDate: string, runDate: string): number | null { + const seen = Date.parse(`${maxSeenDate.slice(0, 10)}T00:00:00Z`); + const now = Date.parse(`${runDate.slice(0, 10)}T00:00:00Z`); + if (!(Number.isFinite(seen) && Number.isFinite(now))) { + return null; + } + return Math.floor((now - seen) / MS_PER_DAY); +} + /** * Pick the QFX download "activity" option for a given account based on * (1) an explicit scope time_range, (2) an existing cursor's - * max_seen_date, or (3) "all" as the bootstrap default. + * max_seen_date — but only while that cursor is still fresh enough for + * "Since last statement" to actually reach it — or (3) "all" as the + * bootstrap default. + * + * The staleness check is the load-bearing part. Without it, the mere + * EXISTENCE of a cursor pinned every future run to "Since last statement", + * so once a gap opened no scheduled run could ever reach back across it. + * A stale cursor now produces a bounded `date_range` export that starts + * where collection actually left off. */ export function chooseActivity( requested: Map, @@ -811,11 +856,60 @@ export function chooseActivity( } const cursor = state.per_account?.[accountId]; if (cursor?.max_seen_date) { - return { activity: "since_last_statement" }; + const age = cursorAgeInDays(cursor.max_seen_date, runDate); + if (age !== null && age >= 0 && age <= SINCE_LAST_STATEMENT_MAX_CURSOR_AGE_DAYS) { + return { activity: "since_last_statement" }; + } + // The cursor is stale (or unparseable, or in the future — a clock skew + // that makes freshness unprovable). "Since last statement" would not + // reach back to it, so export a bounded date range that starts AT the + // cursor instead. Starting at the cursor rather than after it keeps a + // one-day overlap, which is harmless: transaction ids are stable, so a + // re-seen transaction is suppressed rather than duplicated. + return { + activity: "date_range", + dateRange: { from: cursor.max_seen_date.slice(0, 10), to: runDate.slice(0, 10) }, + }; } return { activity: "all" }; } +/** + * Drop `per_account` cursors for accounts the current run did not discover, + * and report which ones were dropped. + * + * Why this matters: the cursor map was carried forward wholesale, so an + * account that disappeared from `discoverAccounts()` kept its cursor + * forever. That is not merely untidy — the stale entry is indistinguishable + * from a live one, so the account is skipped silently with no gap emitted + * and nothing ever reports that it stopped being collected. + * + * `dropped` is returned rather than logged here so the caller can surface it + * as a real finding. An account vanishing from discovery is exactly the kind + * of event that should be visible: it may be a closed account (fine) or a + * discovery regression (not fine), and this function deliberately does not + * guess which. + * + * `known` is the set of accounts discovered THIS run. When discovery itself + * failed the caller must not call this with an empty set — pruning against a + * failed discovery would erase every cursor. See the caller's guard. + */ +export function prunePerAccountCursors( + perAccount: Record, + known: ReadonlySet +): { kept: Record; dropped: string[] } { + const kept: Record = {}; + const dropped: string[] = []; + for (const [accountId, cursor] of Object.entries(perAccount)) { + if (known.has(accountId)) { + kept[accountId] = cursor; + } else { + dropped.push(accountId); + } + } + return { kept, dropped }; +} + // ─── Public labels used by the click-path (exported for entry point) ────── export const ACTIVITY_LABELS: Record = { diff --git a/packages/polyfill-connectors/connectors/chatgpt/branch-reconciliation.test.ts b/packages/polyfill-connectors/connectors/chatgpt/branch-reconciliation.test.ts new file mode 100644 index 000000000..2aac95d48 --- /dev/null +++ b/packages/polyfill-connectors/connectors/chatgpt/branch-reconciliation.test.ts @@ -0,0 +1,231 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * ChatGPT conversations declare `message_count_on_current_branch`, and the + * obvious reconciliation — compare it against the messages we emitted — proves + * nothing. That field is computed by OUR OWN `countBranchMessages` from the + * same `mapping` object, inside the same call that emits those messages. Both + * sides of that comparison read one in-memory graph, so it is a tautology. + * + * Worse, it is a tautology that HIDES the real defect. `flattenTreeCurrentBranch` + * walks parent pointers and stops silently when a parent is missing from the + * mapping. A truncated payload therefore produces a short branch AND a + * correspondingly short declared count — the denominator shrinks to match the + * loss and the conversation reads complete. + * + * The provider assertion worth reconciling against is structural: `current_node` + * and the `parent` chain declare a branch that must be present and must + * terminate at a real root. These tests pin that contract. + * + * Grounding: reconciled against 5,821 live conversations. On-branch message + * counts matched the declared count exactly for 5,815 and never exceeded it, + * so equality is the right contract. Three conversations in the historical + * archive were short by exactly one message, each with a dangling non-system + * parent — the shape `truncatedBranch` reproduces below. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { makeRecordingEmit } from "../../src/test-harness.ts"; +import { processConversationDetail, type StreamDeps } from "./index.ts"; +import { buildConversationRecord, type ConversationDetail } from "./parsers.ts"; +import { validateRecord } from "./schemas.ts"; +import type { ChatGptFetchResult, ChatGptNode, ConversationListItem } from "./types.ts"; + +function makeHarness(requested: readonly string[] = ["conversations", "messages"]) { + const harness = makeRecordingEmit(validateRecord); + const deps: StreamDeps = { + api: { + auth: (): Promise => Promise.reject(new Error("unused")), + fetch: (): Promise => Promise.resolve({ status: 200, json: null }), + }, + emit: harness.emit, + emitRecord: harness.emitRecord, + progress: (): Promise => Promise.resolve(), + requested: new Map(requested.map((name) => [name, { name }])), + }; + // SKIP_RESULT is a protocol message, not a schema rejection, so the gaps this + // contract emits live in protocolMessages. + const skips = (): Record[] => + harness.protocolMessages.filter((m) => (m as { type?: string }).type === "SKIP_RESULT") as unknown as Record< + string, + unknown + >[]; + return { deps, emitted: harness.emitted, messages: harness.protocolMessages, skips }; +} + +function makeConvo(currentNode: string): ConversationListItem { + return { + id: "convo-abc", + title: "Hello world", + create_time: 1_700_000_000, + update_time: 1_700_000_100, + current_node: currentNode, + }; +} + +function emitConversation(deps: StreamDeps) { + return async (c: ConversationListItem, detail: ConversationDetail | null): Promise => { + await deps.emitRecord("conversations", buildConversationRecord(c, detail)); + }; +} + +function userNode(parent: string | null, children: string[] = []): ChatGptNode { + return { + parent, + children, + message: { + author: { role: "user" }, + create_time: 1_700_000_000, + content: { content_type: "text", parts: ["hello"] }, + }, + }; +} + +function assistantNode(parent: string, children: string[] = []): ChatGptNode { + return { + parent, + children, + message: { + author: { role: "assistant" }, + create_time: 1_700_000_001, + end_turn: true, + content: { content_type: "text", parts: ["hi there"] }, + }, + }; +} + +/** A whole conversation: root → u1 → a1, every parent present. */ +function wholeBranch(): Record { + return { + root: { parent: null, children: ["u1"] }, + u1: userNode("root", ["a1"]), + a1: assistantNode("u1"), + }; +} + +/** + * The real defect shape found in the live archive: the branch tip is present + * and walkable, but the opening user turn it descends from was never + * delivered. `flattenTreeCurrentBranch` stops at `a1` and reports a 1-message + * branch, so the declared count agrees with the truncated data and nothing + * looks wrong. + */ +function truncatedBranch(): Record { + return { + a1: assistantNode("aaa2c1fa-missing-user-turn"), + }; +} + +async function run(mapping: Record, currentNode: string) { + const harness = makeHarness(); + const detail: ChatGptFetchResult = { + status: 200, + json: { + title: "Hello world", + create_time: 1_700_000_000, + update_time: 1_700_000_100, + mapping, + current_node: currentNode, + }, + }; + await processConversationDetail(harness.deps, makeConvo(currentNode), detail, emitConversation(harness.deps)); + return harness; +} + +test("chatgpt branch: a whole conversation reconciles clean and reports no gap", async () => { + const { skips, emitted } = await run(wholeBranch(), "a1"); + + assert.equal(emitted.filter((r) => r.stream === "messages").length, 2, "u1 + a1 emit; root is synthetic"); + assert.equal(skips().length, 0, "an intact parent chain must not manufacture a gap"); +}); + +test("chatgpt branch: a truncated branch is surfaced as a gap, not a silent pass", async () => { + // This is the defect the declared-count comparison cannot see: the count and + // the data agree with each other, and both are short. + const { skips } = await run(truncatedBranch(), "a1"); + + const gap = skips().find((s) => s.reason === "branch_truncated"); + assert.ok(gap, "a branch whose parent chain dangles must report a gap"); + assert.equal( + (gap.diagnostics as { missing_parent_id: string }).missing_parent_id, + "aaa2c1fa-missing-user-turn", + "the gap names the message that was not delivered, so it is actionable" + ); +}); + +test("chatgpt branch: the tautological count check would have passed this truncated payload", async () => { + // Proves the point of the whole contract. The conversation record's declared + // count is derived from the same truncated mapping, so declared == emitted + // and a count-based reconciliation reads complete. The gap is the only signal. + const { emitted, skips } = await run(truncatedBranch(), "a1"); + + const convo = emitted.find((r) => r.stream === "conversations"); + const declared = convo?.data.message_count_on_current_branch; + const emittedOnBranch = emitted.filter((r) => r.stream === "messages" && r.data.on_current_branch === true).length; + + assert.equal(declared, 1, "the declared count shrank to match the truncation"); + assert.equal(emittedOnBranch, 1, "so declared == emitted and the counts agree"); + assert.equal( + skips().some((s) => s.reason === "branch_truncated"), + true, + "only the structural check catches it" + ); +}); + +test("chatgpt branch: a current_node absent from the mapping is surfaced", async () => { + // The conversation says it is on a tip the payload does not contain, so the + // branch we walked is not the branch it claims to be on. + const { skips } = await run(wholeBranch(), "tip-we-never-received"); + + const gap = skips().find((s) => s.reason === "branch_tip_missing"); + assert.ok(gap, "an unreachable declared tip must report a gap"); + assert.equal((gap.diagnostics as { conversation_id: string }).conversation_id, "convo-abc"); +}); + +test("chatgpt branch: an off-branch alternative does not trigger a false gap", async () => { + // Branching is normal: holding MORE than the current branch is legitimate and + // must stay silent. Live data showed 28% of conversations hold off-branch + // messages, so a check that fired on these would be useless. + const mapping: Record = { + root: { parent: null, children: ["u1"] }, + u1: userNode("root", ["a1", "a2"]), + a1: assistantNode("u1"), + a2: assistantNode("u1"), + }; + const { skips, emitted } = await run(mapping, "a1"); + + assert.equal(emitted.filter((r) => r.stream === "messages").length, 3, "both branches are held"); + assert.equal(skips().length, 0, "an extra branch is data we have, not data we lost"); +}); + +test("chatgpt branch: a cyclic parent chain terminates instead of hanging", async () => { + // Defensive: a malformed graph must not spin the walk forever. + const mapping: Record = { + a1: assistantNode("a2"), + a2: assistantNode("a1"), + }; + const { skips } = await run(mapping, "a1"); + + assert.equal( + skips().some((s) => s.reason === "branch_truncated"), + false, + "a cycle is fully present in the mapping; it is not a truncation" + ); +}); + +test("chatgpt branch: a conversation with no current_node is not reconciled", async () => { + // Nothing was declared, so there is nothing to hold the payload to. Inventing + // a gap here would be a false positive on a legitimate shape. + const harness = makeHarness(); + const mapping = wholeBranch(); + const detail: ChatGptFetchResult = { + status: 200, + json: { title: "t", create_time: 1, update_time: 2, mapping, current_node: null }, + }; + const convo: ConversationListItem = { ...makeConvo("a1"), current_node: null }; + await processConversationDetail(harness.deps, convo, detail, emitConversation(harness.deps)); + + assert.equal(harness.skips().length, 0, "no declared tip means no claim to reconcile against"); +}); diff --git a/packages/polyfill-connectors/connectors/chatgpt/index.ts b/packages/polyfill-connectors/connectors/chatgpt/index.ts index cc2a0eeab..67bd466fc 100644 --- a/packages/polyfill-connectors/connectors/chatgpt/index.ts +++ b/packages/polyfill-connectors/connectors/chatgpt/index.ts @@ -85,6 +85,7 @@ import type { ChatGptAuth, ChatGptFetchResult, ChatGptJson, + ChatGptNode, ConversationListItem, RawCustomInstructionsBody, RawMemoryEntry, @@ -2025,6 +2026,114 @@ export async function runCustomInstructionsStream( deps.emit(buildFullScanCoverageMessage("custom_instructions", 1)); } +/** + * Reconcile the current branch we walked against the branch the conversation + * actually declares, and surface a shortfall as a gap rather than a silent pass. + * + * The subtlety that shapes this whole check: ChatGPT does NOT hand us an + * independent provider total. `message_count_on_current_branch` is computed by + * our own `countBranchMessages` from the SAME `mapping` object, in the same + * call that emits these messages. Comparing emitted-vs-declared there is a + * tautology — both sides read one in-memory graph — so it would prove nothing. + * + * The real provider assertion is structural: `current_node` plus each node's + * `parent` pointer declare a chain, and `flattenTreeCurrentBranch` walks it by + * following parents until one is absent from the mapping. That walk STOPS + * SILENTLY on a missing parent. A truncated payload therefore yields a short + * branch AND a correspondingly short `message_count_on_current_branch` — the + * declared count shrinks to match the loss, so the conversation reads complete. + * That is the silent-truncation class this check closes. + * + * So the reconciliation is against the graph's own claims: + * - `current_node` must be present in the mapping. If the tip is absent, the + * branch we walked is not the branch the conversation says it is on. + * - The chain must terminate at a real root (a node with no parent), not at a + * dangling pointer into a node the payload did not include. + * + * Both are provider-asserted facts measured at the payload boundary, before any + * emit decision. Live reconciliation over 5,821 collected conversations: + * counting on-branch messages matched the declared count for 5,815 exactly and + * NEVER exceeded it, so equality is the right contract and a shortfall is real. + * + * Emitted as a SKIP_RESULT rather than a DETAIL_GAP for the same reason + * `empty_detail` is: the fetch SUCCEEDED and the conversation is genuinely + * hydrated. Re-fetching returns the same truncated graph, so a retryable gap + * would spin forever. The honest report is "we reached it, and what it gave us + * is internally inconsistent". + */ +async function emitBranchReconciliation( + deps: StreamDeps, + conversationId: string, + mapping: Record, + currentNode: string | null | undefined, + emittedBranchCount: number +): Promise { + if (!currentNode) { + return; + } + // Measured at the payload boundary, independently of what was emitted: does + // the graph contain the tip it claims to be on? + if (!mapping[currentNode]) { + await deps.emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "branch_tip_missing", + message: `conversation ${conversationId} declares a current_node its mapping does not contain; the branch is truncated`, + diagnostics: { + conversation_id: conversationId, + emitted_on_branch: emittedBranchCount, + node_count: Object.keys(mapping).length, + }, + }); + return; + } + // Walk the declared parent chain to its end. A chain that ends on a node + // whose `parent` names an id the mapping lacks was cut short upstream — the + // branch continues in the provider's data but not in ours. + const danglingParent = findDanglingBranchParent(mapping, currentNode); + if (danglingParent !== null) { + await deps.emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "branch_truncated", + message: `conversation ${conversationId} has a current-branch node whose parent is absent from the mapping; earlier messages on this branch were not delivered`, + diagnostics: { + conversation_id: conversationId, + emitted_on_branch: emittedBranchCount, + // A ChatGPT node UUID, not user content — safe to disclose, and the + // only handle that makes the gap actionable. + missing_parent_id: danglingParent, + node_count: Object.keys(mapping).length, + }, + }); + } +} + +/** + * Follow the current branch root-ward and return the first parent id the + * mapping does not contain, or null when the chain terminates cleanly. Mirrors + * `flattenTreeCurrentBranch`'s traversal — including its cycle guard — so the + * two agree on what "the current branch" means. + * + * A parentless root needs no special case: `id = parent` sets `id` falsy and + * the loop condition ends the walk, returning null. An explicit root check was + * tried and removed as dead code — it changed the result on none of the 1,029 + * enumerated parent graphs over three nodes. + */ +function findDanglingBranchParent(mapping: Record, currentNode: string): string | null { + const visited = new Set(); + let id: string | null | undefined = currentNode; + while (id && !visited.has(id) && mapping[id]) { + visited.add(id); + const parent: string | null | undefined = mapping[id]?.parent; + if (parent && !mapping[parent]) { + return parent; + } + id = parent; + } + return null; +} + /** * Process a single fetched conversation detail payload: emit the merged * conversation record first, then emit messages along the current branch @@ -2064,15 +2173,21 @@ export async function processConversationDetail( const currentNode = detail.json.current_node || c.current_node; const currentBranchIds = new Set(flattenTreeCurrentBranch(mapping, currentNode).map((x) => x.nodeId)); let emittedMessageCount = 0; + let emittedBranchCount = 0; for (const [nodeId, node] of Object.entries(mapping)) { - const msg = extractMessage(nodeId, node, c.id, currentBranchIds.has(nodeId)); + const onBranch = currentBranchIds.has(nodeId); + const msg = extractMessage(nodeId, node, c.id, onBranch); if (!msg?.role) { // synthetic root — skip continue; } emittedMessageCount += 1; + if (onBranch) { + emittedBranchCount += 1; + } await deps.emitRecord("messages", msg); } + await emitBranchReconciliation(deps, c.id, mapping, currentNode, emittedBranchCount); if (emittedMessageCount === 0) { // Completeness guard. A 200-with-mapping detail whose graph contains NO // message-bearing node leaves a bare conversation row with zero messages diff --git a/packages/polyfill-connectors/connectors/claude_code/index.ts b/packages/polyfill-connectors/connectors/claude_code/index.ts index 8fa7e8cc9..e8852540a 100755 --- a/packages/polyfill-connectors/connectors/claude_code/index.ts +++ b/packages/polyfill-connectors/connectors/claude_code/index.ts @@ -26,6 +26,7 @@ * Skills/commands live under ~/.claude (overridable via CLAUDE_CODE_HOME). */ +import { createHash } from "node:crypto"; import { createReadStream, type Dirent, type Stats, statSync } from "node:fs"; import { readdir, stat } from "node:fs/promises"; import { homedir } from "node:os"; @@ -519,10 +520,11 @@ async function readFilesRecursively( ): Promise> { const out: Array<{ fullPath: string; relPath: string }> = []; const walk = async (dir: string, prefix: string): Promise => { - let items: Dirent[]; - try { - items = await readdir(dir, { withFileTypes: true }); - } catch { + // Fail closed on an unreadable directory — see `readLocalDirOrFailClosed`. + // A missing directory (ENOENT) is honestly empty; an unreadable one is a + // source-boundary failure and must never be reported as "no files". + const items = await readLocalDirOrFailClosed(dir); + if (items === null) { return; } for (const ent of items.sort((a, b) => a.name.localeCompare(b.name))) { @@ -622,14 +624,77 @@ interface EmitSkillsArgs { requested: Map; } -function markFileMtimeAndShouldSkip( +/** + * Read a local directory, distinguishing legitimate absence from failure. + * + * `null` means the directory genuinely does not exist (ENOENT) — an owner who + * has no `~/.claude/skills` truly has no skills, and an empty enumeration is + * the honest answer. + * + * Any OTHER error (EACCES, EPERM, ENOTDIR, EIO) means the enumeration did not + * happen. The filesystem is this connector's entire source of truth, so an + * unreadable directory is a source-boundary failure, not evidence of emptiness. + * It THROWS rather than returning empty. + * + * WHY: reproduced before this guard — a `chmod 000` on a skills directory + * holding a real skill produced 0 records, ZERO skips, no error, and a STATE + * checkpoint carrying an empty cursor. The run silently recorded "this owner + * has no skills" as fact. Never treat an unreadable directory as "zero files, + * complete". This mirrors codex's `listIfExists`, which already fails closed. + */ +async function readLocalDirOrFailClosed(dir: string): Promise { + try { + return await readdir(dir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { + return null; + } + throw error; + } +} + +/** + * Content-hash gate for the markdown streams (skills, slash_commands, + * memory_notes). + * + * These files are small and already read in full to build their record, so + * hashing the bytes we just read costs nothing extra and is strictly stronger + * than the mtime equality this replaces. + * + * WHY mtime alone was wrong: mtime is owner-controlled metadata, not a content + * fact. `git checkout` and `rsync --times` both restore a previous mtime onto + * genuinely different bytes, and the old gate then skipped the file forever — + * reproduced directly: a SKILL.md whose body changed completely, with its mtime + * restored, emitted zero records on the next run. Silent data loss. + * + * The stored value stays a `number` so the existing `file_mtimes` cursor shape + * is untouched: a 52-bit prefix of the SHA-256 is an exact integer in a double + * and never collides in practice at these file counts. A cursor written by the + * old mtime-based build simply mismatches the new hash once and the file is + * re-emitted — a one-time re-read, never a miss. + */ +function contentGateValue(text: string): number { + const digest = createHash("sha256").update(text, "utf8").digest(); + // Take 52 bits — the largest integer width a JS double holds exactly — by + // dividing rather than shifting (Biome bans bitwise operators here). + // `2n ** 12n` drops the low 12 bits of the leading 64. + return Number(digest.readBigUInt64BE(0) / 4096n); +} + +/** + * Record this file's content gate and report whether it is unchanged since the + * cursor. Callers MUST have the file's text already — the gate is a fact about + * bytes, never about metadata. + */ +function markFileContentAndShouldSkip( fileMtimes: Record, newMtimes: Record, path: string, - mtime: number + text: string ): boolean { - newMtimes[path] = mtime; - return fileMtimes[path] === mtime; + const gate = contentGateValue(text); + newMtimes[path] = gate; + return fileMtimes[path] === gate; } async function readBoundedUtf8(path: string): Promise { @@ -642,10 +707,8 @@ async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMt return; } const skillsDir = join(claudeHome, "skills"); - let entries: Dirent[]; - try { - entries = await readdir(skillsDir, { withFileTypes: true }); - } catch { + const entries = await readLocalDirOrFailClosed(skillsDir); + if (entries === null) { return; } for (const ent of entries) { @@ -663,9 +726,9 @@ async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMt } catch { continue; } - if (markFileMtimeAndShouldSkip(fileMtimes, newMtimes, skillPath, st.mtimeMs)) { - continue; - } + // Read BEFORE gating: the gate is a fact about content, so the bytes must + // be in hand to compute it. These files are small and were already read to + // build the record, so this reorder costs no extra I/O. try { raw = await readBoundedUtf8(skillPath); } catch { @@ -674,6 +737,9 @@ async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMt if (raw === null) { continue; } + if (markFileContentAndShouldSkip(fileMtimes, newMtimes, skillPath, raw)) { + continue; + } const { frontmatter, body } = parseFrontmatter(raw); await emitRecord( "skills", @@ -702,9 +768,7 @@ async function processSlashCommandFile(args: ProcessSlashCommandArgs): Promise => { - let items: Dirent[]; - try { - items = await readdir(dir, { withFileTypes: true }); - } catch { + // Fail closed on an unreadable directory — see `readLocalDirOrFailClosed`. + // A missing directory (ENOENT) is honestly empty; an unreadable one is a + // source-boundary failure and must never be reported as "no files". + const items = await readLocalDirOrFailClosed(dir); + if (items === null) { return; } for (const ent of items) { @@ -793,9 +861,7 @@ async function emitProjectMemoryNotes({ } catch { continue; } - if (markFileMtimeAndShouldSkip(fileMtimes, newMtimes, fullPath, st.mtimeMs)) { - continue; - } + // Content gate, not mtime — see `markFileContentAndShouldSkip`. Read first. try { raw = await readBoundedUtf8(fullPath); } catch { @@ -804,6 +870,9 @@ async function emitProjectMemoryNotes({ if (raw === null) { continue; } + if (markFileContentAndShouldSkip(fileMtimes, newMtimes, fullPath, raw)) { + continue; + } const { frontmatter, body } = parseFrontmatter(raw); await emitRecord( "memory_notes", @@ -1665,6 +1734,12 @@ async function runSkillsAndCommands( newSlashCommandMtimes: Record; } ): Promise { + // A scan that FAILED must not checkpoint. The cursor is a claim about what + // the source contained; writing one after a failed enumeration persists + // "this owner has no skills" as fact and suppresses the files forever on + // subsequent runs. Track each scan's outcome and gate its STATE on success. + let skillsScanned = true; + let slashCommandsScanned = true; try { await emitSkills({ claudeHome, @@ -1674,6 +1749,7 @@ async function runSkillsAndCommands( newMtimes: state.newSkillsMtimes, }); } catch { + skillsScanned = false; await emit({ type: "PROGRESS", message: "Claude Code phase=index pass=index stream=skills scan_skipped=true" }); } try { @@ -1685,24 +1761,43 @@ async function runSkillsAndCommands( newMtimes: state.newSlashCommandMtimes, }); } catch { + slashCommandsScanned = false; await emit({ type: "PROGRESS", message: "Claude Code phase=index pass=index stream=slash_commands scan_skipped=true", }); } if (requested.has("skills")) { - await emit({ - type: "STATE", - stream: "skills", - cursor: { file_mtimes: state.newSkillsMtimes, fetched_at: nowIso() }, - }); + if (skillsScanned) { + await emit({ + type: "STATE", + stream: "skills", + cursor: { file_mtimes: state.newSkillsMtimes, fetched_at: nowIso() }, + }); + } else { + await emit({ + type: "SKIP_RESULT", + stream: "skills", + reason: "source_unreadable", + message: "The Claude Code skills directory could not be enumerated, so its contents are unknown for this run", + }); + } } if (requested.has("slash_commands")) { - await emit({ - type: "STATE", - stream: "slash_commands", - cursor: { file_mtimes: state.newSlashCommandMtimes, fetched_at: nowIso() }, - }); + if (slashCommandsScanned) { + await emit({ + type: "STATE", + stream: "slash_commands", + cursor: { file_mtimes: state.newSlashCommandMtimes, fetched_at: nowIso() }, + }); + } else { + await emit({ + type: "SKIP_RESULT", + stream: "slash_commands", + reason: "source_unreadable", + message: "The Claude Code commands directory could not be enumerated, so its contents are unknown for this run", + }); + } } } diff --git a/packages/polyfill-connectors/connectors/claude_code/markdown-content-gate.test.ts b/packages/polyfill-connectors/connectors/claude_code/markdown-content-gate.test.ts new file mode 100644 index 000000000..6e9420f91 --- /dev/null +++ b/packages/polyfill-connectors/connectors/claude_code/markdown-content-gate.test.ts @@ -0,0 +1,136 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The markdown streams (skills, slash_commands, memory_notes) gate re-emission + * on FILE CONTENT, not on mtime. + * + * mtime is owner-controlled metadata, not a content fact. `git checkout` and + * `rsync --times` both restore a prior mtime onto genuinely different bytes. + * Under the old mtime-equality gate that file was skipped forever — reproduced + * directly before the fix: a SKILL.md whose body changed completely, with its + * mtime restored to the cursor's value, emitted ZERO records on the next run. + * + * These tests pin both directions, because a gate is only honest if it holds + * both: + * - changed content + identical mtime => MUST re-emit (the data-loss bug) + * - identical content + changed mtime => MUST stay suppressed (no churn) + */ + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import type { EmittedMessage } from "../../src/connector-runtime.ts"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; + +const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "claude_code", "index.ts"); + +/** A fixed mtime, reapplied after every write so mtime is never the signal. */ +const PINNED_MTIME = new Date("2026-01-01T00:00:00Z"); + +async function writePinned(path: string, body: string): Promise { + await writeFile(path, body); + await utimes(path, PINNED_MTIME, PINNED_MTIME); +} + +async function runClaudeCode( + home: string, + stream: string, + state: Record +): Promise<{ carried: Record; records: EmittedMessage[] }> { + const result = await runConnectorProtocolSubprocess({ + cwd: PACKAGE_ROOT, + entrypoint: ENTRYPOINT, + env: { + CLAUDE_CODE_HOME: home, + HOME: home, + PDPP_OWNER_TOKEN: "", + PDPP_RS_URL: "", + RS_URL: "", + }, + start: { scope: { streams: [{ name: stream }] }, state, type: "START" }, + }); + const carried: Record = {}; + for (const message of result.messages) { + if (message.type === "STATE" && typeof (message as { stream?: string }).stream === "string") { + carried[(message as { stream: string }).stream] = (message as { cursor?: unknown }).cursor; + } + } + return { + carried, + records: result.messages.filter((m) => m.type === "RECORD" && m.stream === stream), + }; +} + +test("a skill whose content changes under a restored mtime is re-emitted", async () => { + const home = await mkdtemp(join(tmpdir(), "pdpp-cc-content-gate-")); + try { + const skillDir = join(home, "skills", "demo"); + await mkdir(skillDir, { recursive: true }); + const skillPath = join(skillDir, "SKILL.md"); + await writePinned(skillPath, "---\nname: demo\n---\nORIGINAL BODY\n"); + + const first = await runClaudeCode(home, "skills", {}); + assert.equal(first.records.length, 1, "the skill should be emitted on a fresh run"); + + // The exact `git checkout` / `rsync --times` shape: content replaced, + // mtime restored to the value already in the cursor. + await writePinned(skillPath, "---\nname: demo\n---\nCOMPLETELY DIFFERENT BODY\n"); + + const second = await runClaudeCode(home, "skills", first.carried); + // The load-bearing assertion. Under the mtime gate this was 0. + assert.equal(second.records.length, 1, "changed content under an identical mtime must re-emit, not be skipped"); + } finally { + await rm(home, { force: true, recursive: true }); + } +}); + +test("an unchanged skill stays suppressed even when its mtime moves", async () => { + const home = await mkdtemp(join(tmpdir(), "pdpp-cc-content-gate-stable-")); + try { + const skillDir = join(home, "skills", "demo"); + await mkdir(skillDir, { recursive: true }); + const skillPath = join(skillDir, "SKILL.md"); + const body = "---\nname: demo\n---\nSTABLE BODY\n"; + await writePinned(skillPath, body); + + const first = await runClaudeCode(home, "skills", {}); + assert.equal(first.records.length, 1); + + // Byte-identical rewrite, but the mtime jumps forward — a touch, or any + // tool that rewrites in place. Content is the signal, so this must NOT + // churn. Guards the opposite failure from the test above. + const later = new Date("2026-06-01T00:00:00Z"); + await writeFile(skillPath, body); + await utimes(skillPath, later, later); + + const second = await runClaudeCode(home, "skills", first.carried); + assert.equal(second.records.length, 0, "identical content must stay suppressed regardless of mtime"); + } finally { + await rm(home, { force: true, recursive: true }); + } +}); + +test("a slash command whose content changes under a restored mtime is re-emitted", async () => { + const home = await mkdtemp(join(tmpdir(), "pdpp-cc-content-gate-cmd-")); + try { + const commandsDir = join(home, "commands"); + await mkdir(commandsDir, { recursive: true }); + const cmdPath = join(commandsDir, "deploy.md"); + await writePinned(cmdPath, "---\ndescription: original\n---\nrun the original steps\n"); + + const first = await runClaudeCode(home, "slash_commands", {}); + assert.equal(first.records.length, 1); + + await writePinned(cmdPath, "---\ndescription: rewritten\n---\nrun completely different steps\n"); + + const second = await runClaudeCode(home, "slash_commands", first.carried); + assert.equal(second.records.length, 1, "slash_commands must use the same content gate as skills"); + } finally { + await rm(home, { force: true, recursive: true }); + } +}); diff --git a/packages/polyfill-connectors/connectors/claude_code/unreadable-source-fails-closed.test.ts b/packages/polyfill-connectors/connectors/claude_code/unreadable-source-fails-closed.test.ts new file mode 100644 index 000000000..0e6d1f5fe --- /dev/null +++ b/packages/polyfill-connectors/connectors/claude_code/unreadable-source-fails-closed.test.ts @@ -0,0 +1,116 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * An unreadable source directory must never be reported as "zero files". + * + * The filesystem is this connector's whole source of truth, so a directory it + * cannot enumerate is a source-boundary failure — NOT evidence of emptiness. + * A missing directory (ENOENT) is different: an owner with no + * `~/.claude/skills` genuinely has no skills, and an empty answer is correct. + * + * Reproduced before the guard: `chmod 000` on a skills directory holding a real + * skill produced 0 records, ZERO skips, no error, and a STATE checkpoint with + * an empty cursor — the run silently recorded "this owner has no skills" and + * persisted it. That is the fail-open shape these tests forbid. + * + * These tests are skipped when running as root, which bypasses permission bits. + */ + +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; + +const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "claude_code", "index.ts"); +// chmod bits do not restrain root, so the unreadable-dir premise cannot hold. +const RUNNING_AS_ROOT = typeof process.getuid === "function" && process.getuid() === 0; + +async function runSkills(home: string): Promise<{ + records: number; + states: number; + unreadable: number; +}> { + const result = await runConnectorProtocolSubprocess({ + // Matches every existing claude_code protocol test: this connector can exit + // non-zero after DONE for reasons unrelated to the streams under test. + allowFailedDone: true, + cwd: PACKAGE_ROOT, + entrypoint: ENTRYPOINT, + env: { + CLAUDE_CODE_HOME: home, + CLAUDE_CODE_PROJECTS_DIR: join(home, "projects"), + HOME: home, + PDPP_OWNER_TOKEN: "", + PDPP_RS_URL: "", + RS_URL: "", + }, + start: { scope: { streams: [{ name: "skills" }] }, state: {}, type: "START" }, + }); + return { + records: result.messages.filter((m) => m.type === "RECORD" && m.stream === "skills").length, + states: result.messages.filter((m) => m.type === "STATE" && m.stream === "skills").length, + unreadable: result.messages.filter( + (m) => m.type === "SKIP_RESULT" && m.stream === "skills" && m.reason === "source_unreadable" + ).length, + }; +} + +test("an unreadable skills directory does not checkpoint an empty cursor", { skip: RUNNING_AS_ROOT }, async () => { + const home = await mkdtemp(join(tmpdir(), "pdpp-cc-unreadable-")); + const skillsDir = join(home, "skills"); + try { + await mkdir(join(home, "projects"), { recursive: true }); + const skillDir = join(skillsDir, "demo"); + await mkdir(skillDir, { recursive: true }); + await writeFile(join(skillDir, "SKILL.md"), "---\nname: demo\n---\nREAL BODY\n"); + await chmod(skillsDir, 0o000); + + const outcome = await runSkills(home); + + // The load-bearing assertion. Before the guard this was `states: 1` with an + // empty `file_mtimes` map — a persisted claim that the owner has no skills. + assert.equal( + outcome.states, + 0, + "an unreadable source directory must not produce a STATE checkpoint claiming an empty enumeration" + ); + assert.equal(outcome.records, 0, "no records can be read from an unreadable directory"); + assert.equal(outcome.unreadable, 1, "the failed enumeration must surface as a skip, not silence"); + } finally { + await chmod(skillsDir, 0o755).catch(() => undefined); + await rm(home, { force: true, recursive: true }); + } +}); + +test("a genuinely missing skills directory is still an honest empty enumeration", async () => { + const home = await mkdtemp(join(tmpdir(), "pdpp-cc-absent-")); + try { + // A readable home with a `projects` dir but NO `skills` dir: the skills + // enumeration hits ENOENT. This owner truly has no skills, so the run must + // succeed and checkpoint normally. Guards the opposite failure — a + // fail-closed guard that also tripped on legitimate absence would break + // every owner who has never created a skill. + // + // (`projects` is created because the connector requires the surrounding + // home layout to run at all; a bare temp dir exits non-zero on the + // unmodified connector too, so its absence would not test this guard.) + await mkdir(join(home, "projects"), { recursive: true }); + + const outcome = await runSkills(home); + + // The distinction that matters: absence is NOT reported as a source + // failure. (Verified against the unmodified connector, an absent skills + // directory also produces no STATE — the stream is simply not exercised. + // This guard must not change that; it must only stop an UNREADABLE + // directory from checkpointing.) + assert.equal(outcome.unreadable, 0, "an absent directory is legitimate, not a source failure"); + assert.equal(outcome.records, 0); + } finally { + await rm(home, { force: true, recursive: true }); + } +}); diff --git a/packages/polyfill-connectors/connectors/codex/index.ts b/packages/polyfill-connectors/connectors/codex/index.ts index aada3090b..5b71f9991 100644 --- a/packages/polyfill-connectors/connectors/codex/index.ts +++ b/packages/polyfill-connectors/connectors/codex/index.ts @@ -568,11 +568,22 @@ async function emitSkillsStream( ): Promise { // Each skill is a subdirectory with SKILL.md at its root. Follows symlinks // (skills are often symlinked from dotfiles). Skips hidden dirs (.system). + // + // Fail closed on an unreadable directory, matching `emitRulesStream` and + // `emitPromptsStream`, which already route through `listIfExists`. This + // function previously swallowed EVERY error, so a permission failure on the + // skills directory was indistinguishable from "this owner has no skills" — + // the filesystem is the whole source of truth here, so an unreadable + // directory is a source-boundary failure, not evidence of emptiness. ENOENT + // (genuinely absent) remains a legitimate empty enumeration. let entries: Dirent[]; try { entries = await readdir(skillsDir, { withFileTypes: true }); - } catch { - return; + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { + return; + } + throw error; } for (const ent of entries) { if (shouldSkipSkillEntry(ent)) { diff --git a/packages/polyfill-connectors/connectors/codex/skills-source-fails-closed.test.ts b/packages/polyfill-connectors/connectors/codex/skills-source-fails-closed.test.ts new file mode 100644 index 000000000..30bb5384b --- /dev/null +++ b/packages/polyfill-connectors/connectors/codex/skills-source-fails-closed.test.ts @@ -0,0 +1,85 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * `emitSkillsStream` must fail closed on an unreadable skills directory. + * + * codex's other two markdown streams (`rules`, `prompts`) already route through + * `listIfExists`, which distinguishes ENOENT (legitimate absence) from a real + * read failure and rethrows the latter. `emitSkillsStream` was the odd one out: + * it swallowed EVERY error, so a permission failure on the skills directory was + * indistinguishable from "this owner has no skills". + * + * The filesystem is this connector's entire source of truth, so an unreadable + * directory is a source-boundary failure, never evidence of emptiness. + * + * The unreadable case is skipped as root, which bypasses permission bits. + */ + +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; + +const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "codex", "index.ts"); +const RUNNING_AS_ROOT = typeof process.getuid === "function" && process.getuid() === 0; + +async function runSkills(codexHome: string): Promise<{ records: number; states: number }> { + const result = await runConnectorProtocolSubprocess({ + allowFailedDone: true, + cwd: PACKAGE_ROOT, + entrypoint: ENTRYPOINT, + env: { CODEX_HOME: codexHome, HOME: codexHome }, + start: { scope: { streams: [{ name: "skills" }] }, state: {}, type: "START" }, + }); + return { + records: result.messages.filter((m) => m.type === "RECORD" && m.stream === "skills").length, + states: result.messages.filter((m) => m.type === "STATE" && m.stream === "skills").length, + }; +} + +test("an unreadable codex skills directory does not report an empty enumeration", { + skip: RUNNING_AS_ROOT, +}, async () => { + const home = await mkdtemp(join(tmpdir(), "pdpp-codex-unreadable-")); + const skillsDir = join(home, "skills"); + try { + await mkdir(join(skillsDir, "demo"), { recursive: true }); + await writeFile(join(skillsDir, "demo", "SKILL.md"), "---\nname: demo\n---\nREAL BODY\n"); + await chmod(skillsDir, 0o000); + + const outcome = await runSkills(home); + + // The load-bearing assertion: the unreadable directory must not be + // silently reported as a clean, empty, checkpointed enumeration. + assert.equal(outcome.records, 0, "no records can be read from an unreadable directory"); + assert.equal( + outcome.states, + 0, + "an unreadable source directory must not checkpoint a cursor claiming an empty enumeration" + ); + } finally { + await chmod(skillsDir, 0o755).catch(() => undefined); + await rm(home, { force: true, recursive: true }); + } +}); + +test("a readable codex skills directory still emits its skills", async () => { + const home = await mkdtemp(join(tmpdir(), "pdpp-codex-readable-")); + try { + await mkdir(join(home, "skills", "demo"), { recursive: true }); + await writeFile(join(home, "skills", "demo", "SKILL.md"), "---\nname: demo\n---\nREAL BODY\n"); + + const outcome = await runSkills(home); + + // Guards the opposite failure: a fail-closed guard that also tripped on a + // perfectly readable directory would break every owner who has skills. + assert.equal(outcome.records, 1, "a readable skills directory must still be enumerated"); + } finally { + await rm(home, { force: true, recursive: true }); + } +}); diff --git a/packages/polyfill-connectors/connectors/github/index.ts b/packages/polyfill-connectors/connectors/github/index.ts index d0ade7e38..031889795 100755 --- a/packages/polyfill-connectors/connectors/github/index.ts +++ b/packages/polyfill-connectors/connectors/github/index.ts @@ -304,6 +304,43 @@ async function guardGithubPagination( * cannot know its full inventory for the run (e.g. a search-API cap truncation) * MUST NOT call this — it leaves `considered` unknown and relies on its * terminal-gap evidence instead. + * + * ── Why no provider-reported total is bound here ────────────────────────── + * + * GitHub exposes several tempting scalars. Each was measured against this + * instance's live holdings and REJECTED. Do not bind them: + * + * `public_repos` / `public_gists` (from `/user`, already stored on the + * `user_stats` record) measure a strict SUBSET, not this stream's boundary. + * Live: `public_repos: 94` against 575 held repositories — of which 355 are + * private and 465 belong to orgs, neither of which `public_repos` counts. + * `public_gists: 8` matched the 8 public gists exactly while 43 secret gists + * sat outside it. Binding either would assert a permanent ~6x false gap. + * + * `Link: rel="last"` yields a PAGE count, so an item total only under the + * assumption that every page is full — which the last page never is. It also + * cannot survive the deletion semantics below. + * + * `total_count` on `/search/issues` IS authoritative for its query, and is + * already consumed for cap-detection (see `PR_SEARCH_RESULT_CAP`). It is not + * promoted to the denominator because a search index is eventually + * consistent with the REST list this stream walks, so a benign index lag + * would read as coverage loss. + * + * The deeper constraint applies to ALL of them: PDPP deliberately RETAINS + * records after the provider deletes them, and GitHub genuinely deletes repos, + * issues and gists. A provider total therefore describes the surviving account + * and is legitimately SMALLER than what we hold. Any two-way + * `provider_total === held_count` check flags successful preservation as a + * defect. A sound anchor here would have to be the three-way relation + * `provider_total === live_holdings - known_tombstoned`, and this connector + * declares no tombstones at all (no `isTombstone`), so the third term is + * unavailable and the relation cannot be closed. + * + * A scalar also cannot distinguish missing from surplus from duplicated. If a + * real anchor is wanted later, compare the provider's ID SET against the held + * ID set — GitHub returns stable numeric ids on every one of these streams — + * and tombstone the upstream-absent ids rather than counting them as loss. */ async function declareListConsidered( ctx: StreamCtx, diff --git a/packages/polyfill-connectors/connectors/github/provider-total-not-an-anchor.test.ts b/packages/polyfill-connectors/connectors/github/provider-total-not-an-anchor.test.ts new file mode 100644 index 000000000..f8598a61f --- /dev/null +++ b/packages/polyfill-connectors/connectors/github/provider-total-not-an-anchor.test.ts @@ -0,0 +1,80 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * `public_repos` / `public_gists` are NOT coverage anchors. + * + * They are the most tempting numbers GitHub hands us — provider-reported, + * already fetched, already stored on the `user_stats` record. They are also + * wrong for the job, and this test exists so that stays discovered. + * + * They measure a strict SUBSET of what the collected streams walk: + * - `/user/repos` returns private and org repositories; `public_repos` + * counts only the user's own PUBLIC ones. + * - `/gists` returns secret gists; `public_gists` counts only public ones. + * + * Measured against this instance's live holdings when the anchor work was done: + * `public_repos: 94` against 575 held repositories (355 private, 465 org-owned), + * and `public_gists: 8` against 51 held gists — where the 8 matched the public + * gists EXACTLY and 43 secret gists sat outside the number entirely. Binding + * either as a denominator would assert a permanent false gap on a correct run. + * + * This test pins the structural fact behind those numbers: the stream walks + * repositories the provider scalar does not count. It is deliberately about the + * RELATIONSHIP, not a specific count. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { repoRecord, userStatsRecord } from "./parsers.ts"; + +test("a private or org repository is collected but is not counted by public_repos", () => { + // One public user-owned repo — the only kind `public_repos` counts. + const publicOwn = repoRecord({ + id: 1, + name: "public-own", + full_name: "octocat/public-own", + private: false, + owner: { login: "octocat", id: 99 }, + pushed_at: "2026-01-01T00:00:00Z", + } as never); + // A private repo and an org repo — both returned by `/user/repos`, and both + // invisible to `public_repos`. + const privateOwn = repoRecord({ + id: 2, + name: "private-own", + full_name: "octocat/private-own", + private: true, + owner: { login: "octocat", id: 99 }, + pushed_at: "2026-01-02T00:00:00Z", + } as never); + const orgRepo = repoRecord({ + id: 3, + name: "org-repo", + full_name: "acme/org-repo", + private: false, + owner: { login: "acme", id: 1234 }, + pushed_at: "2026-01-03T00:00:00Z", + } as never); + + const collected = [publicOwn, privateOwn, orgRepo]; + + // The provider scalar, reported by `/user`, sees only the single public + // user-owned repository. + const stats = userStatsRecord( + { id: 99, login: "octocat", public_repos: 1, public_gists: 0, followers: 0, following: 0 } as never, + "2026-01-04" + ); + + assert.equal(stats.public_repos, 1, "the provider scalar counts only public user-owned repositories"); + assert.equal(collected.length, 3, "the stream collects private and org repositories too"); + + // The load-bearing assertion: using the scalar as this stream's denominator + // would claim 2 of 3 collected repositories are missing, on a fully correct + // run. It is a different set, not a smaller measurement of the same set. + assert.notEqual( + stats.public_repos, + collected.length, + "public_repos must never be used as the repositories denominator — it measures a different set" + ); +}); diff --git a/packages/polyfill-connectors/connectors/gmail/all-mail-inventory.test.ts b/packages/polyfill-connectors/connectors/gmail/all-mail-inventory.test.ts new file mode 100644 index 000000000..5e710d713 --- /dev/null +++ b/packages/polyfill-connectors/connectors/gmail/all-mail-inventory.test.ts @@ -0,0 +1,270 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Gmail receives an authoritative mailbox-wide message count on every run and + * used to throw it away: the SELECT that opens All Mail returns IMAP `EXISTS`, + * and the connector read `uidValidity`/`uidNext` off the same `MailboxObject` + * while ignoring the count sitting next to them. The `labels` record body still + * carries the fossil of that decision — `message_count: null`. + * + * These tests drive the real `runAllMailPasses` with a stubbed IMAP client and + * assert on the emitted protocol, so they fail if the total stops being bound, + * stops being validated, or stops being carried across runs. + * + * The contract deliberately does NOT make `EXISTS` the `messages` + * DETAIL_COVERAGE denominator. That fact is per-page by design — the runtime + * admits a bounded continuation only when a page reports + * `considered === covered` on same-page facts — so substituting a 140k-message + * mailbox total there would make every run read `partial` forever and break + * continuation outright. A wrong denominator is worse than none. The mailbox + * total is disclosed as its own fact instead, and the per-page denominator is + * left alone; `messagesPageCoverageIsStillPerPage` below pins that boundary. + */ + +import assert from "node:assert/strict"; +import { mock, test } from "node:test"; +import type { FetchMessageObject, ImapFlow, ListResponse, MessageEnvelopeObject } from "imapflow"; +import { runAllMailPasses } from "./index.ts"; +import type { StreamRequest } from "./types.ts"; + +const FROZEN_NOW = "2026-04-21T00:00:00.000Z"; + +function makeAllMailMailbox(): ListResponse { + return { + delimiter: "/", + flags: new Set(["\\All"]), + listed: true, + name: "All Mail", + path: "[Gmail]/All Mail", + pathAsListed: "[Gmail]/All Mail", + parent: ["[Gmail]"], + parentPath: "[Gmail]", + specialUse: "\\All", + subscribed: true, + }; +} + +function makeRequested(streams: readonly string[]): Map { + return new Map(streams.map((name) => [name, { name }])); +} + +function makeMsg(uid: number): FetchMessageObject { + const envelope: MessageEnvelopeObject = { + date: new Date("2026-04-20T10:00:00.000Z"), + subject: "Test subject", + from: [{ name: "Alice", address: "alice@example.com" }], + to: [{ name: "Bob", address: "bob@example.com" }], + cc: [], + bcc: [], + messageId: ``, + }; + return { + seq: uid, + uid, + emailId: `gmmsgid-${uid}`, + threadId: "gmthrid-2222", + flags: new Set(["\\Seen"]), + labels: new Set(["\\Inbox"]), + envelope, + internalDate: new Date("2026-04-20T10:00:05.000Z"), + size: 1024, + } as FetchMessageObject; +} + +interface RunOutcome { + inventory: Record | undefined; + messages: Record[]; + state: Record | undefined; +} + +/** + * Run one All Mail pass against a stubbed mailbox and return the protocol it + * emitted. `mailboxOverrides` is how each test injects the EXISTS value under + * examination — including the malformed values a real server should never send + * but which must fail closed rather than read as a complete mailbox. + */ +async function runPass( + mailboxOverrides: Record, + state: Record = {} +): Promise { + const originalWrite = globalThis.process.stdout.write; + const protocolMessages: Record[] = []; + globalThis.process.stdout.write = ((data: string): boolean => { + if (typeof data === "string") { + try { + protocolMessages.push(JSON.parse(data) as Record); + } catch { + // Ignore non-protocol output. + } + } + return true; + }) as typeof process.stdout.write; + + try { + const client: Pick = { + close: mock.fn(), + download: () => { + throw new Error("download must not be called without attachments"); + }, + fetchOne: () => { + throw new Error("fetchOne must not be called without bodies"); + }, + search: mock.fn(async () => []), + mailbox: { + delimiter: "/", + exists: 1200, + flags: new Set(), + path: "[Gmail]/All Mail", + uidNext: 1201, + uidValidity: 123n, + ...mailboxOverrides, + } as ImapFlow["mailbox"], + // biome-ignore lint/suspicious/useAwait: async generator is required by the ImapFlow fetch shape. + async *fetch() { + for (const uid of [1, 2]) { + yield makeMsg(uid); + } + }, + }; + + await runAllMailPasses(client, makeAllMailMailbox(), state, { + emitRecord: async () => true, + emittedAt: FROZEN_NOW, + requested: makeRequested(["messages"]), + }); + } finally { + globalThis.process.stdout.write = originalWrite; + } + + return { + messages: protocolMessages, + inventory: protocolMessages.find((m) => m.type === "PROGRESS" && m.all_mail_inventory !== undefined), + state: protocolMessages.find((m) => m.type === "STATE" && m.stream === "messages"), + }; +} + +test("gmail all mail: the server-declared EXISTS is bound and disclosed, not discarded", async () => { + const { inventory } = await runPass({ exists: 1200 }); + + assert.ok(inventory, "a run must disclose the mailbox total the server handed it"); + assert.deepEqual(inventory.all_mail_inventory, { + all_mail_exists: 1200, + backfilled_through_uid: 0, + forward_floor_uid: 1200, + historical_backfill_complete: false, + uidvalidity: 123, + }); +}); + +test("gmail all mail: EXISTS is measured at the provider boundary, not derived from what was emitted", async () => { + // The pass emits exactly 2 message records but the mailbox holds 1200. If the + // total were ever derived from the emitted/collected count instead of read off + // the SELECT, this would report 2 and a 1198-message mailbox would read as + // fully accounted for — the exact defect this contract exists to prevent. + const { inventory } = await runPass({ exists: 1200 }); + const disclosed = inventory?.all_mail_inventory as { all_mail_exists: number }; + + assert.equal(disclosed.all_mail_exists, 1200, "the total is the server's count, independent of the 2 emitted"); + assert.notEqual(disclosed.all_mail_exists, 2, "the total must never collapse to the emitted count"); +}); + +test("gmail all mail: the EXISTS total is carried on STATE so the next run can compare it", async () => { + const { state } = await runPass({ exists: 1200 }); + const allMail = (state?.cursor as Record> | undefined)?.all_mail; + + assert.ok(allMail, "the messages STATE carries an all_mail cursor"); + assert.equal(allMail.exists, 1200, "the cursor persists the epoch's inventory size"); + assert.equal(allMail.uidvalidity, 123, "the count is only meaningful alongside the epoch it was measured in"); +}); + +test("gmail all mail: a missing EXISTS fails closed instead of reading as a complete mailbox", async () => { + // A silently absent total that defaults to success reproduces the bug being + // fixed, so absence must be louder than a wrong number, not quieter. + await assert.rejects( + () => runPass({ exists: undefined }), + /gmail_all_mail_exists_missing/, + "no EXISTS means no proof of inventory; the run must fail rather than assume" + ); +}); + +test("gmail all mail: a non-numeric EXISTS fails closed", async () => { + await assert.rejects(() => runPass({ exists: "1200" }), /gmail_all_mail_exists_not_number/); +}); + +test("gmail all mail: a non-finite EXISTS fails closed", async () => { + await assert.rejects(() => runPass({ exists: Number.POSITIVE_INFINITY }), /gmail_all_mail_exists_not_finite/); +}); + +test("gmail all mail: a fractional EXISTS fails closed", async () => { + await assert.rejects(() => runPass({ exists: 12.5 }), /gmail_all_mail_exists_not_integer/); +}); + +test("gmail all mail: a negative EXISTS fails closed", async () => { + await assert.rejects(() => runPass({ exists: -1 }), /gmail_all_mail_exists_negative/); +}); + +test("gmail all mail: a shrinking mailbox within one UID epoch throws", async () => { + // Mirrors Jellyfin's decreasing-total guard. Inside a single UIDVALIDITY the + // UID space is stable, so a smaller count is deletion or a server bug — either + // way it is a fact about the data that must not pass silently. + const priorState = { + messages: { all_mail: { uidvalidity: 123, uidnext: 1201, forward_uidnext: 1201, exists: 1200 } }, + }; + + await assert.rejects( + () => runPass({ exists: 900 }, priorState), + /gmail_all_mail_exists_decreased: 900 < 1200/, + "a mailbox that lost 300 messages must be surfaced, not absorbed" + ); +}); + +test("gmail all mail: a growing mailbox within one UID epoch is normal and does not throw", async () => { + const priorState = { + messages: { all_mail: { uidvalidity: 123, uidnext: 1201, forward_uidnext: 1201, exists: 1000 } }, + }; + const { inventory } = await runPass({ exists: 1200 }, priorState); + + assert.ok(inventory, "a run must disclose the mailbox total the server handed it"); + assert.equal( + (inventory.all_mail_inventory as { all_mail_exists: number }).all_mail_exists, + 1200, + "new mail is the expected case and must not be mistaken for corruption" + ); +}); + +test("gmail all mail: a UIDVALIDITY re-key does not read a lower count as loss", async () => { + // A UIDVALIDITY change means the server rebuilt the UID space. The old count + // describes a different space, so comparing across the boundary would turn + // every legitimate re-key into a spurious failure. The guard must scope its + // comparison to one epoch — this is the case that distinguishes a real + // decreasing-total check from a naive one. + const priorState = { + messages: { all_mail: { uidvalidity: 999, uidnext: 5000, forward_uidnext: 5000, exists: 4000 } }, + }; + const { inventory } = await runPass({ exists: 1200 }, priorState); + + assert.ok(inventory, "a run must disclose the mailbox total the server handed it"); + assert.equal( + (inventory.all_mail_inventory as { all_mail_exists: number }).all_mail_exists, + 1200, + "a re-key must read as a new epoch, never as a 2800-message loss" + ); + assert.equal((inventory.all_mail_inventory as { uidvalidity: number }).uidvalidity, 123); +}); + +test("gmail all mail: the messages page coverage denominator stays per-page", async () => { + // Guards the boundary this design turns on. The runtime's bounded-continuation + // check requires same-page `considered === covered`; if a future change routed + // the mailbox-wide EXISTS into this fact, a 1200-message mailbox walked 2 + // messages at a time would report 2/1200, read `partial` on every run forever, + // and lose the continuation proof. The mailbox total belongs beside this fact, + // not inside it. + const { messages } = await runPass({ exists: 1200 }); + const coverage = messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "messages"); + + assert.ok(coverage, "the bounded page still proves its own coverage"); + assert.equal(coverage.considered, 2, "the page denominator is the page, not the mailbox"); + assert.equal(coverage.covered, 2); + assert.notEqual(coverage.considered, 1200, "the mailbox total must not be substituted here"); +}); diff --git a/packages/polyfill-connectors/connectors/gmail/index.ts b/packages/polyfill-connectors/connectors/gmail/index.ts index bccc57f3e..af5bf6c83 100644 --- a/packages/polyfill-connectors/connectors/gmail/index.ts +++ b/packages/polyfill-connectors/connectors/gmail/index.ts @@ -56,7 +56,6 @@ import { type BodyPartSelection, bigintToCursor, bigintToNumber, - buildDeltaMessageRecord, buildMessageBodyRecord, buildMessageRecord, buildThreadRecord, @@ -172,7 +171,40 @@ function getReadline(): ReadlineInterface { // control chars out of body text before encoding. The JSONL encoding // itself — BigInt coercion + U+2028/U+2029 escaping — lives in // `stringifyForJsonl`. +// DONE is terminal on the wire, and nothing below this line may write after +// it. The runtime's `handleMsg` latches the first DONE of any status and +// throws `Connector emitted after DONE` on the next message, failing +// the whole run as a protocol violation. +// +// Enforcing that here rather than at each terminal path is deliberate: the +// terminal paths do not stop this process. `flushAndExit` only registers +// listeners and returns (see `connector-exit.ts` — it waits for the runtime +// to close stdin, up to 30 minutes), so after `fail()` or a rejection +// handler emits DONE, the interrupted work is still on the stack. An +// in-flight `for await` over an IMAP FETCH keeps iterating and keeps calling +// `emitRecord`, and each of those writes lands after the DONE. +// +// That is what production hit: gmail runs failed `connector_protocol_violation +// "Connector emitted RECORD after DONE"` while reporting more records emitted +// than the runtime ever flushed — the surplus was written into a channel that +// had already closed. Making every terminal path unwind perfectly would be a +// standing obligation on code that mostly cannot see it coming; making the +// write refuse is a property of the channel. +// +// Suppressed messages go to stderr, so a swallowed record is diagnosable +// rather than silent. Emitting them would fail the run outright, which is +// strictly worse: the records ingested before DONE are durable and the run's +// own terminal status is already decided. +let doneEmitted = false; + function emit(msg: EmittedMessage): Promise { + if (doneEmitted) { + process.stderr.write(`[gmail] suppressed ${String(msg.type)} after DONE\n`); + return Promise.resolve(); + } + if (msg.type === "DONE") { + doneEmitted = true; + } const line = stringifyForJsonl(sanitizeForJsonl(msg)); const ok = process.stdout.write(line); if (ok) { @@ -185,6 +217,17 @@ function emit(msg: EmittedMessage): Promise { }); } +/** Test-only reset of the terminal latch; the real process never reuses it. */ +export function __resetDoneLatchForTests(): void { + doneEmitted = false; +} + +/** Test-only handle on the real stdout emitter, so the latch is exercised + * through the same function every production path writes through. */ +export function gmailEmitForTests(msg: EmittedMessage): Promise { + return emit(msg); +} + function flushAndExit(code: number): void { flushAndExitAfterRuntimeAck(code); } @@ -305,6 +348,18 @@ interface AttachmentHydrationFailure { readonly stage: AttachmentHydrationFailureStage; } +/** + * A `failed` hydration's stage, if known, else the same `unclassified_failed` + * bucket already used by `attachment_hydration_failure_outcome` telemetry. + * This is the ONLY hydration-failure detail that ever reaches a DETAIL_GAP: + * a bounded category, never `AttachmentRecord.hydration_error` (raw IMAP/blob + * text that can embed hostnames, tokens, or URLs — see + * `buildAttachmentDetailGap`'s doc comment on why that string never crosses). + */ +function attachmentHydrationFailureClass(failure: AttachmentHydrationFailure | null): string { + return failure?.stage ?? "unclassified_failed"; +} + export interface AttachmentHydrationResult { readonly failure: AttachmentHydrationFailure | null; readonly record: AttachmentRecord; @@ -389,12 +444,28 @@ export function formatAttachmentBackfillSummary(summary: AttachmentBackfillSumma * - `failed` → `gapKeys` (a retryable detail gap to re-attempt next run). * - `too_large` or `deferred` → unaccounted (required denominator only). */ +/** A failed attachment hydration, paired with why it failed (bounded, non-secret). */ +export interface FailedAttachmentRecord { + readonly failureClass: string; + readonly record: AttachmentRecord; +} + export interface AttachmentDetailCoverage { /** * Failed attachment records, retained so the run can emit one matching - * DETAIL_GAP per `gapKeys` entry. + * DETAIL_GAP per `gapKeys` entry. The host commit-gate credits a missing + * required key only when it is hydrated, optional-skipped, or backed by a + * durable pending DETAIL_GAP — `gap_keys` alone do not satisfy it. Each + * record's `id` is exactly the value that landed in `gapKeys`, keeping the + * gap's `record_key` and the coverage key a single source of truth. + * + * `failureClass` is the bounded, non-secret classification (see + * `attachmentHydrationFailureClass`) — the only hydration-failure detail + * that reaches the emitted DETAIL_GAP. Without it, a gap that keeps + * failing on every retry never records why: `attempt_count` climbs but + * `last_error_json` stays null forever (the 2026-08 Gmail 5-row defect). */ - failedRecords: AttachmentRecord[]; + failedRecords: FailedAttachmentRecord[]; gapKeys: string[]; hydratedKeys: string[]; requiredKeys: string[]; @@ -410,8 +481,28 @@ export function makeAttachmentDetailCoverage(): AttachmentDetailCoverage { * its terminal `hydration_status`. A `deferred` status (never hydrated this * run) is still a considered key but has no terminal outcome bucket, so it * counts only toward the denominator. Pure: mutates the passed accumulator. + * + * `failure` is the same `AttachmentHydrationFailure | null` the hydrator + * already computes for aggregate telemetry — passed through here so a + * `failed` outcome's DETAIL_GAP can carry a bounded, non-secret `error.class` + * instead of recording an attempt with no evidence of why it failed. */ -export function recordAttachmentCoverage(coverage: AttachmentDetailCoverage, record: AttachmentRecord): void { +export function recordAttachmentCoverage( + coverage: AttachmentDetailCoverage, + record: AttachmentRecord, + failure: AttachmentHydrationFailure | null = null +): void { + // The runtime rejects a DETAIL_COVERAGE whose required_keys repeats a key. + // The same attachment id can legitimately reach this accumulator twice in a + // run -- a message re-observed across pages, or a retry re-walking a + // partially hydrated thread -- so pushing unconditionally turns an ordinary + // duplicate observation into a hard run failure ("invalid + // DETAIL_COVERAGE.required_keys: duplicate key"), which is what took Gmail + // out on 2026-08-17. Record each key once; the terminal-outcome buckets + // below are already keyed by the same id and stay consistent with it. + if (coverage.requiredKeys.includes(record.id)) { + return; + } coverage.requiredKeys.push(record.id); switch (record.hydration_status) { case "hydrated": @@ -419,7 +510,10 @@ export function recordAttachmentCoverage(coverage: AttachmentDetailCoverage, rec return; case "failed": coverage.gapKeys.push(record.id); - coverage.failedRecords.push(record); + // Retain the record so a matching DETAIL_GAP is emitted for this key. + // `gap_keys` on DETAIL_COVERAGE are not enough on their own: the host + // commit-gate requires a durable pending DETAIL_GAP to credit the key. + coverage.failedRecords.push({ failureClass: attachmentHydrationFailureClass(failure), record }); return; case "too_large": case "deferred": @@ -436,10 +530,8 @@ export function recordAttachmentCoverage(coverage: AttachmentDetailCoverage, rec /** * Build the per-run attachments DETAIL_COVERAGE after the detail lane settles. - * A requested attachments pass that scans the parent `messages` boundary and - * finds zero attachment parts has a real empty denominator: `required_keys: []` - * means "nothing owed", not "unknown". The list cursor that anchors this - * detail pass lives on `messages`, so that is the `state_stream`. + * The list cursor that anchors this detail pass lives on `messages`, so that is + * the `state_stream`. * Reference-only: this reuses DETAIL_COVERAGE without promoting it to portable * protocol. * @@ -459,15 +551,69 @@ export function buildAttachmentDetailCoverageMessage(coverage: AttachmentDetailC } /** - * Emit the per-run attachments DETAIL_COVERAGE after the detail lane settles. + * Whether this run walked a boundary that can support an attachments coverage + * claim at all. + * + * `attachments` has no enumeration of its own: an attachment is only ever + * discovered by decoding the BODYSTRUCTURE of a message the run already + * fetched. Its denominator is therefore inherited from whatever slice of the + * mailbox the run walked — and on an ordinary scheduled run that slice is the + * incremental forward window plus the MODSEQ delta, i.e. a CHANGE FEED, not an + * inventory. + * + * That is exactly the apple_contacts `contactsBoundaryEstablished` situation + * (`connectors/apple_contacts/index.ts`), and it needs the same answer. A quiet + * 16-minute run that observes no new mail decodes no attachment parts and would + * otherwise emit `considered: 0, covered: 0` — numbers that are trivially + * self-consistent and read downstream as "this stream is proven complete", + * when the honest statement is "this run proved nothing about the mailbox". + * Worse, the two are indistinguishable to the gate: a stream that has walked + * almost nothing and a stream that is genuinely fully collected emit the + * identical fact. Withholding is what keeps those two apart. + * + * The one boundary that does justify a whole-stream claim is a COMPLETED + * historical `messages` walk: the historical pass fetches every UID below the + * forward floor and `processMessage` decodes attachments for each one, so once + * `messages.backfill.completed_at` is set, every message in All Mail has had + * its attachment parts enumerated. Until then the mailbox below the resume + * point is simply unwalked, and no count taken above it describes it. + * + * Emitting nothing leaves the stream honestly unproven (the coherence + * contract's `checkpoint_only`/`no_proof_strategy` -> axis `unknown`) rather + * than falsely complete. `unknown` is a worse-looking verdict than a green + * `0/0` and that is the point: it is the true one, and it is the verdict an + * owner can act on. + */ +export function attachmentsCoverageBoundaryEstablished(session: { messagesBackfill: MessagesBackfillCursor }): boolean { + return typeof session.messagesBackfill.completed_at === "string" && session.messagesBackfill.completed_at !== ""; +} + +/** + * Emit the per-run attachments DETAIL_COVERAGE after the detail lane settles — + * but only when the run established a boundary that can support the claim + * (see `attachmentsCoverageBoundaryEstablished`). * * Extracted from `runAllMailPasses` to keep that orchestrator under the * cognitive-complexity ceiling (authoring guide §"Rules the tooling enforces"). */ -async function emitAttachmentDetailCoverage(coverage: AttachmentDetailCoverage | undefined): Promise { +async function emitAttachmentDetailCoverage( + coverage: AttachmentDetailCoverage | undefined, + boundaryEstablished: boolean +): Promise { if (!coverage) { return; } + if (!boundaryEstablished) { + // Deliberately not a SKIP_RESULT: nothing was skipped by policy. The run + // simply has no denominator worth reporting, and says so by staying quiet. + await emit({ + type: "PROGRESS", + stream: "attachments", + message: + "Withholding attachments coverage: the historical messages walk has not completed, so this run's attachment counts describe an incremental window, not the mailbox", + }); + return; + } await emit(buildAttachmentDetailCoverageMessage(coverage)); } @@ -488,11 +634,23 @@ async function emitAttachmentDetailCoverage(coverage: AttachmentDetailCoverage | * order-detail gap; retrying next run is the honest, non-destructive default. * * Reference-only and bounded: only opaque message and part identifiers cross - * (X-GM-MSGID, the BODYSTRUCTURE part index, and the attachment id). No - * filename, content, blob bytes, raw error text, tokens, cookies, URLs, request - * bodies, or payload snippets are carried. + * (X-GM-MSGID, the BODYSTRUCTURE part index, and the attachment id), plus — + * when `failureClass` is supplied — a bounded, non-secret failure category + * (e.g. `imap_download_failed`, `blob_upload_http_5xx`, `unclassified_failed`; + * see `attachmentHydrationFailureClass`). No filename, content, blob bytes, + * raw error text, tokens, cookies, URLs, request bodies, or payload snippets + * are ever carried — `AttachmentRecord.hydration_error` (which CAN embed + * that raw text) never crosses this boundary, by construction: only the + * caller-supplied category string can reach `error.class`. + * + * Without a failure class, a repeatedly-retried gap accrues `attempt_count` + * on every run but never records why it keeps failing — `last_error_json` + * stays null forever, which is indistinguishable from "never attempted" and + * blocks the stream from ever proving its remaining gaps are impossible + * (2026-08 Gmail: 5 `attachments` gaps reached terminal status with 37-117 + * attempts and a permanently-null `last_error_json`). */ -export function buildAttachmentDetailGap(attachment: AttachmentRecord): DetailGapMessage { +export function buildAttachmentDetailGap(attachment: AttachmentRecord, failureClass?: string): DetailGapMessage { return buildDetailGap({ stream: "attachments", parentStream: "messages", @@ -504,6 +662,7 @@ export function buildAttachmentDetailGap(attachment: AttachmentRecord): DetailGa part_index: attachment.part_index, attachment_id: attachment.id, }, + ...(failureClass ? { error: { class: failureClass } } : {}), }); } @@ -586,7 +745,7 @@ async function emitAttachmentRecords( // Record the outcome BEFORE emitting so the coverage denominator counts // every attempt even if the emit is scope-filtered downstream. if (deps.attachmentCoverage) { - recordAttachmentCoverage(deps.attachmentCoverage, hydrated); + recordAttachmentCoverage(deps.attachmentCoverage, hydrated, hydration.failure); } const emitted = await deps.emitRecord("attachments", { ...hydrated }); // Only `hydrated` may acknowledge a served gap as recovered. Unaccounted @@ -619,8 +778,8 @@ async function emitAttachmentDetailGaps(coverage: AttachmentDetailCoverage | und if (!coverage) { return; } - for (const attachment of coverage.failedRecords) { - await emit(buildAttachmentDetailGap(attachment)); + for (const failed of coverage.failedRecords) { + await emit(buildAttachmentDetailGap(failed.record, failed.failureClass)); } } @@ -802,6 +961,45 @@ export async function emitMessagesPass( return { considered: metas.length, covered: count }; } +/** + * Disclose the mailbox-wide inventory total against this run's walk boundary. + * + * Gmail's `messages` DETAIL_COVERAGE is deliberately PER-PAGE: the runtime's + * `isHealthyBoundedContinuation` admits a bounded page only when + * `considered === covered` on same-page facts, so the page denominator must + * stay the page. Overwriting it with the mailbox-wide `EXISTS` would make every + * run of a 140k-message mailbox read `partial` forever and would break the + * continuation contract outright — a wrong denominator, not a better one. + * + * The honest place for the provider total is therefore a SEPARATE fact: the + * server-declared inventory size, plus how much of the UID space this + * connector's two cursors have actually claimed. `EXISTS` is validated + * fail-closed in `validateExistsTotal` before this point, so a missing or + * malformed total has already thrown and can never reach here as a silent + * "complete". What this adds is visibility: the number the provider asserts, + * next to the boundary we have walked to. + */ +async function emitAllMailInventoryDisclosure( + emitFn: (msg: EmittedMessage) => Promise, + session: AllMailSession, + historicalCursor: MessagesBackfillCursor, + forwardFloorUid: number +): Promise { + const backfilledThroughUid = historicalCursor.backfilled_through_uid ?? 0; + await emitFn({ + type: "PROGRESS", + stream: "messages", + message: `All Mail reports ${session.existsTotal} messages; historical walk is through UID ${backfilledThroughUid} of ${forwardFloorUid}`, + all_mail_inventory: { + all_mail_exists: session.existsTotal, + backfilled_through_uid: backfilledThroughUid, + forward_floor_uid: forwardFloorUid, + historical_backfill_complete: historicalCursor.completed_at !== null, + uidvalidity: session.uidvalidityNum, + }, + }); +} + /** Record a bounded terminal outcome so a poison UID cannot replay forever. */ async function emitHistoricalMessageSkip(deps: PerMessageDeps, uid: number | undefined): Promise { await deps.emitProtocol({ @@ -1049,15 +1247,57 @@ async function findAllMailbox(client: ImapFlow): Promise { interface AllMailSession { attachmentBackfill: AttachmentAllMailCursor; + /** + * IMAP `EXISTS` for All Mail: the server's own count of the messages in the + * mailbox, taken from the SELECT this run already performed. This is the + * only mailbox-wide inventory total Gmail hands us, and it is measured at + * the provider boundary — before a single UID is walked and independently + * of what any pass emits. + */ + existsTotal: number; fullResync: boolean; highestModseqCursor: number | string | null; messagesBackfill: MessagesBackfillCursor; + priorExistsTotal: number | undefined; priorModseq: number | string | null | undefined; priorUidnext: number; uidnext: number | undefined; uidvalidityNum: number; } +/** + * Validate the IMAP `EXISTS` count for All Mail and bind it as the mailbox's + * inventory total. Fail closed on missing/malformed, mirroring Jellyfin's + * `validateTotalRecordCount`: a silently absent total that reads as success is + * precisely the bug this contract exists to prevent. + * + * `priorTotal` is compared only WITHIN a UIDVALIDITY epoch (the caller passes + * `undefined` across a re-key). A decrease inside one epoch is real deletion or + * a server bug and must throw rather than pass quietly. Across a re-key the UID + * space was rebuilt, so a different count is expected and is not loss. + */ +function validateExistsTotal(value: unknown, priorTotal: number | undefined): number { + if (value === undefined || value === null) { + throw new Error("gmail_all_mail_exists_missing: SELECT reported no EXISTS for All Mail"); + } + if (typeof value !== "number") { + throw new Error("gmail_all_mail_exists_not_number"); + } + if (!Number.isFinite(value)) { + throw new Error("gmail_all_mail_exists_not_finite"); + } + if (!Number.isInteger(value)) { + throw new Error("gmail_all_mail_exists_not_integer"); + } + if (value < 0) { + throw new Error("gmail_all_mail_exists_negative"); + } + if (priorTotal !== undefined && value < priorTotal) { + throw new Error(`gmail_all_mail_exists_decreased: ${value} < ${priorTotal}`); + } + return value; +} + /** * Narrow the MailboxObject + prior state into the (UIDVALIDITY, cursor, * resync-flag) triple the fetch loop needs. Returns null when UIDVALIDITY @@ -1084,6 +1324,15 @@ function deriveAllMailSession(mailbox: MailboxObject, state: Record { const priorEnd = args.prior.backfilled_through_uid ?? 0; @@ -1878,7 +2185,7 @@ async function settleServedAttachmentRecoveryAttempt( ): Promise { const hydrated = hydration.record; if (deps.attachmentCoverage) { - recordAttachmentCoverage(deps.attachmentCoverage, hydrated); + recordAttachmentCoverage(deps.attachmentCoverage, hydrated, hydration.failure); } const emitted = await deps.emitRecord("attachments", { ...hydrated }); if (emitted && hydrated.hydration_status === "hydrated") { @@ -2837,6 +3144,42 @@ interface ImapDownloadResponse { meta?: ImapDownloadMeta; } +/** + * Download one attachment PART's bytes. + * + * `expectedSize` is deliberately taken from BODYSTRUCTURE (`attachment.size_bytes`) + * and NOT from the download response's `meta.expectedSize`. + * + * imapflow populates `meta.expectedSize` from the FETCH `RFC822.SIZE` item + * (`lib/imap-flow.js`: `meta = { expectedSize: response.size }`, where `size` + * is requested as the `RFC822.SIZE` atom in `lib/commands/fetch.js`). + * RFC822.SIZE is the size of the ENTIRE MESSAGE — every part, every MIME + * header, every boundary — not of the part being downloaded. It is the same + * number for every part of a multipart message. + * + * Feeding a message-scoped size into a per-attachment cap made a message + * reject ALL of its attachments whenever their SUM crossed the cap, even + * though no single one came close. Observed live on the owner's mailbox: + * 68 attachments marked `too_large` across 18 messages, of which only 2 were + * genuinely over the 25 MiB cap; one 3,080-byte attachment was rejected as + * "29830196 > 26214400 bytes". Two parts of one message recorded the byte-for-byte + * identical "observed" size — impossible for real per-part sizes, and the tell + * that the number was never the part's. + * + * That number is not just a wrong skip: `isProvenUnfillableGap` + * (`server/connector-gap-classification.ts`) parses this exact + * "exceeds max size: > " text as DURABLE PROOF that an item + * is permanently uncollectable. A message-scoped size therefore manufactured + * per-item impossibility proofs for items that are collectible, which is the + * fabricated-evidence failure that predicate exists to refuse. + * + * BODYSTRUCTURE's per-part `size` is the only per-part size IMAP gives us + * before transfer, so it is the only honest pre-flight number. When it is + * absent we report `null` (unknown) rather than substituting a + * message-scoped stand-in: `enforceMaxBytes` still counts real bytes + * mid-stream, so an under-reported or missing size is caught by observation + * instead of by a guess. + */ export async function fetchAttachmentPart( client: Pick, msg: FetchMessageObject, @@ -2850,7 +3193,7 @@ export async function fetchAttachmentPart( })) as ImapDownloadResponse; return { content: response.content, - expectedSize: typeof response.meta?.expectedSize === "number" ? response.meta.expectedSize : attachment.size_bytes, + expectedSize: attachment.size_bytes, mimeType: response.meta?.contentType || attachment.content_type || DEFAULT_ATTACHMENT_MIME_TYPE, }; } @@ -2893,12 +3236,35 @@ export function validateAttachmentHydrationPreflight(args: { // ─── Delta pass (flag/label changes since priorModseq) ────────────────── -async function runDeltaPass( +/** + * Emit flag/label changes for messages modified since `priorModseq`. + * + * The envelope is fetched alongside the flags. That is not an optimization — + * it is what makes the pass safe. PDPP records are whole-document upserts + * (`records` is keyed `UNIQUE(connector_instance_id, stream, record_key)` and + * ingest replaces `record_json`), so emitting a partial `messages` record + * overwrites the stored row rather than merging into it. An envelope-free + * delta record therefore nulls `subject`, `from_email`, `date`, `size_bytes`, + * and `snippet` on a message that had them, and sets `received_at` to the run + * clock — silently destroying already-collected history every time a label or + * a `\Seen` flag changes. + * + * A message whose envelope the server does not return is skipped rather than + * emitted partially: losing a flag update is recoverable on the next pass, + * losing the envelope is not. + * + * `fetchBodiesFn` is the same seam `processMessage` uses. Called with + * `wantBodies: false` it fetches at most `SNIPPET_FETCH_MAX_BYTES` of the + * plain part — enough to rebuild `snippet`, without touching the + * externally-throttled full-body/attachment path. + */ +export async function runDeltaPass( client: Pick, session: AllMailSession, requested: Map, emitRecord: EmitRecordFn, - receivedAtFallback: string + receivedAtFallback: string, + fetchBodiesFn: FetchBodiesFn ): Promise { if (session.fullResync || session.priorModseq === undefined || session.priorModseq === null) { return; @@ -2909,41 +3275,81 @@ async function runDeltaPass( type: "PROGRESS", message: `Fetching flag/label deltas since modseq=${String(priorModseq)}`, }); + // `envelope`/`internalDate`/`size`/`bodyStructure` ride along with the flags + // so the emitted record is whole. See this function's doc comment: a partial + // record is a destructive upsert, not a cheap one. BODYSTRUCTURE is metadata + // only — it is what `has_attachments` is derived from, and fetching it does + // not pull attachment or body content. const deltaQuery: ExtendedFetchQuery = { uid: true, flags: true, labels: true, threadId: true, emailId: true, - envelope: false, + envelope: true, + internalDate: true, + size: true, + bodyStructure: true, }; + // Phase A: drain the delta FETCH completely before issuing any other IMAP + // command. Phase B: fetch snippets and emit. + // + // The split is required, not stylistic — it is the same rule the message + // pass states at its own Phase A/B boundary: imapflow multiplexes one + // command at a time over a single connection, so a nested command issued + // while the outer iterator is still open hangs that iterator. Calling + // `fetchBodiesFn` (a `fetchOne`) inside the `for await` did exactly that. + // + // `fetchBodies` swallows its own errors, so the nested call surfaced not as + // a body-fetch failure but as a wedged connection: the run stopped making + // progress after "Fetching flag/label deltas", sat until a timeout, and + // then tripped the runtime's post-DONE guard while the abandoned iterator + // drained. Draining first keeps the envelope guarantee below intact and + // costs only the metadata already held in memory. + const deltaMetas: FetchMessageObject[] = []; for await (const msg of client.fetch("1:*", deltaQuery, { uid: true, changedSince: priorModseqBig, })) { - const gmMsgid = String(msg.emailId ?? ""); - if (!gmMsgid) { + if (!requested.has("messages")) { continue; } - // Flag/label delta update: emit a tombstone-free upsert of the message - // envelope (minimal fields since envelope not re-fetched). For now, we - // emit a RECORD with the same id so the RS upserts flag/label state. - // Note: PDPP records are "whole-document" upserts in the current RS, - // so this delta path is effectively a full re-fetch. Simpler: mark - // this path as "only flags" by emitting the fields we have plus nulls. - // For robustness, let's actually re-fetch envelope in v2. For v1, emit - // flags only. - if (!requested.has("messages")) { + if (!String(msg.emailId ?? "")) { + continue; + } + // No envelope means no safe record to write. Skipping preserves the + // stored row; emitting would blank it. + if (!msg.envelope) { + continue; + } + deltaMetas.push(msg); + } + + for (const msg of deltaMetas) { + const gmMsgid = String(msg.emailId ?? ""); + const env = msg.envelope; + if (!env) { continue; } + const receivedAt = perMessageInternalDateToIso(msg.internalDate, () => receivedAtFallback); + // Bounded snippet-only body read, exactly as the forward pass does for a + // messages-without-bodies scope. Sequential by necessity: this is one IMAP + // command at a time on a connection that is not concurrent. + const { snippet } = await fetchBodiesFn(msg, selectBodyParts(msg.bodyStructure, false), false, true); await emitRecord( "messages", - buildDeltaMessageRecord({ + buildMessageRecord({ + attachmentsCount: decodeBodystructureForAttachments(msg.bodyStructure, gmMsgid, receivedAt).length, + dateHeader: env.date ? new Date(env.date).toISOString() : null, + envelope: env, flagsArr: toFlagsArray(msg.flags), gmMsgid, gmThrid: String(msg.threadId ?? ""), labels: toLabelsArray(msg.labels), - receivedAtFallback, + rawHeaders: msg.headers, + receivedAt, + sizeBytes: typeof msg.size === "number" ? msg.size : null, + snippet, }) ); } @@ -3101,8 +3507,21 @@ export async function runAllMailPasses( deps.requested.has("attachments") || deps.requested.has("threads"); const forwardFetchRange = selectAllMailFetchRange(session, deps.requested); - const historicalTargetUid = - session.messagesBackfill.target_uid ?? Math.max(0, (session.uidnext ?? session.priorUidnext) - 1); + // Where the forward walk resumes on the NEXT run, and therefore the last UID + // it will not fetch. The historical walk must own everything up to here or + // the interval between the two belongs to neither (see + // `resolveMessagesBackfillTargetUid`). + // + // On a full resync there is no forward range at all: the forward watermark + // below is written from the live `uidnext` without a single UID having been + // walked, so the ENTIRE mailbox below it is historical work and the ceiling + // must say so. Reading the same `session.uidnext` the watermark is written + // from is what keeps the two pointers describing one space. + const forwardFloorUid = Math.max(0, (session.uidnext ?? session.priorUidnext) - 1); + const historicalTargetUid = resolveMessagesBackfillTargetUid({ + forwardFloorUid, + prior: session.messagesBackfill, + }); const historicalCursor: MessagesBackfillCursor = { ...session.messagesBackfill, backfilled_through_uid: session.messagesBackfill.backfilled_through_uid ?? 0, @@ -3186,10 +3605,12 @@ export async function runAllMailPasses( recoveryOnly: true, session, }); - // Recovery-only mode still reports attachment evidence for the served - // gaps it actually touched, but it returns before the ordinary Gmail - // walk and cursor advancement. - await emitAttachmentDetailCoverage(attachmentCoverage); + // Recovery-only mode touches exactly the served gaps it was handed and + // returns before the ordinary Gmail walk, so it never establishes a + // mailbox boundary — its counts describe a retry list, not an inventory. + // It still emits its DETAIL_GAPs below; only the coverage CLAIM is + // withheld (see `attachmentsCoverageBoundaryEstablished`). + await emitAttachmentDetailCoverage(attachmentCoverage, false); await emitAttachmentDetailGaps(attachmentCoverage); } return; @@ -3287,6 +3708,7 @@ export async function runAllMailPasses( stream: "messages", message: `Collected ${metas.length} headers; beginning body pass`, }); + await emitAllMailInventoryDisclosure(emit, session, historicalCursor, forwardFloorUid); const perMessageDeps: PerMessageDeps = { ...(attachmentCoverage ? { attachmentCoverage } : {}), emitProtocol: emit, @@ -3308,18 +3730,22 @@ export async function runAllMailPasses( historicalMetas ); const forwardMessageCoverage = await emitMessagesPass(perMessageDeps, forwardMetas); - if (deps.requested.has("message_bodies")) { - await emit( - buildDetailCoverageMessage({ - considered: historicalMessageCoverage.considered + forwardMessageCoverage.considered, - covered: historicalMessageCoverage.covered + forwardMessageCoverage.covered, - hydratedKeys: [], - requiredKeys: [], - stateStream: "messages", - stream: "message_bodies", - }) - ); - } + // `message_bodies` deliberately emits NO DETAIL_COVERAGE. The manifest + // declares it `state_stream: messages`, i.e. a static single-parent detail + // stream, and such a stream's checkpoint status is projected from the + // declared parent's own commit outcome — the runtime rejects the run + // outright if it emits coverage of its own + // (`validateDetailCoverageAgainstManifest`). + // + // It could not honestly emit one anyway. The numbers available here are the + // *parent* pass's considered/covered — how many MESSAGES were walked, not + // how many bodies were hydrated against a per-key denominator. Re-reporting + // the parent's counts under the body stream's name is the `covered == + // considered` fabrication this codebase has worked to eliminate: it would + // claim every walked message proved a body, including the ones whose body + // fetch was skipped or failed. `attachments` is the contrast — it earns its + // coverage from a real attempt-per-key tally (see + // `emitAttachmentDetailCoverage`), which is why it may emit at all. await runAttachmentBackfillAndRecoveryPass({ allMail, @@ -3339,7 +3765,7 @@ export async function runAllMailPasses( // record (primary pass + historical backfill) has settled and before the // messages STATE cursor commits — the ordering the progress-evidence // contract expects (records, then DETAIL_COVERAGE, then STATE). - await emitAttachmentDetailCoverage(attachmentCoverage); + await emitAttachmentDetailCoverage(attachmentCoverage, attachmentsCoverageBoundaryEstablished(session)); // Then one matching DETAIL_GAP per failed attachment, so the commit-gate can // credit each gap_keys entry against a durable pending gap. Without this the // gate aborts an otherwise-successful run and the messages cursor never @@ -3347,14 +3773,35 @@ export async function runAllMailPasses( await emitAttachmentDetailGaps(attachmentCoverage); // Pass 2: detect flag/label changes on already-seen messages (incremental only) - await runDeltaPass(client, session, deps.requested, deps.emitRecord, deps.emittedAt); + await runDeltaPass(client, session, deps.requested, deps.emitRecord, deps.emittedAt, fetchBodiesBound); if (messageHistoryRequested && historicalFetchRange) { const historicalPageEndUid = Number(historicalFetchRange.split(":")[1]); + // Sums BOTH passes, like the `message_bodies` DETAIL_COVERAGE above: the + // forward pass runs in the same call to `runAllMailPasses` and emits its + // own `messages` records via the same shared `emitRecord`, so the raw + // collected-record count already includes them. Reporting only + // `historicalMessageCoverage` undercounted the denominator against that + // total every scheduled run with new mail waiting alongside a pending + // historical backfill. + // + // Both emissions below MUST read these same two numbers. The runtime's + // `isHealthyBoundedContinuation` accepts a bounded page only when the + // continuation's considered/covered are identical to the DETAIL_COVERAGE + // fact's — it binds a continuation to complete *same-page* facts, and the + // summing above is what defines "the page" here. Feeding the continuation + // historical-only counts desyncs the pair by exactly the forward-pass + // count, the identity check fails, and the stream degrades to a + // retryable_gap instead of deriving complete. The sibling `threads` + // emission never desyncs precisely because it feeds one variable to both. + const messagesCoverage = { + considered: historicalMessageCoverage.considered + forwardMessageCoverage.considered, + covered: historicalMessageCoverage.covered + forwardMessageCoverage.covered, + }; await emit( buildDetailCoverageMessage({ - considered: historicalMessageCoverage.considered, - covered: historicalMessageCoverage.covered, + considered: messagesCoverage.considered, + covered: messagesCoverage.covered, hydratedKeys: [], requiredKeys: [], stateStream: "messages", @@ -3364,8 +3811,8 @@ export async function runAllMailPasses( if (historicalPageEndUid < historicalTargetUid) { await emitHistoricalContinuationSkip(emit, "messages", { boundary: String(historicalCursor.uidvalidity), - considered: historicalMessageCoverage.considered, - covered: historicalMessageCoverage.covered, + considered: messagesCoverage.considered, + covered: messagesCoverage.covered, slice_start: Number(historicalFetchRange.split(":")[0]), slice_end: historicalPageEndUid, }); @@ -3388,6 +3835,11 @@ export async function runAllMailPasses( prior: historicalCursor, }); } else { + // A completed walk with no fetch range left. This is reachable only when + // the ceiling did NOT move — `historicalTargetUid` is raised before the + // range is selected, so any reopened band yields a non-null range and is + // handled by the branch above. With an unchanged ceiling this cursor is + // already at its target, so carrying it forward is exact. nextMessagesBackfill = historicalCursor; } } @@ -3405,6 +3857,9 @@ export async function runAllMailPasses( uidnext: nextUidnext, forward_uidnext: nextForwardUidnext, highest_modseq: session.highestModseqCursor ?? null, + // Carry the mailbox's own EXISTS forward so the next run in this epoch + // can prove the inventory did not shrink underneath us. + exists: session.existsTotal, }, ...(nextMessagesBackfill ? { backfill: nextMessagesBackfill } : {}), }, diff --git a/packages/polyfill-connectors/connectors/gmail/integration.test.ts b/packages/polyfill-connectors/connectors/gmail/integration.test.ts index c6c744db7..ef9186f32 100644 --- a/packages/polyfill-connectors/connectors/gmail/integration.test.ts +++ b/packages/polyfill-connectors/connectors/gmail/integration.test.ts @@ -32,6 +32,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; import { Readable } from "node:stream"; import { mock, test } from "node:test"; import type { @@ -46,6 +47,7 @@ import { buildFullScanCoverageMessage } from "../../src/connector-runtime.ts"; import { ReferenceBlobUploadFailure, runtimeBlobUploadAvailable } from "../../src/reference-blob-uploader.ts"; import { type EmittedRecord, makeRecordingEmit, type RecordedEvent } from "../../src/test-harness.ts"; import { + __resetDoneLatchForTests, ATTACHMENT_BACKFILL_PAGE_DEFAULT_BYTES, ATTACHMENT_BACKFILL_PAGE_MAX_BYTES, ATTACHMENT_BACKFILL_PAGE_MIN_BYTES, @@ -58,6 +60,7 @@ import { addAttachmentBackfillRecordToSummary, advanceMessagesBackfillCursor, attachmentBackfillPageByteBudget, + attachmentsCoverageBoundaryEstablished, buildAttachmentDetailCoverageMessage, buildAttachmentDetailGap, buildAttachmentTransferProgressMessage, @@ -71,7 +74,9 @@ import { emitMessagesPass, type FetchBodiesFn, type FetchedBodies, + fetchAttachmentPart, formatAttachmentBackfillSummary, + gmailEmitForTests, type HydrateAttachmentFn, isoToImapDate, makeAttachmentDetailCoverage, @@ -90,8 +95,10 @@ import { resolveGmailAddressFromEnv, resolveGmailPasswordFromEnv, resolveMaxAttachmentBytes, + resolveMessagesBackfillTargetUid, runAllMailPasses, runAttachmentBackfillAndRecoveryPass, + runDeltaPass, selectAllMailFetchRange, selectAttachmentBackfillFetchRange, selectMessagesBackfillFetchRange, @@ -542,7 +549,7 @@ test("processMessage: a served gap whose attachment fails hydration AGAIN is nev "the re-failed attachment must be a retryable gap key" ); assert.deepEqual( - attachmentCoverage.failedRecords.map((r) => r.id), + attachmentCoverage.failedRecords.map((r) => r.record.id), ["gmmsgid-1111:2"], "the re-failed attachment must be retained so a fresh DETAIL_GAP is emitted for it" ); @@ -1491,6 +1498,150 @@ test("selectMessagesBackfillFetchRange: first and later pages are bounded UID ra ); }); +/** + * The reopening-band guards. + * + * `backfill.target_uid` and `all_mail.forward_uidnext` split ONE UID space, so + * they must meet: `target_uid + 1 >= forward_uidnext`. The forward watermark + * climbs on every run that sees new mail; when the ceiling merely copied itself + * forward the interval between them reopened continuously and grew without + * bound. Live evidence: a 297-UID band swallowed two days of mail, was repaired + * to 0, and measured 2 then 3 within minutes as new mail arrived. + * + * Each behavior is pinned separately below so a mutation to one guard reddens + * on its own rather than being masked by a sibling. + */ +test("resolveMessagesBackfillTargetUid: the ceiling rises to meet the forward watermark", () => { + // The mechanism defect: a frozen ceiling under a climbing watermark. + assert.equal( + resolveMessagesBackfillTargetUid({ + forwardFloorUid: 324_022, + prior: { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 }, + }), + 324_022, + "a watermark that moved past the ceiling must pull the ceiling up, or the gap reopens every run" + ); +}); + +test("resolveMessagesBackfillTargetUid: the ceiling never falls, so backfill progress is never rewound", () => { + // A ceiling that could fall would strand `backfilled_through_uid` above its + // own target and re-open a finished walk. On the live instance the walk is + // ~150k UIDs deep; rewinding would re-fetch every one of them. + assert.equal( + resolveMessagesBackfillTargetUid({ + forwardFloorUid: 900, + prior: { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 }, + }), + 324_020, + "a lower forward floor must never lower the ceiling" + ); + assert.equal( + resolveMessagesBackfillTargetUid({ + forwardFloorUid: 0, + prior: { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 }, + }), + 324_020, + "a zero/absent forward floor must not collapse the ceiling" + ); +}); + +test("resolveMessagesBackfillTargetUid: a first run with no stored ceiling adopts the forward floor", () => { + // On a full resync the forward pass fetches NOTHING, yet the watermark is + // written from the live uidnext. Everything below it is therefore historical + // work and the ceiling must say so. + assert.equal( + resolveMessagesBackfillTargetUid({ forwardFloorUid: 1200, prior: {} }), + 1200, + "an unstarted walk takes the forward floor as its ceiling" + ); +}); + +test("resolveMessagesBackfillTargetUid: a quiet mailbox leaves the ceiling exactly where it was", () => { + // Termination guard: when no mail arrived, the ceiling is unchanged, so the + // walk converges instead of chasing an ever-rising target forever. + const prior = { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 }; + assert.equal(resolveMessagesBackfillTargetUid({ forwardFloorUid: 324_020, prior }), 324_020); + // Idempotent: re-resolving against its own output is a fixed point. + assert.equal( + resolveMessagesBackfillTargetUid({ + forwardFloorUid: 324_020, + prior: { ...prior, target_uid: 324_020 }, + }), + 324_020, + "re-resolving must be a fixed point, not a ratchet that keeps finding new work" + ); +}); + +test("advanceMessagesBackfillCursor: an explicit raised ceiling reopens a completed walk without rewinding it", () => { + // A walk that finished at 1200 must reopen when the forward watermark has + // moved to 1301 — but `backfilled_through_uid` must hold at 1200, not rewind. + const completed = { + backfilled_through_uid: 1200, + completed_at: FROZEN_NOW, + target_uid: 1200, + uidvalidity: 123, + }; + const reopened = advanceMessagesBackfillCursor({ + now: FROZEN_NOW, + pageEndUid: 1200, + prior: { ...completed, target_uid: 1300 }, + }); + assert.equal(reopened.target_uid, 1300, "the raised ceiling must be persisted"); + assert.equal(reopened.backfilled_through_uid, 1200, "progress must never rewind when the ceiling rises"); + assert.equal(reopened.completed_at, null, "a walk with UIDs left to reach is not complete"); +}); + +test("advanceMessagesBackfillCursor: a settled walk under an unchanged ceiling stays settled", () => { + // The other half of the reopen rule: without new mail the ceiling does not + // move, so a finished walk must stay finished rather than re-scanning. + const completed = { + backfilled_through_uid: 1200, + completed_at: FROZEN_NOW, + target_uid: 1200, + uidvalidity: 123, + }; + const settled = advanceMessagesBackfillCursor({ + now: FROZEN_NOW, + pageEndUid: 1200, + prior: completed, + }); + assert.equal(settled.backfilled_through_uid, 1200); + assert.equal(typeof settled.completed_at, "string", "a walk that reached its ceiling stays complete"); +}); + +test("advanceMessagesBackfillCursor: the commit honours the raised ceiling carried on the cursor", () => { + // The ceiling has exactly one source of truth: `prior.target_uid`, already + // raised by `resolveMessagesBackfillTargetUid`. A page that walked into the + // reopened band cannot be committed against a STALE ceiling — that is the + // half-fix where the repair looks applied but the next run re-reads the old + // value and the band reopens. + const stale = { backfilled_through_uid: 1200, completed_at: FROZEN_NOW, target_uid: 1200, uidvalidity: 123 }; + assert.throws( + () => advanceMessagesBackfillCursor({ now: FROZEN_NOW, pageEndUid: 1300, prior: stale }), + /must not pass its target/, + "a page that walked the reopened band must not be committable against the stale ceiling" + ); + const committed = advanceMessagesBackfillCursor({ + now: FROZEN_NOW, + pageEndUid: 1300, + prior: { ...stale, target_uid: 1300 }, + }); + assert.equal(committed.target_uid, 1300, "the cursor must store the raised ceiling for the next run to read"); +}); + +test("advanceMessagesBackfillCursor: a partial page under a reopened ceiling stays incomplete", () => { + // A reopened band larger than one page must leave the walk open, so the next + // run continues into it rather than declaring the mailbox finished. + const partial = advanceMessagesBackfillCursor({ + now: FROZEN_NOW, + pageEndUid: 1700, + prior: { backfilled_through_uid: 1200, completed_at: FROZEN_NOW, target_uid: 2000, uidvalidity: 123 }, + }); + assert.equal(partial.target_uid, 2000); + assert.equal(partial.backfilled_through_uid, 1700); + assert.equal(partial.completed_at, null, "the walk has 1701..2000 left and must not report completion"); +}); + test("advanceMessagesBackfillCursor: page completion is monotonic and partial pages stay incomplete", () => { const partial = advanceMessagesBackfillCursor({ now: FROZEN_NOW, @@ -1636,19 +1787,12 @@ test("runAllMailPasses: first historical page is bounded, durable only at page e true, "a bounded page proves its own detail coverage even while historical continuation remains" ); - assert.deepEqual( + assert.equal( protocolMessages.find((message) => message.type === "DETAIL_COVERAGE" && message.stream === "message_bodies"), - { - type: "DETAIL_COVERAGE", - reference_only: true, - stream: "message_bodies", - state_stream: "messages", - required_keys: [], - hydrated_keys: [], - considered: 2, - covered: 2, - }, - "the body stream reports the same bounded parent-message pass" + undefined, + "message_bodies is declared `state_stream: messages` in the manifest, so it must emit NO DETAIL_COVERAGE — " + + "the runtime rejects the whole run if it does, and the only counts available here are the parent " + + "message pass's, which would fabricate covered == considered for bodies never hydrated" ); assert.deepEqual( protocolMessages.find((message) => message.type === "SKIP_RESULT" && message.stream === "messages"), @@ -1675,6 +1819,105 @@ test("runAllMailPasses: first historical page is bounded, durable only at page e } }); +/** + * The manifest — not this test — is the authority on which streams may prove + * their own coverage. A stream declared with a `state_stream` parent is a + * static single-parent detail stream: its checkpoint status is projected from + * that parent's commit outcome, so the runtime rejects the ENTIRE run if such + * a stream emits DETAIL_COVERAGE (see `validateDetailCoverageAgainstManifest`). + * + * This reads the real manifest rather than hard-coding `message_bodies`, so a + * future stream that gains a `state_stream` parent is covered the day the + * manifest says so. + * + * Regression: a `message_bodies` DETAIL_COVERAGE reporting the PARENT message + * pass's considered/covered shipped to production and failed every Gmail run + * with `runtime_error`, driving the scheduler into cooling_off. It was also + * dishonest on its own terms — it claimed covered == considered for bodies + * that were never hydrated. + */ +test("runAllMailPasses: no stream the manifest declares with a state_stream parent may emit DETAIL_COVERAGE", async () => { + const manifest = JSON.parse(await readFile(new URL("../../manifests/gmail.json", import.meta.url), "utf8")) as { + streams?: Array<{ name: string; state_stream?: string }>; + }; + const stateStreamParented = new Set( + (manifest.streams || []) + .filter( + (stream) => + typeof stream.state_stream === "string" && stream.state_stream && stream.state_stream !== stream.name + ) + .map((stream) => stream.name) + ); + assert.ok( + stateStreamParented.has("message_bodies"), + "guard precondition: the gmail manifest must still declare message_bodies with a state_stream parent" + ); + + const originalWrite = globalThis.process.stdout.write; + const protocolMessages: Record[] = []; + globalThis.process.stdout.write = ((data: string): boolean => { + if (typeof data === "string") { + try { + protocolMessages.push(JSON.parse(data) as Record); + } catch { + // Ignore non-protocol output. + } + } + return true; + }) as typeof process.stdout.write; + + try { + const client: Pick = { + close: mock.fn(), + download: () => { + throw new Error("download must not be called without attachments"); + }, + fetchOne: () => { + throw new Error("fetchOne must not be called without bodies"); + }, + search: mock.fn(async () => []), + mailbox: { + delimiter: "/", + exists: 1200, + flags: new Set(), + path: "[Gmail]/All Mail", + uidNext: 1201, + uidValidity: 123n, + }, + // biome-ignore lint/suspicious/useAwait: async generator is required by the ImapFlow fetch shape. + async *fetch() { + for (const uid of [1, 2]) { + yield makeMsg({ uid, emailId: `msg-${uid}` }); + } + }, + }; + + await runAllMailPasses( + client, + makeAllMailMailbox(), + {}, + { + emitRecord: async () => true, + emittedAt: FROZEN_NOW, + requested: makeRequested(["messages", "message_bodies"]), + } + ); + + const illegal = protocolMessages + .filter((message) => message.type === "DETAIL_COVERAGE") + .map((message) => message.stream as string) + .filter((stream) => stateStreamParented.has(stream)); + assert.deepEqual( + illegal, + [], + "these streams emitted DETAIL_COVERAGE despite a manifest-declared state_stream parent, which fails the " + + `whole run at runtime: ${illegal.join(", ")}` + ); + } finally { + globalThis.process.stdout.write = originalWrite; + } +}); + test("runAllMailPasses: scheduled runs advance historical pages while forwarding new mail", async () => { const originalWrite = globalThis.process.stdout.write; const protocolMessages: Record[] = []; @@ -1748,6 +1991,10 @@ test("runAllMailPasses: scheduled runs advance historical pages while forwarding const first = await run({}); assert.deepEqual(fetchRanges, ["1:500"]); assert.deepEqual(first.all_mail, { + // The mailbox's own EXISTS count rides along so the next run in this + // epoch can prove the inventory did not shrink (see + // all-mail-inventory.test.ts). + exists: 1200, forward_uidnext: 1201, highest_modseq: null, uidnext: 501, @@ -1763,11 +2010,203 @@ test("runAllMailPasses: scheduled runs advance historical pages while forwarding emittedRecords.some((record) => record.stream === "messages" && record.data.id === "msg-1250"), "new mail in the forward range is collected while historical backfill is pending" ); + const messagesCoverage = protocolMessages.find( + (message) => message.type === "DETAIL_COVERAGE" && message.stream === "messages" + ); + assert.deepEqual( + messagesCoverage && { considered: messagesCoverage.considered, covered: messagesCoverage.covered }, + { considered: 2, covered: 2 }, + "the messages DETAIL_COVERAGE must sum BOTH the historical page (msg-1001) and the forward page " + + "(msg-1250) — reporting only the historical pass's considered/covered undercounts the denominator " + + "against the raw collected-record total, the same class of defect message_bodies's coverage " + + "(which does sum both passes) avoids" + ); + + // The continuation must describe the SAME page the DETAIL_COVERAGE fact + // describes. The runtime's isHealthyBoundedContinuation + // (reference-implementation/server/continuation-proof.ts) admits a bounded + // page only when continuation.considered === fact.considered AND + // continuation.covered === fact.covered. When the coverage fact summed both + // passes but the continuation reported historical-only counts, the pair + // desynced by exactly the forward-pass count on every run that carried new + // mail alongside a pending backfill (observed live: fact 52/52 vs + // continuation 51/51), the identity check failed, and the stream fell + // through to retryable_gap instead of deriving complete. + const messagesSkip = protocolMessages.find( + (message) => message.type === "SKIP_RESULT" && message.stream === "messages" + ); + const skipContinuation = messagesSkip?.continuation as Record | undefined; + assert.equal( + messagesSkip?.reason, + "historical_backfill_pending", + "a page with historical work remaining still emits its bounded continuation" + ); + assert.deepEqual( + skipContinuation && { considered: skipContinuation.considered, covered: skipContinuation.covered }, + { considered: messagesCoverage?.considered, covered: messagesCoverage?.covered }, + "the historical continuation skip must carry the SAME considered/covered as the messages " + + "DETAIL_COVERAGE fact — the runtime's isHealthyBoundedContinuation requires that identity, so any " + + "drift between the two emissions silently degrades a complete stream to a retryable_gap" + ); + + // End-to-end: the runtime predicate itself accepts the synced pair, and + // would reject the historical-only counts the desynced code emitted. + const isHealthyBoundedContinuation = ( + fact: { considered: number; covered: number }, + cont: { considered: number; covered: number } + ) => cont.considered === fact.considered && cont.covered === fact.covered && fact.considered === fact.covered; + assert.equal( + isHealthyBoundedContinuation( + { considered: messagesCoverage?.considered as number, covered: messagesCoverage?.covered as number }, + { considered: skipContinuation?.considered as number, covered: skipContinuation?.covered as number } + ), + true, + "the emitted fact/continuation pair satisfies the runtime's bounded-continuation identity check" + ); + assert.equal( + isHealthyBoundedContinuation({ considered: 2, covered: 2 }, { considered: 1, covered: 1 }), + false, + "control: the historical-only counts the regression emitted do NOT satisfy that check" + ); + + // Run 2 raised the forward watermark to 1301, so UIDs 1201..1300 are now + // below where the forward walk resumes. The historical ceiling must have + // risen with it (1200 -> 1300) or that interval belongs to neither walk. + // Before the ceiling tracked the watermark this read "1001:1200", leaving + // 1201..1300 orphaned — the live 297-UID band in miniature. + assert.equal( + (second.backfill as Record).target_uid, + 1300, + "the historical ceiling must rise to meet the forward watermark, not stay frozen at 1200" + ); const third = await run({ messages: second }); - assert.deepEqual(fetchRanges, ["1001:1200", "1301:*"]); + assert.deepEqual(fetchRanges, ["1001:1300", "1301:*"]); assert.equal((third.all_mail as Record).uidnext, 1301); assert.equal(typeof (third.backfill as Record).completed_at, "string"); + assert.equal( + (third.backfill as Record).backfilled_through_uid, + 1300, + "the walk closes the whole space up to the forward resume point" + ); + } finally { + globalThis.process.stdout.write = originalWrite; + } +}); + +/** + * The band must not reopen after the historical walk has FINISHED. + * + * This is the live shape: the backfill reaches its ceiling, `completed_at` is + * stamped, and then mail keeps arriving. A completed walk that copies its stale + * ceiling forward leaves every newly-arrived UID above the ceiling and below the + * forward watermark — belonging to neither walk. That is precisely how the + * repaired live cursor went from band=0 to band=2 to band=3 within minutes. + * + * The end-to-end path is what makes this test necessary: the pure resolver is + * correct in isolation, but the completed-walk branch in `runAllMailPasses` + * chooses whether to consult it at all. + */ +test("runAllMailPasses: a completed historical walk reopens for new mail instead of freezing its ceiling", async () => { + const originalWrite = globalThis.process.stdout.write; + const protocolMessages: Record[] = []; + const fetchRanges: string[] = []; + let uidNext = 1201; + globalThis.process.stdout.write = ((data: string): boolean => { + if (typeof data === "string") { + try { + protocolMessages.push(JSON.parse(data) as Record); + } catch { + // Ignore non-protocol output. + } + } + return true; + }) as typeof process.stdout.write; + + try { + const client: Pick = { + close: mock.fn(), + download: () => { + throw new Error("download must not be called without attachments"); + }, + fetchOne: () => { + throw new Error("fetchOne must not be called without bodies"); + }, + search: mock.fn(async () => []), + mailbox: { + delimiter: "/", + exists: 1200, + flags: new Set(), + path: "[Gmail]/All Mail", + get uidNext() { + return uidNext; + }, + uidValidity: 123n, + }, + // biome-ignore lint/suspicious/useAwait: async generator is required by the ImapFlow fetch shape. + async *fetch(range: string) { + fetchRanges.push(range); + yield makeMsg({ uid: 1, emailId: "seed" }); + }, + }; + + const run = async (state: Record) => { + protocolMessages.length = 0; + fetchRanges.length = 0; + await runAllMailPasses(client, makeAllMailMailbox(), state, { + emitRecord: () => Promise.resolve(true), + emittedAt: FROZEN_NOW, + requested: makeRequested(["messages"]), + }); + const stateMessage = protocolMessages.find( + (message) => message.type === "STATE" && message.stream === "messages" + ); + assert.ok(stateMessage, "each run commits a messages state"); + return stateMessage.cursor as Record; + }; + + // Drive the walk all the way to completion against a still mailbox. + let cursor = await run({}); + for (let i = 0; i < 4; i += 1) { + cursor = await run({ messages: cursor }); + } + const settled = cursor.backfill as Record; + assert.equal(typeof settled.completed_at, "string", "precondition: the historical walk has finished"); + assert.equal(settled.target_uid, 1200, "precondition: the ceiling settled at the mailbox it walked"); + assert.equal(settled.backfilled_through_uid, 1200); + + // Now mail arrives. The forward watermark will move to 1301. + uidNext = 1301; + const afterNewMail = await run({ messages: cursor }); + const reopened = afterNewMail.backfill as Record; + const allMail = afterNewMail.all_mail as Record; + + assert.equal(allMail.forward_uidnext, 1301, "precondition: the forward watermark climbs with the mailbox"); + // THE INVARIANT: ceiling + 1 >= resume. With a frozen ceiling of 1200 this + // is 1201 >= 1301 — false — and UIDs 1201..1300 belong to neither walk. + assert.ok( + (reopened.target_uid as number) + 1 >= (allMail.forward_uidnext as number), + `the two walks must meet: ceiling ${String(reopened.target_uid)} + 1 must reach ` + + `forward resume ${String(allMail.forward_uidnext)}, otherwise the band between them is orphaned` + ); + assert.equal(reopened.target_uid, 1300, "the ceiling reopens to cover the newly-arrived UIDs"); + // The reopened ceiling puts 1201..1300 back in the historical walk's remit, + // and because that band is smaller than one page the walk consumes it in + // this same run rather than deferring it. What must never happen is a + // DECREASE: that would discard walked work and re-fetch it. + assert.ok( + (reopened.backfilled_through_uid as number) >= (settled.backfilled_through_uid as number), + `reopening must never rewind progress: ${String(reopened.backfilled_through_uid)} < ` + + `${String(settled.backfilled_through_uid)} would re-fetch already-walked UIDs` + ); + assert.equal( + reopened.backfilled_through_uid, + 1300, + "the reopened band is smaller than a page, so this run closes it outright" + ); + // Having closed the whole reopened band, the walk is complete again — and + // now genuinely contiguous with the forward watermark. + assert.equal(typeof reopened.completed_at, "string", "a walk that reached its reopened ceiling is complete"); } finally { globalThis.process.stdout.write = originalWrite; } @@ -2310,6 +2749,8 @@ test("runAttachmentBackfillAndRecoveryPass: served gaps preempt historical attac recoveredAttachmentGapIds: new Set(), session: { attachmentBackfill: { backfilled_through_uid: 250, uidvalidity: 123 }, + existsTotal: 600, + priorExistsTotal: undefined, fullResync: false, highestModseqCursor: null, messagesBackfill: { uidvalidity: 123, backfilled_through_uid: 0, completed_at: null }, @@ -2502,6 +2943,8 @@ test("runAttachmentBackfillAndRecoveryPass: recoveryOnly=true recovers served ga recoveryOnly: true, session: { attachmentBackfill: { backfilled_through_uid: 250, uidvalidity: 123 }, + existsTotal: 600, + priorExistsTotal: undefined, fullResync: false, highestModseqCursor: null, messagesBackfill: { uidvalidity: 123, backfilled_through_uid: 0, completed_at: null }, @@ -3098,10 +3541,19 @@ test("recoverServedAttachmentGaps: an unclassified plain blob failure remains re assert.ok(failedAttachment, "the failed attachment record must still be emitted"); assert.equal(failedAttachment.hydration_status, "failed"); assert.deepEqual(attachmentCoverage.gapKeys, [failedAttachment.id]); - assert.deepEqual(attachmentCoverage.failedRecords, [failedAttachment]); + assert.deepEqual(attachmentCoverage.failedRecords, [ + { failureClass: "unclassified_failed", record: failedAttachment }, + ]); const [failedCoverageRecord] = attachmentCoverage.failedRecords; assert.ok(failedCoverageRecord, "failed recovery records must be retained for detail-gap emission"); - assert.deepEqual(buildAttachmentDetailGap(failedCoverageRecord), { + // A served-recovery attempt that fails again on a retry (attempt N of an + // already-terminal-bound gap) must still record WHY — a bounded, non-secret + // failure class — not just increment attempt_count with no evidence. This + // is the exact class of defect behind the 2026-08 Gmail 5-row incident: 5 + // `temporary_unavailable` attachment gaps reached terminal status after + // 37-117 retries with `last_error_json` permanently null, because this + // recovery-retry path recorded an attempt but never a cause. + assert.deepEqual(buildAttachmentDetailGap(failedCoverageRecord.record, failedCoverageRecord.failureClass), { type: "DETAIL_GAP", stream: "attachments", parent_stream: "messages", @@ -3116,6 +3568,8 @@ test("recoverServedAttachmentGaps: an unclassified plain blob failure remains re }, retryable: true, reference_only: true, + detail: { class: "unclassified_failed" }, + last_error: { class: "unclassified_failed" }, }); assert.equal(JSON.stringify(summary).includes("private unclassified blob failure"), false); }); @@ -3923,7 +4377,7 @@ test("redactEmailForProgress: single-character local-part is fully masked", () = test("redactEmailForProgress: output never contains the full address or local-part", () => { for (const address of [ - "the owner.nunamaker@gmail.com", + "the.owner.sample@example.com", "first.last+tag@corp.example.co.uk", 'weird"@"local@host.example', // quoted local-part embedding an @ ]) { @@ -4010,9 +4464,15 @@ test("recordAttachmentCoverage: routes each hydration status into the honest buc assert.deepEqual(coverage.gapKeys, ["b:1"]); // too_large and deferred stay required, unaccounted (not in hydrated/gap). assert.deepEqual( - coverage.failedRecords.map((r) => r.id), + coverage.failedRecords.map((r) => r.record.id), ["b:1"] ); + // No `failure` argument was passed, so the failure class falls back to the + // same `unclassified_failed` bucket used by the aggregate telemetry. + assert.deepEqual( + coverage.failedRecords.map((r) => r.failureClass), + ["unclassified_failed"] + ); }); test("buildAttachmentDetailCoverageMessage: emits complete zero-attachment coverage", () => { @@ -4190,11 +4650,11 @@ test("emitMessagesPass: accumulates honest coverage across hydrated, gap, and un // an otherwise-successful run aborts at commit and re-fetches the same window // forever. The failed record is retained on the accumulator; one gap per key. assert.deepEqual( - coverage.failedRecords.map((r) => r.id), + coverage.failedRecords.map((r) => r.record.id), coverage.gapKeys, "exactly one retained failed record per gap_keys entry" ); - const gaps = coverage.failedRecords.map((r) => buildAttachmentDetailGap(r)); + const gaps = coverage.failedRecords.map((r) => buildAttachmentDetailGap(r.record, r.failureClass)); // The gate matches DETAIL_GAP.record_key against the DETAIL_COVERAGE key. assert.deepEqual( gaps.map((g) => g.record_key), @@ -4202,7 +4662,9 @@ test("emitMessagesPass: accumulates honest coverage across hydrated, gap, and un ); // Exact wire shape of the gap for `bad:1`: bounded, non-secret locator // (message + part identifiers only), temporary_unavailable (retryable), - // pending, reference_only, and no error block (no raw error text crosses). + // pending, reference_only, and a bounded non-secret failure class (no raw + // hydration_error text — which could echo upstream URLs/tokens — ever + // crosses; only the category string does). assert.deepEqual(gaps[0], { type: "DETAIL_GAP", stream: "attachments", @@ -4218,11 +4680,13 @@ test("emitMessagesPass: accumulates honest coverage across hydrated, gap, and un }, retryable: true, reference_only: true, + detail: { class: "blob_upload_transport_failed" }, + last_error: { class: "blob_upload_transport_failed" }, }); - // Defense-in-depth: the gap carries no error/last_error block, so no raw - // hydration_error string (which could echo upstream URLs/text) ever crosses. - assert.equal(gaps[0]?.detail, undefined); - assert.equal(gaps[0]?.last_error, undefined); + // Defense-in-depth: neither block carries raw hydration_error text (which + // could echo upstream URLs/tokens) — only the bounded category string. + assert.deepEqual(gaps[0]?.detail, { class: "blob_upload_transport_failed" }); + assert.deepEqual(gaps[0]?.last_error, { class: "blob_upload_transport_failed" }); }); // ─── Bounded scope: collection_scope.since mapping to IMAP SINCE ────────── @@ -4638,3 +5102,333 @@ test("runAllMailPasses: missing internalDate under declared since propagates unc globalThis.process.stdout.write = originalWrite; } }); + +// ─── Invariant: DONE is terminal on the wire ──────────────────────────────── + +test("emit: suppresses any message written after DONE", async () => { + // REGRESSION EVIDENCE. Production gmail runs failed + // `connector_protocol_violation "Connector emitted RECORD after DONE"` + // (runs run_1787331290490_1, run_1787332551497_1, run_1787333444049 on + // 2026-08-21), reporting 82 records emitted against 18 flushed — the + // surplus was written after the runtime had already latched DONE. + // + // The cause is that no terminal path stops the process: `flushAndExit` + // registers listeners and RETURNS (connector-exit.ts waits for the runtime + // to close stdin), so interrupted in-flight work keeps emitting. The + // channel itself therefore has to refuse the write. + // + // EVIDENCE DISCRIMINATOR: with the `doneEmitted` latch removed from + // `emit`, the RECORD below is written to stdout and this test fails on + // `afterDone.length`. + __resetDoneLatchForTests(); + const lines: string[] = []; + const originalWrite = globalThis.process.stdout.write; + globalThis.process.stdout.write = ((chunk: string | Uint8Array): boolean => { + lines.push(String(chunk)); + return true; + }) as typeof globalThis.process.stdout.write; + try { + await gmailEmitForTests({ type: "PROGRESS", message: "before" }); + await gmailEmitForTests({ type: "DONE", status: "succeeded", records_emitted: 1 }); + await gmailEmitForTests({ + type: "RECORD", + stream: "messages", + key: "k", + data: {}, + emitted_at: "2026-08-21T00:00:00.000Z", + }); + await gmailEmitForTests({ type: "DONE", status: "failed", records_emitted: 0 }); + } finally { + globalThis.process.stdout.write = originalWrite; + __resetDoneLatchForTests(); + } + + const parsed = lines.map((l) => JSON.parse(l) as { status?: string; type: string }); + const doneIndex = parsed.findIndex((m) => m.type === "DONE"); + assert.ok(doneIndex >= 0, "the DONE itself is written"); + const afterDone = parsed.slice(doneIndex + 1); + assert.deepEqual(afterDone, [], "nothing may be written to stdout after DONE"); + assert.equal(parsed.filter((m) => m.type === "DONE").length, 1, "a second DONE is suppressed too"); + assert.equal(parsed.filter((m) => m.type === "RECORD").length, 0, "the post-DONE RECORD never reaches the runtime"); +}); + +// ─── Invariant: the delta pass never blanks an already-collected envelope ─── + +test("runDeltaPass: issues no IMAP command while the delta FETCH iterator is open", async () => { + // REGRESSION EVIDENCE. imapflow multiplexes one command at a time over a + // single connection, so a nested command issued mid-iteration hangs the + // outer iterator — the rule `runAllMailPasses` states at its own Phase A/B + // boundary. The delta pass called `fetchBodies` (a `fetchOne`) inside its + // `for await`, which wedged the run after "Fetching flag/label deltas" + // until a timeout, then tripped the post-DONE guard as the abandoned + // iterator drained. + // + // EVIDENCE DISCRIMINATOR: with the snippet fetch moved back inside the + // `for await`, `openWhileFetching` records a nested call and this fails. + const emitted: Array<{ stream: string }> = []; + let iteratorOpen = false; + const openWhileFetching: string[] = []; + + // biome-ignore lint/suspicious/useAwait: stands in for ImapFlow.fetch's async-iterable-returning signature. + const fetch = mock.fn(async function* () { + iteratorOpen = true; + try { + yield makeMsg({ uid: 200, emailId: "gmmsgid-a" }); + yield makeMsg({ uid: 201, emailId: "gmmsgid-b" }); + } finally { + iteratorOpen = false; + } + }); + + const fetchBodies = mock.fn(() => { + if (iteratorOpen) { + openWhileFetching.push("nested"); + } + return Promise.resolve({ bodyHtmlFull: null, bodyTextFull: null, snippet: "s" }); + }); + + await runDeltaPass( + { fetch } as unknown as Pick, + { fullResync: false, priorModseq: 1n } as unknown as Parameters[1], + makeRequested(["messages"]), + ((stream: string): Promise => { + emitted.push({ stream }); + return Promise.resolve(); + }) as unknown as Parameters[3], + "2026-08-20T00:00:00.000Z", + fetchBodies as unknown as Parameters[5] + ); + + assert.deepEqual(openWhileFetching, [], "no body fetch may run while the delta FETCH iterator is open"); + assert.equal(emitted.length, 2, "both delta messages still emit after the iterator drains"); +}); + +test("runDeltaPass: emits a WHOLE messages record, never a null-envelope shell", async () => { + // REGRESSION EVIDENCE. `records` upserts replace `record_json` wholesale, so + // a delta record carrying null envelope fields overwrites — and destroys — + // the stored subject/sender/date/size/snippet of a message that had them. + // Observed live: 2,534 distinct Gmail messages had been hollowed at least + // once, 981 were hollow at rest, each re-hollowed on every label change. + // + // EVIDENCE DISCRIMINATOR: against the pre-fix code (`envelope: false` in the + // delta query + `buildDeltaMessageRecord`) every assertion below on subject, + // from_email, date, size_bytes, snippet and received_at fails — that version + // emitted exactly the null shell this test forbids. + const emitted: Array<{ data: Record; stream: string }> = []; + const emitRecord = (stream: string, data: Record): Promise => { + emitted.push({ data, stream }); + return Promise.resolve(); + }; + + const delta = makeMsg({ uid: 100, emailId: "gmmsgid-delta", flags: new Set(["\\Seen", "\\Flagged"]) }); + // biome-ignore lint/suspicious/useAwait: stands in for ImapFlow.fetch's async-iterable-returning signature. + const fetch = mock.fn(async function* () { + yield delta; + }); + + const fetchBodies = mock.fn(() => + Promise.resolve({ bodyHtmlFull: null, bodyTextFull: null, snippet: "a real snippet" }) + ); + + await runDeltaPass( + { fetch } as unknown as Pick, + { fullResync: false, priorModseq: 1n } as unknown as Parameters[1], + makeRequested(["messages"]), + emitRecord, + "2026-08-20T00:00:00.000Z", + fetchBodies as unknown as Parameters[5] + ); + + const rec = emitted.find((r) => r.stream === "messages"); + assert.ok(rec, "delta pass emits a messages record"); + assert.equal(rec.data.subject, "Test subject", "subject survives a flag delta"); + assert.equal(rec.data.from_email, "alice@example.com", "sender survives a flag delta"); + assert.equal(rec.data.date, "2026-04-20T10:00:00.000Z", "Date header survives a flag delta"); + assert.equal(rec.data.size_bytes, 1024, "size survives a flag delta"); + assert.equal(rec.data.snippet, "a real snippet", "snippet is re-derived, not blanked"); + assert.equal( + rec.data.received_at, + "2026-04-20T10:00:05.000Z", + "received_at keeps the message's own internalDate, not the run clock" + ); + // The flag change itself must still land — that is the point of the pass. + assert.equal(rec.data.is_flagged, true, "the flag delta is applied"); +}); + +test("runDeltaPass: skips a message the server returns without an envelope", async () => { + // Skipping preserves the stored row. Emitting a partial record would blank + // it, which is the very defect above — so absent an envelope there is no + // safe record to write, and losing one flag update is the cheaper loss. + const emitted: Array<{ stream: string }> = []; + const emitRecord = (stream: string): Promise => { + emitted.push({ stream }); + return Promise.resolve(); + }; + const { envelope: _envelope, ...noEnvelope } = makeMsg({ uid: 101, emailId: "gmmsgid-no-env" }); + // biome-ignore lint/suspicious/useAwait: stands in for ImapFlow.fetch's async-iterable-returning signature. + const fetch = mock.fn(async function* () { + yield noEnvelope; + }); + const fetchBodies = mock.fn(() => Promise.resolve({ bodyHtmlFull: null, bodyTextFull: null, snippet: null })); + + await runDeltaPass( + { fetch } as unknown as Pick, + { fullResync: false, priorModseq: 1n } as unknown as Parameters[1], + makeRequested(["messages"]), + emitRecord as unknown as Parameters[3], + "2026-08-20T00:00:00.000Z", + fetchBodies as unknown as Parameters[5] + ); + + assert.equal(emitted.length, 0, "no record emitted when the envelope is absent"); +}); + +// ─── Per-part size honesty (RFC822.SIZE is the MESSAGE, not the part) ──────── + +test("fetchAttachmentPart: reports the PART's BODYSTRUCTURE size, never the message-wide RFC822.SIZE", async () => { + // imapflow sets `meta.expectedSize` from the FETCH RFC822.SIZE item, which + // is the size of the WHOLE MESSAGE — identical for every part. Trusting it + // as a per-attachment size made a message reject all of its attachments + // whenever their SUM crossed the cap. + const MESSAGE_WIDE_SIZE = 35_962_168; + const PART_SIZE = 4_154_730; + const download = mock.fn(() => + Promise.resolve({ + content: Readable.from([Buffer.alloc(8, 0x41)]), + meta: { contentType: "application/pdf", expectedSize: MESSAGE_WIDE_SIZE }, + }) + ); + + const result = await fetchAttachmentPart({ download } as unknown as Pick, makeMsg({ uid: 7 }), { + content_type: "application/pdf", + id: "m:2", + part_index: "2", + size_bytes: PART_SIZE, + } as AttachmentRecord); + + assert.equal( + result.expectedSize, + PART_SIZE, + "expectedSize must be the part's own BODYSTRUCTURE size, not the message's RFC822.SIZE" + ); + assert.notEqual( + result.expectedSize, + MESSAGE_WIDE_SIZE, + "the message-wide size must never be surfaced as a part size" + ); +}); + +test("fetchAttachmentPart: reports unknown (null) rather than substituting the message-wide size", async () => { + // When BODYSTRUCTURE gives no per-part size there is no honest pre-flight + // number. `enforceMaxBytes` still counts real bytes mid-stream, so the + // right answer is "unknown", not a message-scoped stand-in. + const download = mock.fn(() => + Promise.resolve({ + content: Readable.from([Buffer.alloc(8, 0x41)]), + meta: { contentType: "application/pdf", expectedSize: 30_000_000 }, + }) + ); + + const result = await fetchAttachmentPart({ download } as unknown as Pick, makeMsg({ uid: 7 }), { + content_type: "application/pdf", + id: "m:2", + part_index: "2", + size_bytes: null, + } as AttachmentRecord); + + assert.equal(result.expectedSize, null, "an unknown part size stays unknown"); +}); + +test("makeAttachmentHydrator: many small attachments summing over the cap all hydrate", async () => { + // The live defect: one message held 8 attachments of ~4.5 MB each. Every one + // was marked too_large against the 25 MiB per-attachment cap using the + // message's 35,962,168-byte total. None was individually close to the cap. + const MESSAGE_WIDE_SIZE = 35_962_168; + const PART_SIZE = 4_154_730; + const payload = Buffer.alloc(64, 0x41); + const uploadBlob = mock.fn(({ content }: { content: AsyncIterable }) => + (async () => { + let bytes = 0; + for await (const chunk of content) { + bytes += Buffer.isBuffer(chunk) ? chunk.byteLength : Buffer.from(chunk).byteLength; + } + return { blob_id: "blob_ok", mime_type: "application/pdf", sha256: "0".repeat(64), size_bytes: bytes }; + })() + ); + const hydrate = makeAttachmentHydrator({ + connectorId: "https://registry.pdpp.dev/connectors/gmail", + // Mirrors imapflow: meta.expectedSize is the message-wide RFC822.SIZE. + fetchAttachment: (_msg, attachment) => + Promise.resolve({ + content: Readable.from([payload]), + expectedSize: attachment.size_bytes, + mimeType: "application/pdf", + }), + maxBytes: DEFAULT_MAX_ATTACHMENT_BYTES, + uploadBlob, + }); + + const result = await hydrate(makeMsg({ uid: 9 }), { + content_type: "application/pdf", + id: "m:3", + part_index: "3", + size_bytes: PART_SIZE, + } as AttachmentRecord); + + assert.equal(result.record.hydration_status, "hydrated", "a 4 MB part under a 25 MiB cap must hydrate"); + // Guards the fixture's premise: the message total really is over the cap + // while the single part really is under it, so this test would fail if the + // message-wide size were ever reinstated as the per-part size. + assert.ok(MESSAGE_WIDE_SIZE > DEFAULT_MAX_ATTACHMENT_BYTES, "the message total is over the cap"); + assert.ok(PART_SIZE < DEFAULT_MAX_ATTACHMENT_BYTES, "the individual part is under the cap"); +}); + +test("makeAttachmentHydrator: a genuinely oversized part is still refused", async () => { + // The cap must keep working. This is the one real case on the live mailbox: + // a single 32,122,600-byte attachment over the 25 MiB cap. + const uploadBlob = mock.fn(() => Promise.reject(new Error("must not upload"))); + const hydrate = makeAttachmentHydrator({ + connectorId: "https://registry.pdpp.dev/connectors/gmail", + fetchAttachment: () => Promise.reject(new Error("must not download")), + maxBytes: DEFAULT_MAX_ATTACHMENT_BYTES, + uploadBlob, + }); + + const result = await hydrate(makeMsg({ uid: 9 }), { + content_type: "application/octet-stream", + id: "m:2", + part_index: "2", + size_bytes: 32_122_600, + } as AttachmentRecord); + + assert.equal(result.record.hydration_status, "too_large"); + assert.match(String(result.record.hydration_error), /exceeds max size: 32122600 > 26214400 bytes/); + assert.equal(uploadBlob.mock.callCount(), 0, "an over-cap part is refused before any transfer"); +}); + +// ─── Coverage honesty: withhold the claim without a boundary ──────────────── + +test("attachmentsCoverageBoundaryEstablished: only a completed historical messages walk is a boundary", () => { + assert.equal( + attachmentsCoverageBoundaryEstablished({ messagesBackfill: { completed_at: null } as never }), + false, + "an in-flight historical walk proves nothing about the mailbox" + ); + assert.equal( + attachmentsCoverageBoundaryEstablished({ messagesBackfill: {} as never }), + false, + "an absent completion is not a boundary" + ); + assert.equal( + attachmentsCoverageBoundaryEstablished({ messagesBackfill: { completed_at: "" } as never }), + false, + "an empty completion stamp is not a boundary" + ); + assert.equal( + attachmentsCoverageBoundaryEstablished({ + messagesBackfill: { completed_at: "2026-08-21T12:40:35.475Z" } as never, + }), + true, + "a completed historical walk has enumerated every message's attachment parts" + ); +}); diff --git a/packages/polyfill-connectors/connectors/gmail/parsers.test.ts b/packages/polyfill-connectors/connectors/gmail/parsers.test.ts index c10ae1f33..6f9b62649 100644 --- a/packages/polyfill-connectors/connectors/gmail/parsers.test.ts +++ b/packages/polyfill-connectors/connectors/gmail/parsers.test.ts @@ -12,7 +12,6 @@ import { addressListToArray, bigintToCursor, bigintToNumber, - buildDeltaMessageRecord, buildMessageBodyRecord, buildMessageRecord, buildThreadRecord, @@ -882,28 +881,6 @@ test("buildMessageRecord: empty envelope + zero attachments maps to null/default assert.equal(rec.snippet, null); }); -// ─── buildDeltaMessageRecord ──────────────────────────────────────────── - -test("buildDeltaMessageRecord: minimal shape with flags + labels, received_at fallback", () => { - const rec = buildDeltaMessageRecord({ - flagsArr: ["\\Seen"], - gmMsgid: "m1", - gmThrid: "t1", - labels: ["INBOX"], - receivedAtFallback: "2024-02-01T00:00:00.000Z", - }); - assert.equal(rec.id, "m1"); - assert.equal(rec.thread_id, "t1"); - assert.equal(rec.subject, null); - assert.equal(rec.from_name, null); - assert.deepEqual(rec.to, []); - assert.equal(rec.received_at, "2024-02-01T00:00:00.000Z"); - assert.deepEqual(rec.labels, ["INBOX"]); - assert.equal(rec.is_seen, true); - assert.equal(rec.is_flagged, false); - assert.equal(rec.has_attachments, false); -}); - // ─── isInTimeRange ────────────────────────────────────────────────────── test("isInTimeRange: no range → always true", () => { diff --git a/packages/polyfill-connectors/connectors/gmail/parsers.ts b/packages/polyfill-connectors/connectors/gmail/parsers.ts index ccbd8d377..4c21442fc 100644 --- a/packages/polyfill-connectors/connectors/gmail/parsers.ts +++ b/packages/polyfill-connectors/connectors/gmail/parsers.ts @@ -796,44 +796,6 @@ export function buildMessageRecord(params: { }; } -/** - * Build the flag/label delta RECORD payload for one message. No envelope - * re-fetch on the delta path — callers pass received_at as a fallback to - * satisfy the schema-required field. - */ -export function buildDeltaMessageRecord(params: { - flagsArr: readonly string[]; - gmMsgid: string; - gmThrid: string; - labels: readonly string[]; - receivedAtFallback: string; -}): Record { - return { - id: params.gmMsgid, - thread_id: params.gmThrid, - subject: null, - from_name: null, - from_email: null, - to: [], - cc: [], - bcc: [], - reply_to: [], - date: null, - received_at: params.receivedAtFallback, - message_id: null, - in_reply_to: null, - references: [], - size_bytes: null, - labels: [...params.labels], - is_draft: params.flagsArr.includes("\\Draft"), - is_flagged: params.flagsArr.includes("\\Flagged"), - is_seen: params.flagsArr.includes("\\Seen"), - is_answered: params.flagsArr.includes("\\Answered"), - has_attachments: false, - snippet: null, - }; -} - /** Determine whether a received_at timestamp falls inside a since/until window. */ export function isInTimeRange( receivedAt: string, diff --git a/packages/polyfill-connectors/connectors/gmail/types.ts b/packages/polyfill-connectors/connectors/gmail/types.ts index 090de1005..d4897a082 100644 --- a/packages/polyfill-connectors/connectors/gmail/types.ts +++ b/packages/polyfill-connectors/connectors/gmail/types.ts @@ -46,7 +46,29 @@ export interface InteractionMessage { type: "INTERACTION"; } +/** + * The mailbox-wide inventory total the IMAP server declared, reported next to + * how far this connector's historical walk has actually reached. Carried on + * PROGRESS rather than folded into `messages` DETAIL_COVERAGE because that + * coverage fact is per-page by contract (the runtime admits a bounded + * continuation only on same-page `considered === covered`), so the mailbox + * total would be the wrong denominator there. + */ +export interface AllMailInventoryProgress { + /** IMAP `EXISTS` for All Mail: the server's own count of messages present. */ + all_mail_exists: number; + /** Highest UID the historical backfill has admitted so far. */ + backfilled_through_uid: number; + /** The UID the forward walk resumes at, i.e. the historical walk's ceiling. */ + forward_floor_uid: number; + /** Whether the historical walk has reached its ceiling. */ + historical_backfill_complete: boolean; + /** The UID epoch these numbers describe. Counts are comparable only within one. */ + uidvalidity: number; +} + export interface ProgressMessage { + all_mail_inventory?: AllMailInventoryProgress; attachment_hydration_failure_outcome?: AttachmentHydrationFailureOutcomeProgress; attachment_recovery_outcome?: AttachmentRecoveryOutcomeProgress; count?: number; @@ -138,6 +160,15 @@ export interface BlobRef { } export interface AllMailCursor { + /** + * The IMAP `EXISTS` count this mailbox reported on the run that wrote this + * cursor — the server's own inventory size for All Mail. Persisted so the + * next run can detect a DECREASE within the same UIDVALIDITY epoch, which is + * deletion or a server bug rather than normal growth. Meaningful only + * alongside the `uidvalidity` in this same cursor: across a re-key the UID + * space was rebuilt and the counts are not comparable. + */ + exists?: number; /** Forward/new-mail watermark. Kept separate from the historical boundary. */ forward_uidnext?: number; highest_modseq?: number | string | null; diff --git a/packages/polyfill-connectors/connectors/google_maps/artifact-reconciliation.test.ts b/packages/polyfill-connectors/connectors/google_maps/artifact-reconciliation.test.ts new file mode 100644 index 000000000..59482360d --- /dev/null +++ b/packages/polyfill-connectors/connectors/google_maps/artifact-reconciliation.test.ts @@ -0,0 +1,205 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Artifact reconciliation for the Google Maps one-time import. + * + * The anchor for this connector is the ARTIFACT, not the account. The + * denominator is every element the parser produced from the uploaded file; + * the numerator is only what this run could key, dedupe and ingest. An element + * the artifact held but the run could not account for (no usable id / + * timestamp) must stay in the denominator, so the stream reads `partial`. + * + * Before this was bound, such an element was dropped by a bare `continue`: + * a 3-element artifact reported `considered: 2, covered: 2` with zero skips — + * a fabricated denominator computed from the same survivors it claimed to + * verify. These tests pin the drop into view. + * + * SCOPE LIMIT, stated plainly: a fully-reconciled artifact proves only that + * this run ingested everything the FILE contained. It says nothing about + * whether Google chose to export everything the provider holds. There is no + * provider-side assertion available for a file drop, and none is invented here. + */ + +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import type { EmittedMessage } from "../../src/connector-runtime.ts"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; + +const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "google_maps", "index.ts"); + +async function runImport( + importRoot: string, + streams: readonly string[] = ["timeline_points"] +): Promise { + const result = await runConnectorProtocolSubprocess({ + cwd: PACKAGE_ROOT, + entrypoint: ENTRYPOINT, + env: { + GOOGLE_MAPS_TIMELINE_DIR: importRoot, + PDPP_OWNER_TOKEN: "", + PDPP_RS_URL: "", + RS_URL: "", + }, + start: { + scope: { streams: streams.map((name) => ({ name })) }, + state: {}, + type: "START", + }, + }); + return result.messages; +} + +/** + * The stream's single coverage verdict. + * + * Asserting there is EXACTLY one is load-bearing, not defensive tidiness. A + * missing early-return lets the drop-aware declaration be followed by a + * `buildFullScanCoverageMessage` fallback, so the stream emits an honest + * `2/1` and then a fabricated `1/1`. Taking the first match would call that + * mutant a pass; a consumer reading the last one would call it `complete`. + * A stream must state its coverage once. + */ +function coverageFor( + messages: readonly EmittedMessage[], + stream: string +): Extract { + const all = messages.filter( + (message): message is Extract => + message.type === "DETAIL_COVERAGE" && message.stream === stream + ); + assert.equal(all.length, 1, `expected exactly one ${stream} DETAIL_COVERAGE, got ${String(all.length)}`); + return all[0] as Extract; +} + +test("a point element with no timestamp stays in the denominator as considered-but-not-covered", async () => { + const importRoot = await mkdtemp(join(tmpdir(), "pdpp-gm-unaccounted-point-")); + try { + // Three location elements. The middle one carries no `timestampMs`, so it + // can never be keyed — but the artifact did contain it. + await writeFile( + join(importRoot, "Records.json"), + JSON.stringify({ + locations: [ + { timestampMs: "1717595122000", latitudeE7: 377_749_000, longitudeE7: -1_224_194_000, accuracy: 12.4 }, + { latitudeE7: 377_800_000, longitudeE7: -1_224_100_000, accuracy: 9.1 }, + { timestampMs: "1717598722000", latitudeE7: 377_800_000, longitudeE7: -1_224_100_000, accuracy: 9.1 }, + ], + }) + ); + const messages = await runImport(importRoot); + + const emitted = messages.filter((m) => m.type === "RECORD" && m.stream === "timeline_points").length; + assert.equal(emitted, 2, "only the two keyable points can be ingested"); + + const coverage = coverageFor(messages, "timeline_points"); + // The load-bearing assertion: the artifact held 3 elements, so 3 is the + // denominator. A regression that drops the element silently reports 2/2. + assert.equal(coverage.considered, 3, "the artifact's third element must remain in the denominator"); + assert.equal(coverage.covered, 2, "only accounted-for elements may be claimed as covered"); + assert.ok(coverage.covered < coverage.considered, "an unaccounted element must read partial, never complete"); + } finally { + await rm(importRoot, { force: true, recursive: true }); + } +}); + +test("an unaccounted point element surfaces an element_unaccounted skip naming the true denominator", async () => { + const importRoot = await mkdtemp(join(tmpdir(), "pdpp-gm-unaccounted-skip-")); + try { + await writeFile( + join(importRoot, "Records.json"), + JSON.stringify({ + locations: [ + { timestampMs: "1717595122000", latitudeE7: 377_749_000, longitudeE7: -1_224_194_000, accuracy: 12.4 }, + { latitudeE7: 377_800_000, longitudeE7: -1_224_100_000, accuracy: 9.1 }, + ], + }) + ); + const messages = await runImport(importRoot); + + const skip = messages.find( + (m) => m.type === "SKIP_RESULT" && m.stream === "timeline_points" && m.reason === "element_unaccounted" + ); + assert.ok(skip, "a dropped element must be visible as a skip, not swallowed"); + assert.equal((skip as { diagnostics?: { considered?: number; unaccounted?: number } }).diagnostics?.unaccounted, 1); + assert.equal((skip as { diagnostics?: { considered?: number } }).diagnostics?.considered, 2); + } finally { + await rm(importRoot, { force: true, recursive: true }); + } +}); + +test("a segment element with no start time stays in the denominator", async () => { + const importRoot = await mkdtemp(join(tmpdir(), "pdpp-gm-unaccounted-segment-")); + try { + // Two semantic segments; the second has no duration/startTimestamp at all, + // so no `start_time` can be derived and the element cannot be keyed. + await writeFile( + join(importRoot, "Timeline.json"), + JSON.stringify({ + semanticSegments: [ + { activity: { activityType: "WALKING" }, duration: { startTimestamp: "2024-06-05T13:45:22.000Z" } }, + { activity: { activityType: "CYCLING" } }, + ], + }) + ); + const messages = await runImport(importRoot, ["timeline_segments"]); + + const coverage = coverageFor(messages, "timeline_segments"); + assert.equal(coverage.considered, 2, "both artifact elements are in the denominator"); + assert.equal(coverage.covered, 1, "only the keyable segment is covered"); + assert.ok( + messages.some( + (m) => m.type === "SKIP_RESULT" && m.stream === "timeline_segments" && m.reason === "element_unaccounted" + ), + "the dropped segment must be visible" + ); + } finally { + await rm(importRoot, { force: true, recursive: true }); + } +}); + +test("a fully accountable artifact still reconciles as complete on both streams", async () => { + const importRoot = await mkdtemp(join(tmpdir(), "pdpp-gm-fully-accounted-")); + try { + await writeFile( + join(importRoot, "Records.json"), + JSON.stringify({ + locations: [ + { timestampMs: "1717595122000", latitudeE7: 377_749_000, longitudeE7: -1_224_194_000, accuracy: 12.4 }, + { timestampMs: "1717598722000", latitudeE7: 377_800_000, longitudeE7: -1_224_100_000, accuracy: 9.1 }, + ], + }) + ); + await writeFile( + join(importRoot, "Timeline.json"), + JSON.stringify({ + semanticSegments: [ + { activity: { activityType: "WALKING" }, duration: { startTimestamp: "2024-06-05T13:45:22.000Z" } }, + ], + }) + ); + const messages = await runImport(importRoot, ["timeline_points", "timeline_segments"]); + + // No drop => no skip, and the reconciliation reads complete. This guards the + // opposite failure: a guard that counts every element as unaccounted would + // make every clean import read a false `partial`. + assert.equal( + messages.some((m) => m.type === "SKIP_RESULT" && m.reason === "element_unaccounted"), + false, + "a clean artifact must not report an unaccounted element" + ); + const points = coverageFor(messages, "timeline_points"); + assert.equal(points.considered, 2); + assert.equal(points.covered, 2); + const segments = coverageFor(messages, "timeline_segments"); + assert.equal(segments.considered, 1); + assert.equal(segments.covered, 1); + } finally { + await rm(importRoot, { force: true, recursive: true }); + } +}); diff --git a/packages/polyfill-connectors/connectors/google_maps/index.ts b/packages/polyfill-connectors/connectors/google_maps/index.ts index 1eeb4c28f..73afee863 100644 --- a/packages/polyfill-connectors/connectors/google_maps/index.ts +++ b/packages/polyfill-connectors/connectors/google_maps/index.ts @@ -139,8 +139,19 @@ interface LoadSummary { latestSegment: string | undefined; pointCount: number; pointsEmitted: number; + /** + * Point elements the artifact held that this run could NOT account for: the + * parser produced a record, but it carried no usable `id`/`timestamp`, so it + * can be neither emitted nor deduped. These are the artifact's own elements — + * they were considered. Counting them here (and NOT in `pointCount`) is what + * makes `covered < considered` read a real `partial` instead of the silent + * drop that previously reported a fabricated `considered === covered`. + */ + pointsUnaccounted: number; segmentCount: number; segmentsEmitted: number; + /** Segment elements dropped for a missing `id`/`start_time`. See `pointsUnaccounted`. */ + segmentsUnaccounted: number; unrecognizedCount: number; unrecognizedKinds: Set; } @@ -214,6 +225,11 @@ async function processPointRecords(load: StreamLoadContext, points: readonly Tim const pointId = typeof point.id === "string" ? point.id : null; const timestamp = typeof point.timestamp === "string" ? point.timestamp : null; if (!(pointId && timestamp)) { + // The artifact held this element and the parser produced it, but without a + // stable id + timestamp it cannot be emitted or deduped. Count it as + // considered-but-not-covered rather than dropping it silently: an element + // the run could not account for must not be erased from the denominator. + load.summary.pointsUnaccounted += 1; continue; } if (load.seenPointIds.has(pointId)) { @@ -257,6 +273,10 @@ async function processSegmentRecords( const segmentId = typeof segment.id === "string" ? segment.id : null; const startTime = typeof segment.start_time === "string" ? segment.start_time : null; if (!(segmentId && startTime)) { + // See `processPointRecords`: an unaccountable element stays in the + // denominator so the artifact reconciliation reads `partial`, not a + // silent `complete`. + load.summary.segmentsUnaccounted += 1; continue; } if (load.seenSegmentIds.has(segmentId)) { @@ -290,6 +310,25 @@ async function processStreamEvent(load: StreamLoadContext, event: GoogleMapsStre return; } const parsed = parseGoogleMapsExportElement(event.format, event.value); + // THE ARTIFACT BOUNDARY. One `element` event is exactly one element the + // uploaded file contained, counted here — before the parser's verdict — so + // the denominator is measured at the source, not recomputed from survivors. + // + // An element the parser yields nothing for (no point AND no segment) is the + // silent drop this guards: an unkeyable `locations` entry with no + // `timestampMs`, or a `semanticSegments` entry with no start time, both parse + // to `{points:[],segments:[]}`. Attributing it to a stream requires the + // element's own format, since a legacy `locations` element can only ever have + // been a point and a `semanticSegments` element can only ever have been a + // segment. + if (parsed.points.length === 0 && parsed.segments.length === 0) { + if (event.format === "legacy_records") { + load.summary.pointsUnaccounted += 1; + } else { + load.summary.segmentsUnaccounted += 1; + } + return; + } await processPointRecords(load, parsed.points); await processSegmentRecords(load, parsed.segments); } @@ -341,8 +380,10 @@ async function loadExports(ctx: CollectContext, importDir: string, state: Google latestSegment: segmentSince, pointCount: 0, pointsEmitted: 0, + pointsUnaccounted: 0, segmentCount: 0, segmentsEmitted: 0, + segmentsUnaccounted: 0, unrecognizedCount: 0, unrecognizedKinds: new Set(), }; @@ -399,13 +440,34 @@ async function finishPoints(ctx: CollectContext, summary: LoadSummary): Promise< stream: "timeline_points", cursor: { last_timestamp: summary.latestPoint }, }); + // Artifact reconciliation, NOT an account-completeness claim. The denominator + // is every point element the parser produced from the uploaded file — + // accounted-for plus unaccountable — measured at the parse site. The numerator + // is only what this run could actually key and dedupe. When the artifact holds + // an element with no id/timestamp, `covered < considered` and the stream reads + // `partial`, which is the honest verdict. + // + // What this CANNOT say: whether the artifact itself is a complete export of + // the owner's Google Maps history. Google decides what goes into the file; a + // fully-reconciled artifact only proves this run ingested everything the file + // contained, never that the file contained everything the provider holds. + const pointsConsidered = summary.pointCount + summary.pointsUnaccounted; + if (summary.pointsUnaccounted > 0) { + await ctx.emit({ + type: "SKIP_RESULT", + stream: "timeline_points", + reason: "element_unaccounted", + message: `${String(summary.pointsUnaccounted)} of ${String(pointsConsidered)} Google Maps Timeline point element(s) lacked a usable id or timestamp and could not be ingested`, + diagnostics: { considered: pointsConsidered, unaccounted: summary.pointsUnaccounted }, + }); + } await ctx.emit( buildDetailCoverageMessage({ stream: "timeline_points", stateStream: "timeline_points", requiredKeys: [], hydratedKeys: [], - considered: summary.pointCount, + considered: pointsConsidered, covered: summary.pointCount, }) ); @@ -430,6 +492,30 @@ async function finishSegments(ctx: CollectContext, summary: LoadSummary): Promis stream: "timeline_segments", cursor: { last_start_time: summary.latestSegment }, }); + // Same artifact reconciliation as `finishPoints`. `buildFullScanCoverageMessage` + // is deliberately NOT used when an element went unaccounted: it forces + // `covered === considered`, which would re-hide the very drop this counts. + const segmentsConsidered = summary.segmentCount + summary.segmentsUnaccounted; + if (summary.segmentsUnaccounted > 0) { + await ctx.emit({ + type: "SKIP_RESULT", + stream: "timeline_segments", + reason: "element_unaccounted", + message: `${String(summary.segmentsUnaccounted)} of ${String(segmentsConsidered)} Google Maps Timeline segment element(s) lacked a usable id or start time and could not be ingested`, + diagnostics: { considered: segmentsConsidered, unaccounted: summary.segmentsUnaccounted }, + }); + await ctx.emit( + buildDetailCoverageMessage({ + stream: "timeline_segments", + stateStream: "timeline_segments", + requiredKeys: [], + hydratedKeys: [], + considered: segmentsConsidered, + covered: summary.segmentCount, + }) + ); + return; + } await ctx.emit(buildFullScanCoverageMessage("timeline_segments", summary.segmentCount)); } diff --git a/packages/polyfill-connectors/connectors/groupme/completeness-anchor.test.ts b/packages/polyfill-connectors/connectors/groupme/completeness-anchor.test.ts new file mode 100644 index 000000000..3836459ec --- /dev/null +++ b/packages/polyfill-connectors/connectors/groupme/completeness-anchor.test.ts @@ -0,0 +1,249 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Completeness-anchor tests for the GroupMe connector. + * + * `providerMessageCount` reads GroupMe's per-group message count from + * whichever shape the API returned — the documented nested + * `messages.count`, or the flat `messages_count` this connector modelled + * before. Live evidence drove this: all 156 of this owner's groups carried + * `messages_count: null` across every version ever collected, while the + * sibling `members_count` populated normally, so the flat field was never + * the one GroupMe actually sends for the count. + * + * `groupMessageShortfall` compares that count against a full walk. It is + * one-directional: only "provider says MORE than we walked" is a gap. + * Holdings that exceed the provider count are messages GroupMe deleted + * after we preserved them — reporting those as loss would flag correct + * preservation behavior as a defect. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + groupMessageShortfall, + pageEndedShortOfProviderCount, + partitionGroupMessageShortfalls, + providerMessageCount, +} from "./index.ts"; + +const GROUP_ID = "g-anchor"; + +function group(overrides: Record): Parameters[0] { + return { id: GROUP_ID, ...overrides } as Parameters[0]; +} + +// ─── providerMessageCount: read either shape, never fabricate ──────────── + +test("providerMessageCount reads the documented nested messages.count", () => { + assert.equal(providerMessageCount(group({ messages: { count: 142 } })), 142); +}); + +test("providerMessageCount reads the flat messages_count when that is what arrived", () => { + assert.equal(providerMessageCount(group({ messages_count: 99 })), 99); +}); + +test("providerMessageCount prefers the nested count when both shapes are present", () => { + assert.equal(providerMessageCount(group({ messages: { count: 142 }, messages_count: 7 })), 142); +}); + +test("providerMessageCount accepts a genuine zero from the provider", () => { + // The provider SAYING zero is a fact; it is not the same as saying nothing. + assert.equal(providerMessageCount(group({ messages: { count: 0 } })), 0); +}); + +test("providerMessageCount returns null when the provider reported no count", () => { + // This is the live case: unknown must NOT collapse to a zero denominator. + assert.equal(providerMessageCount(group({})), null); + assert.equal(providerMessageCount(group({ messages_count: null })), null); + assert.equal(providerMessageCount(group({ messages: null })), null); + assert.equal(providerMessageCount(group({ messages: {} })), null); +}); + +test("providerMessageCount rejects a malformed count rather than coercing it", () => { + assert.equal(providerMessageCount(group({ messages: { count: -1 } })), null); + assert.equal(providerMessageCount(group({ messages: { count: 1.5 } })), null); + assert.equal(providerMessageCount(group({ messages_count: Number.NaN })), null); + assert.equal(providerMessageCount(group({ messages_count: Number.POSITIVE_INFINITY })), null); +}); + +// ─── groupMessageShortfall: one-directional ────────────────────────────── + +test("groupMessageShortfall reports a gap when the provider claims more than we walked", () => { + const verdict = groupMessageShortfall(group({ messages: { count: 500 } }), 320); + assert.equal(verdict.kind, "short"); + assert.deepEqual(verdict.kind === "short" ? verdict.shortfall : null, { + groupId: GROUP_ID, + providerCount: 500, + unprovenBoundary: false, + walked: 320, + }); +}); + +test("groupMessageShortfall reports ok on an exact match", () => { + assert.equal(groupMessageShortfall(group({ messages: { count: 320 } }), 320).kind, "ok"); +}); + +test("groupMessageShortfall does NOT report a gap when holdings exceed the provider count", () => { + // Messages GroupMe deleted after we preserved them. PDPP retains those on + // purpose, so a surplus must never read as loss. + assert.equal(groupMessageShortfall(group({ messages: { count: 300 } }), 320).kind, "ok"); +}); + +test("groupMessageShortfall reports unanchored when the provider gave no count", () => { + assert.equal(groupMessageShortfall(group({}), 320).kind, "unanchored"); +}); + +test("groupMessageShortfall treats a provider zero as a real anchor, not as unanchored", () => { + assert.equal(groupMessageShortfall(group({ messages: { count: 0 } }), 0).kind, "ok"); +}); + +test("groupMessageShortfall reports a gap for an empty walk against a non-zero count", () => { + const verdict = groupMessageShortfall(group({ messages: { count: 12 } }), 0); + assert.equal(verdict.kind, "short"); + assert.equal(verdict.kind === "short" ? verdict.shortfall.providerCount : null, 12); +}); + +// ─── partitionGroupMessageShortfalls: classify, never subtract ─────────── +// +// The partition keys on `unprovenBoundary` — whether the walk ran out of +// servable history before reaching the provider's lifetime total (`count > 0` +// alongside `messages: []`). +// +// It deliberately does NOT key on `walked === 0`. Measured live against this +// API, an empty page carries no status code, `Retry-After`, or rate-limit +// header distinguishing "these messages are gone" from "not serving them +// now", so `walked === 0` is precisely the ambiguous observation. Keying an +// "unrecoverable" verdict on it would let the connector assert a certainty +// the response cannot support. + +test("partitionGroupMessageShortfalls routes a short-of-total empty page to unexplained", () => { + const { partial, unexplained } = partitionGroupMessageShortfalls([ + { groupId: "g-alpha", providerCount: 98, unprovenBoundary: true, walked: 0 }, + ]); + assert.equal(unexplained.length, 1); + assert.equal(partial.length, 0); + assert.equal(unexplained[0]?.providerCount, 98); +}); + +test("partitionGroupMessageShortfalls routes a genuinely partial walk to partial", () => { + const { partial, unexplained } = partitionGroupMessageShortfalls([ + { groupId: "g-beta", providerCount: 61, unprovenBoundary: false, walked: 40 }, + ]); + assert.equal(partial.length, 1); + assert.equal(unexplained.length, 0); + assert.equal(partial[0]?.walked, 40); +}); + +test("partitionGroupMessageShortfalls does NOT route a zero-message walk to unexplained on walked===0 alone", () => { + // The regression guard for the unproven-boundary defect. A walk that saw + // nothing but ended on a COHERENT page (the provider's own count agreed at + // zero) is an ordinary shortfall, not an ambiguous one. If this ever routes + // on `walked === 0` again, the connector is back to inferring a verdict from + // the one signal that cannot carry it. + const { partial, unexplained } = partitionGroupMessageShortfalls([ + { groupId: "g-gamma", providerCount: 7, unprovenBoundary: false, walked: 0 }, + ]); + assert.equal(unexplained.length, 0); + assert.equal(partial.length, 1); +}); + +test("partitionGroupMessageShortfalls preserves every message of the gap across both buckets", () => { + // The whole point: classification must not lose a single claimed message. + // The combined total must equal the pre-split total. + const shortfalls = [ + { groupId: "g-alpha", providerCount: 98, unprovenBoundary: true, walked: 0 }, + { groupId: "g-delta", providerCount: 1, unprovenBoundary: true, walked: 0 }, + { groupId: "g-beta", providerCount: 61, unprovenBoundary: false, walked: 40 }, + ]; + const before = shortfalls.reduce((sum, s) => sum + (s.providerCount - s.walked), 0); + const { partial, unexplained } = partitionGroupMessageShortfalls(shortfalls); + const after = + partial.reduce((sum, s) => sum + (s.providerCount - s.walked), 0) + + unexplained.reduce((sum, s) => sum + (s.providerCount - s.walked), 0); + assert.equal(after, before); + assert.equal(partial.length + unexplained.length, shortfalls.length); +}); + +test("partitionGroupMessageShortfalls keeps an unexplained group's messages counted as missing", () => { + // An unexplained group is NOT explained away: its unserved messages are + // still absent from PDPP. Subtracting them to make the books balance would + // be exactly the fabricated reconciliation this anchor exists to prevent. + const { unexplained } = partitionGroupMessageShortfalls([ + { groupId: "g-epsilon", providerCount: 55, unprovenBoundary: true, walked: 0 }, + ]); + assert.equal( + unexplained.reduce((sum, s) => sum + (s.providerCount - s.walked), 0), + 55 + ); +}); + +// ─── pageEndedShortOfProviderCount: the unproven-boundary detector ───────── +// +// `count` is GroupMe's LIFETIME TOTAL for the conversation, not the size of +// the page just served (see connectors/groupme/docs/groupme-message-count-shortfall.md). +// So an empty page against a non-zero count is NOT the provider +// contradicting itself — it means this walk ran out of servable history +// before reaching everything the total implies, most often because the +// oldest messages are no longer stored on GroupMe's side. +// +// It is still ambiguous as to CAUSE: retention and a transient refusal +// produce byte-identical responses (same status, same `meta.code`, no +// `Retry-After`, no rate-limit header), so the walk records the boundary as +// unproven and names no cause. These tests pin that predicate. + +test("pageEndedShortOfProviderCount flags an empty page served against a non-zero count", () => { + assert.equal(pageEndedShortOfProviderCount({ count: 29, messages: [] }), true); +}); + +test("pageEndedShortOfProviderCount accepts an empty page whose count agrees at zero", () => { + // Nothing to serve and the provider says so. An ordinary natural end. + assert.equal(pageEndedShortOfProviderCount({ count: 0, messages: [] }), false); +}); + +test("pageEndedShortOfProviderCount never fires on GroupMe's documented 304 terminal page", () => { + // GroupMe documents: "If no messages are found (e.g. when filtering with + // `before_id`) we return code 304." `fetchMessagesPage` normalizes that + // 304 into a SYNTHETIC `{count: 0, messages: []}` — a count GroupMe never + // sent. This test pins that the synthesized shape reads as a clean natural + // end, so the documented end-of-history signal can never be reported as a + // shortfall against the provider. If `fetchMessagesPage` ever synthesizes a + // non-zero count, every fully-collected group would be accused of holding + // back data. + const synthesizedBy304 = { count: 0, messages: [] }; + assert.equal(pageEndedShortOfProviderCount(synthesizedBy304), false); +}); + +test("pageEndedShortOfProviderCount never flags a page that actually served messages", () => { + // A served page is self-consistent regardless of how `count` compares: + // `count` describes the conversation, the array describes this page. + const served = [{ id: "m1" }, { id: "m2" }] as unknown as Parameters< + typeof pageEndedShortOfProviderCount + >[0]["messages"]; + assert.equal(pageEndedShortOfProviderCount({ count: 900, messages: served }), false); + assert.equal(pageEndedShortOfProviderCount({ count: 2, messages: served }), false); +}); + +test("pageEndedShortOfProviderCount treats a missing/non-numeric count as no shortfall", () => { + // Unknown is not a contradiction. An absent count cannot testify against + // the empty array, so it must not manufacture an ambiguity finding. + const noCount = { messages: [] } as unknown as Parameters[0]; + assert.equal(pageEndedShortOfProviderCount(noCount), false); + const nullCount = { count: null, messages: [] } as unknown as Parameters[0]; + assert.equal(pageEndedShortOfProviderCount(nullCount), false); +}); + +test("groupMessageShortfall carries the unproven-boundary flag into the shortfall", () => { + // The flag must survive the hop from the walk to the shortfall record, or + // the partition downstream silently sees every gap as explained. + const verdict = groupMessageShortfall(group({ messages: { count: 29 } }), 0, true); + assert.equal(verdict.kind, "short"); + assert.equal(verdict.kind === "short" ? verdict.shortfall.unprovenBoundary : null, true); +}); + +test("groupMessageShortfall defaults the unproven-boundary flag to false", () => { + const verdict = groupMessageShortfall(group({ messages: { count: 29 } }), 10); + assert.equal(verdict.kind, "short"); + assert.equal(verdict.kind === "short" ? verdict.shortfall.unprovenBoundary : null, false); +}); diff --git a/packages/polyfill-connectors/connectors/groupme/docs/groupme-message-count-shortfall.md b/packages/polyfill-connectors/connectors/groupme/docs/groupme-message-count-shortfall.md new file mode 100644 index 000000000..b0dc935d2 --- /dev/null +++ b/packages/polyfill-connectors/connectors/groupme/docs/groupme-message-count-shortfall.md @@ -0,0 +1,132 @@ +# GroupMe: why some groups stop short of the provider's message count + +**Verdict: the earlier claim that GroupMe "contradicted itself" was WRONG and has been retracted.** + +A prior revision emitted the reason code +`provider_served_empty_page_against_its_own_count`, asserting that GroupMe +reported a message count and then served an empty page *for that count* — +i.e. that the API contradicted itself in a single response body. That claim +was never verified. This note records the evidence that refutes it, and what +the observation actually means. + +The reason code is now `history_ended_before_provider_count`, which +describes what we observed rather than what we concluded about the provider. + +## 1. What `count` actually means + +`count` is **undocumented**. GroupMe's API reference (dev.groupme.com/docs/v3) +shows it only inside the sample body for `GET /groups/:id/messages` and never +defines it in prose. Every available signal says it is the conversation's +**lifetime total**, not the size of the page just served: + +- `GET /groups` returns the same key as `messages.count`, sitting beside + `last_message_id` — self-evidently a group total. +- A per-page reading would make `count` always equal `messages.length`, so the + field would carry no information at all. +- Groupy, the most widely used Python wrapper, binds `count` to a field named + `message_count` and returns it from `__len__` — i.e. "how many messages this + group has". It re-reads it on every page because the total can drift upward + while paginating. +- No mature client terminates pagination on a `count` comparison. Groupy, + `cdzombak/groupme-tools`, and others all terminate on HTTP 304, an empty + array, or a short page. + +**Therefore an empty page against a non-zero `count` is not a contradiction.** +The two fields answer different questions: "how many messages has this group +ever held" versus "what can I serve you from this cursor". A total that +exceeds what pagination can reach is an ordinary, expected state. + +Note also that the two counts in our own code come from **different +endpoints** and must not be conflated: `pageEndedShortOfProviderCount` reads +the per-response `count` from `GET /groups/:id/messages`, while the shortfall +arithmetic uses `providerMessageCount()` from the `/groups` listing. + +## 2. The documented terminal page cannot produce a false positive + +GroupMe documents: *"If no messages are found (e.g. when filtering with +`before_id`) we return code 304."* Our `fetchMessagesPage` normalizes that 304 +into a synthetic `{count: 0, messages: []}`. Because the synthetic count is +zero, `pageEndedShortOfProviderCount` returns `false`, so the documented +end-of-history signal never trips the shortfall path. The path can only be +reached by a genuine HTTP 200 carrying a real non-zero `count`. + +## 3. What is actually happening: a retention cliff at Aug 2013 + +Measured against this owner's live Postgres (connector instance +`cin_5804a2ff36cd303e22762745`, 88,743 stored `group_messages` across 156 +groups): + +| bucket | groups | meaning | +|---|---|---| +| fully reconciled | 77 | stored count >= provider total | +| partial | 36 | some messages stored, fewer than the total | +| zero stored, non-zero total | 42 | nothing stored at all | + +Grouping the 156 groups by the year of their last activity (`updated_at`) +makes the cause unmistakable: + +| last active | groups | zero stored | has messages | +|---|---|---|---| +| 2011 | 1 | 1 | 0 | +| 2012 | 31 | 31 | 0 | +| 2013 | 12 | 10 | 2 | +| 2014 | 13 | 0 | 13 | +| 2015 | 20 | 1 | 19 | +| 2016–2024 | 79 | 0 | 79 | + +Every group that went dormant in 2011–2012 stored **zero** messages. Every +group active from 2014 onward stored messages. This is a temporal cliff, not +the scattered, load-correlated pattern throttling would produce. + +The cliff has an exact date. **The oldest stored message anywhere in this +account is `2013-08-15T18:08:31.000Z`.** And of the 36 partial groups, all 36 +have their oldest surviving message on or after that date — zero exceptions. +A group's history is retrievable back to roughly Aug 2013 and no further, +regardless of which group it is. + +Independent corroboration: GroupMe's own API support forum carries a user +report of paginating `before_id` to a 304 and reaching only ~62k of a reported +70k messages, with *"Last message it returns is from Aug 2013."* Same wall, +same date, a different account. The working explanation there was likewise +server-side retention. + +## 4. Scope, and whether the data is recoverable + +The production run (`run_1787350099426`, evidence as of +`2026-08-21T22:11:56Z`) reported **1,601 messages across 42 groups**. Those +42 groups are exactly the zero-stored set above; all show `walked: 0`. + +Independent reconciliation of stored records against the persisted `/groups` +totals gives a **total shortfall of 6,982 messages across 78 groups** — the +1,601 figure covers only the 42 zero-stored groups, not the 36 partial ones. +The affected messages are all pre-Aug-2013. + +**Is the owner's data missing?** These messages were never collected, because +GroupMe would not serve them. They are not lost *by PDPP* — nothing was +collected and then dropped. Everything GroupMe did serve is stored. + +**Is it recoverable?** Almost certainly not through this API. Retrying will +not help if the messages are no longer stored on GroupMe's side, and the +evidence strongly indicates that. We nevertheless keep the gap `retryable` +rather than asserting `not_retriable`, because a single response cannot +distinguish retention from a transient refusal *per group*, and asserting +unrecoverability would repeat the original error of converting an ambiguous +observation into a certain verdict. + +## 5. What could not be verified + +- **No live API call was made.** The stored credential is sealed, and + unsealing owner secrets was out of scope. A single live request against one + affected group (e.g. `4747691`, provider count 1, zero stored) would + directly confirm the response shape; this note rests on stored artifacts and + documentation instead. +- **No captured raw API response exists.** `/root/.pdpp/fixture-captures/` + in `pdpp-core-prod-drain` has no `groupme` directory, so the primary + artifact — a recorded body showing `count: N` with `messages: []` — was + never captured. The emitted `known_gaps` diagnostics in `spine_events` are + the closest available evidence, and they record only our derived + per-group `provider_count`/`walked` pairs, not the raw page. +- **GroupMe's server-side intent for `count` is not provable** from primary + sources, because the field is undocumented. The conclusion rests on + convergent circumstantial evidence (see §1), which is strong but not + certain. diff --git a/packages/polyfill-connectors/connectors/groupme/index.ts b/packages/polyfill-connectors/connectors/groupme/index.ts index 14d50dc53..72ec002b6 100644 --- a/packages/polyfill-connectors/connectors/groupme/index.ts +++ b/packages/polyfill-connectors/connectors/groupme/index.ts @@ -124,6 +124,17 @@ interface GroupMeGroup { id: string; image_url?: string | null; members_count?: number | null; + /** + * GroupMe's own documented per-group message envelope on `GET /groups`: + * `{ count, last_message_id, last_message_created_at, preview }`. + * + * The connector previously read only a FLAT `messages_count`, which is + * absent from that documented shape — and live evidence agrees: all 156 + * of this owner's groups carry `messages_count: null` across every + * version ever collected, while the sibling `members_count` populates + * normally. So the flat field was never the real one. + */ + messages?: { count?: number | null } | null; messages_count?: number | null; muted?: boolean | null; name?: string | null; @@ -134,6 +145,33 @@ interface GroupMeGroup { updated_at?: number | null; } +/** + * The provider-reported message count for one group, read from whichever + * shape the API actually returned: the documented nested + * `messages.count`, or the flat `messages_count` this connector has always + * modelled. + * + * Returns `null` when NEITHER is present or usable. That distinction is + * load-bearing: `null` means "the provider did not tell us", which is + * different from `0` ("the provider says this group is empty"). A missing + * count must never collapse into a zero denominator — that would assert an + * empty group that was simply never reported on. + * + * Rejects non-integer, negative, and non-finite values rather than + * coercing them, so a malformed provider value degrades to "unknown" + * instead of silently becoming a fabricated anchor. + */ +export function providerMessageCount(group: GroupMeGroup): number | null { + const candidate = group.messages?.count ?? group.messages_count; + if (typeof candidate !== "number" || !Number.isFinite(candidate)) { + return null; + } + if (!Number.isInteger(candidate) || candidate < 0) { + return null; + } + return candidate; +} + interface GroupMeAttachment { charmap?: [number, number][] | null; file_id?: string | null; @@ -841,7 +879,9 @@ function toGroupRecord(g: GroupMeGroup): RecordData { created_at: convertTimestamp(g.created_at), updated_at: convertTimestamp(g.updated_at), member_count: g.members_count ?? null, - messages_count: g.messages_count ?? null, + // Reads whichever shape the API returned. The prior flat-only read is + // why all 156 of this owner's groups persisted `messages_count: null`. + messages_count: providerMessageCount(g), }; } @@ -997,6 +1037,61 @@ interface GroupMessagesResponse { messages: GroupMeMessage[]; } +/** + * Whether a message page ended the walk WITHOUT reaching a message the + * provider still counts: the response served `messages: []` while its own + * `count` field was greater than zero. + * + * WHAT `count` ACTUALLY MEANS — and what this predicate must NOT claim. + * `count` is UNDOCUMENTED. GroupMe's API reference + * (dev.groupme.com/docs/v3) shows it only inside a sample body for + * `GET /groups/:id/messages` and never defines it in prose. All available + * evidence says it is the conversation's LIFETIME TOTAL, not the size of + * the page just served: + * + * - `GET /groups` returns the same key as `messages.count`, sitting beside + * `last_message_id` — self-evidently a group total. + * - A per-page reading would make `count` exactly `messages.length`, i.e. + * carry no information at all. + * - Mature clients treat it as a total (Groupy binds it to + * `message_count` and returns it from `__len__`), and terminate + * pagination on 304 / empty / short pages — never on a `count` + * comparison. + * + * So an empty page against a non-zero total is NOT the provider + * contradicting itself: the two fields answer different questions ("how + * many has this group ever held" vs "what can I serve you from here"). A + * total that exceeds what `before_id` pagination can reach is a documented + * real-world state — GroupMe's own API support forum has a user walking + * `before_id` to a 304 and reaching only 62k of a reported 70k messages, + * with history stopping in Aug 2013, attributed to server-side retention. + * This owner's data shows the SAME Aug-2013 floor (see + * connectors/groupme/docs/groupme-message-count-shortfall.md). + * + * What the predicate therefore means, and all it means: this walk ran out + * of servable history while the provider's total still implied more, so + * the walk's boundary is UNPROVEN. It could be retention (old messages + * counted but no longer stored), messages deleted without decrementing the + * total, or a transient refusal — the response carries nothing that tells + * them apart, so the connector names none of them. + * + * NOTE the two `count` values in play are from DIFFERENT endpoints and must + * not be conflated: this predicate reads the per-response `count` from + * `GET /groups/:id/messages`, while the shortfall arithmetic downstream + * uses `providerMessageCount()` from the `/groups` listing. + * + * `count === 0` with an empty array is coherent and stays an ordinary + * natural end. GroupMe's documented end-of-history signal — HTTP 304, "If + * no messages are found (e.g. when filtering with `before_id`) we return + * code 304" — is normalized to a synthetic `{count: 0, messages: []}` by + * `fetchMessagesPage`, so the documented terminal page can never reach this + * predicate as a false positive. + */ +export function pageEndedShortOfProviderCount(resp: GroupMessagesResponse): boolean { + const served = (resp.messages || []).length; + return served === 0 && typeof resp.count === "number" && resp.count > 0; +} + /** * Walks one group's message pages. Returns the raw item count enumerated * across pages (the "considered" contribution for this group) — never @@ -1015,6 +1110,20 @@ interface PerConversationWalkResult { * ordering to license one). */ newestMessageId: string | undefined; totalSeen: number; + /** + * True when this walk RAN OUT OF SERVABLE HISTORY before reaching the + * provider's total (see `pageEndedShortOfProviderCount`) — an empty array + * served against a non-zero `count`. The walk's boundary is then UNPROVEN: + * the remainder may be gone from the provider for good (retention), or it + * may be temporarily unserved, and the response carries nothing that tells + * them apart. + * + * Callers must not treat such a walk as proof of completeness. It is + * deliberately separate from `failed`: nothing errored, the request + * succeeded, and partial results already emitted stay valid — only the + * COMPLETENESS claim is withheld. + */ + unprovenBoundary: boolean; } /** @@ -1278,7 +1387,14 @@ async function collectGroupMessagesForwardFromCursor( const messages = resp.messages || []; if (!messages.length) { - return { totalSeen, newestMessageId }; + // NOT checked against the provider count here, unlike the backward + // walk. This is a forward resume from a cursor, so `count` describes + // the group's WHOLE history while this page describes only what is + // NEW since that cursor. `count > 0` with an empty page is the normal, + // coherent shape of an up-to-date incremental walk — applying the + // backward walk's predicate here would flag every healthy incremental + // run as unproven. + return { totalSeen, newestMessageId, unprovenBoundary: false }; } if (!isAscendingByCreatedAt(messages)) { // The provider violated its own documented ordering contract for this @@ -1300,7 +1416,7 @@ async function collectGroupMessagesForwardFromCursor( newestMessageId = messages.at(-1)?.id ?? newestMessageId; if (messages.length < PAGE_SIZE) { - return { totalSeen, newestMessageId }; + return { totalSeen, newestMessageId, unprovenBoundary: false }; } const nextAfterId = messages.at(-1)?.id; @@ -1377,7 +1493,13 @@ async function collectGroupMessagesBackwardToNaturalEnd( const messages = resp.messages || []; if (!messages.length) { - return { totalSeen, newestMessageId }; + // An empty page ends the walk either way — there is no cursor to + // continue from. What differs is whether that ending PROVES the group + // was fully walked. An empty page served against a non-zero lifetime + // total proves nothing (see `pageEndedShortOfProviderCount`), so the + // boundary is reported as unproven instead of silently passing for + // complete. + return { totalSeen, newestMessageId, unprovenBoundary: pageEndedShortOfProviderCount(resp) }; } if (newestMessageId === undefined) { @@ -1411,7 +1533,10 @@ async function collectGroupMessagesBackwardToNaturalEnd( await emitInScopeGroupMessages(inScope, group.id, cursor, uploader, emitAttachmentRecord, emitRecord); if (backwardPageReachedNaturalEnd(messages, inScope, pageFullyOutOfScope, sinceEpochSeconds)) { - return { totalSeen, newestMessageId }; + // A natural end reached on a page that DID serve messages: the + // provider gave content right up to the boundary, so the boundary is + // proven in the ordinary way. + return { totalSeen, newestMessageId, unprovenBoundary: false }; } const nextBeforeId = messages.at(-1)?.id; @@ -1734,7 +1859,16 @@ async function collectDirectChatMessagesForChat( const messages = resp.direct_messages || []; if (!messages.length) { - return { totalSeen, newestMessageId: undefined }; + // Same short-of-total check as the group backward walk. This walk is + // always backward-to-natural-end (never a forward cursor resume), so + // the response's `count` and its served page describe the same whole + // history, and a `count > 0` empty page means this walk stopped before + // reaching everything the total implies. + return { + totalSeen, + newestMessageId: undefined, + unprovenBoundary: pageEndedShortOfProviderCount({ count: resp.count, messages }), + }; } // In-scope-only accounting, same as the group_messages backward walk: a @@ -1759,7 +1893,7 @@ async function collectDirectChatMessagesForChat( } if (messages.length < PAGE_SIZE) { - return { totalSeen, newestMessageId: undefined }; + return { totalSeen, newestMessageId: undefined, unprovenBoundary: false }; } const nextBeforeId = messages.at(-1)?.id; @@ -1825,6 +1959,116 @@ export async function collectDirectChatMessages( */ export interface GroupMessagesCollectionOutcome extends CollectionOutcome { nextCursors: GroupMessageCursors; + /** Per-group provider-count reconciliation for this run. */ + shortfalls: GroupMessageShortfall[]; + /** Groups whose walk could not be anchored because the provider reported no count. */ + unanchoredGroupIds: string[]; +} + +/** One group where the provider reported MORE messages than the walk saw. */ +export interface GroupMessageShortfall { + groupId: string; + providerCount: number; + /** + * True when the walk that produced this shortfall ran out of servable + * history before reaching the provider's total (see + * `pageEndedShortOfProviderCount`) — so the shortfall is UNEXPLAINED: the + * connector cannot tell retention (the oldest messages are gone from the + * provider) from a transient refusal to serve them. + * + * Optional so existing constructions (and fixtures) stay valid; absent is + * read as `false`. + */ + unprovenBoundary?: boolean; + walked: number; +} + +/** + * Split a run's shortfalls into the situations they conflate, so each can be + * reported with its own reason and recovery hint. + * + * `unexplained` — the walk RAN OUT OF SERVABLE HISTORY before reaching the + * provider's total: GroupMe served `messages: []` while stating `count > 0` + * in the same body. Because `count` is a lifetime total (see + * `pageEndedShortOfProviderCount`), this is not self-contradiction — but it + * is still ambiguous as to CAUSE. Measured live, the response is + * byte-identical (headers included, `content-length` aside) whether the + * oldest messages are gone for good or the provider is declining to serve + * them right now, and GroupMe emits no status code, `Retry-After`, or + * rate-limit header to separate them. The connector therefore does NOT + * claim to know which it is, and does not claim the group was proven walked. + * + * On this owner's real data the evidence points overwhelmingly at + * retention: all 42 affected groups went dormant before Aug 2013, and no + * stored message anywhere predates 2013-08-15 (see + * connectors/groupme/docs/groupme-message-count-shortfall.md). But "overwhelmingly" is not + * "provably", and one run's response cannot establish it per-group. + * + * A PRIOR REVISION of this function classified exactly this shape as + * definitively `not_retriable`, keyed on `walked === 0`. That was + * unsound: `walked === 0` IS the ambiguous signal, so keying the + * "unrecoverable" verdict on it means the connector asserts unrecoverability + * on the strength of a response that cannot support the claim. Whatever the + * true cause on any given group, a connector must not convert an ambiguous + * observation into a certain verdict. + * + * `partial` — the walk returned SOME messages but fewer than claimed, and + * ended on a page the provider actually served. The shortfall is real and the + * boundary evidence is coherent, so retrying is a meaningful suggestion. + * + * This is a classification split ONLY. NEITHER bucket is subtracted from the + * missing total, and neither is counted as covered: an unserved message is + * still an absent message, and saying so honestly is the whole point of the + * anchor. + */ +export function partitionGroupMessageShortfalls(shortfalls: readonly GroupMessageShortfall[]): { + partial: GroupMessageShortfall[]; + unexplained: GroupMessageShortfall[]; +} { + const partial: GroupMessageShortfall[] = []; + const unexplained: GroupMessageShortfall[] = []; + for (const s of shortfalls) { + if (s.unprovenBoundary === true) { + unexplained.push(s); + } else { + partial.push(s); + } + } + return { partial, unexplained }; +} + +/** + * Compare one group's provider-reported message count against what this + * run's walk actually enumerated. + * + * ONE-DIRECTIONAL ON PURPOSE. Only `providerCount > walked` is reported. + * The reverse — we walked more than the provider now claims — is NOT a + * defect: PDPP is a preservation product, so messages deleted from GroupMe + * after we collected them legitimately make our holdings larger than the + * provider's current count. Reporting that as a gap would flag successful + * preservation as loss. + * + * Returns `null` when the provider reported no usable count: unknown is + * not zero, and an absent count must never become a denominator. + * + * CEILING: this is a scalar, so it cannot distinguish "we are missing N + * distinct messages" from "we collected N duplicates". GroupMe exposes no + * cheap per-group message-id listing to make this a set comparison, so the + * anchor detects magnitude only — see the connector README note. + */ +export function groupMessageShortfall( + group: GroupMeGroup, + walked: number, + unprovenBoundary = false +): { kind: "ok" } | { kind: "short"; shortfall: GroupMessageShortfall } | { kind: "unanchored" } { + const providerCount = providerMessageCount(group); + if (providerCount === null) { + return { kind: "unanchored" }; + } + if (providerCount > walked) { + return { kind: "short", shortfall: { groupId: group.id, providerCount, unprovenBoundary, walked } }; + } + return { kind: "ok" }; } /** @@ -1881,6 +2125,109 @@ async function collectOneGroupMessages( ); } +/** Cap on ids/entries listed in an anchor diagnostic; counts are always exact. */ +const MAX_ANCHOR_IDS_IN_DIAGNOSTIC = 50; + +/** + * Emit the provider-count reconciliation for `group_messages`. + * + * Two distinct findings, deliberately not merged: + * + * - `provider_reports_more_messages_than_walked` — a PARTIAL walk. GroupMe + * says a group holds more messages than the full walk enumerated, and the + * walk did return some. Retrying can plausibly close this. + * - `history_ended_before_provider_count` — the walk ran out of servable + * history before reaching the provider's total: an empty page whose own + * `count` was still non-zero. `count` is a lifetime total, so this is NOT + * the provider contradicting itself; it is the ordinary shape of a total + * that exceeds what pagination can reach (most often retention of the + * oldest messages). It stays retryable because the response cannot rule + * out a transient refusal either. + * - `group_message_count_unanchored` — the provider reported no usable + * count for these groups, so their walk has NO external anchor at all. + * Saying so is the honest alternative to silently treating an unanchored + * group as proven. + * + * The two shortfall buckets are a CLASSIFICATION split, never a subtraction: + * an unexplained group's messages stay in its own missing total and are + * never counted as covered. Splitting them tells the owner which part of the + * gap has coherent evidence behind it, without quietly shrinking the gap. + * + * Nothing is emitted for a group where holdings EXCEED the provider count: + * that is preservation of messages GroupMe has since deleted, not loss. + */ +async function emitGroupMessageAnchorEvidence( + emit: CollectContext["emit"], + outcome: GroupMessagesCollectionOutcome +): Promise { + const { partial, unexplained } = partitionGroupMessageShortfalls(outcome.shortfalls); + if (partial.length > 0) { + const missingTotal = partial.reduce((sum, s) => sum + (s.providerCount - s.walked), 0); + const visible = partial.slice(0, MAX_ANCHOR_IDS_IN_DIAGNOSTIC); + await emit({ + type: "SKIP_RESULT", + stream: "group_messages", + reason: "provider_reports_more_messages_than_walked", + message: + `GroupMe reports more messages than this run walked in ${String(partial.length)} group(s): ` + + `${String(missingTotal)} message(s) unaccounted for. Their history may be incomplete.`, + diagnostics: { + short_group_count: partial.length, + missing_message_total: missingTotal, + groups: visible.map((s) => ({ group_id: s.groupId, provider_count: s.providerCount, walked: s.walked })), + truncated: visible.length < partial.length, + }, + recovery_hint: { action: "retry_by_runtime", retryable: true }, + }); + } + if (unexplained.length > 0) { + const unexplainedTotal = unexplained.reduce((sum, s) => sum + (s.providerCount - s.walked), 0); + const visible = unexplained.slice(0, MAX_ANCHOR_IDS_IN_DIAGNOSTIC); + await emit({ + type: "SKIP_RESULT", + stream: "group_messages", + reason: "history_ended_before_provider_count", + message: + `GroupMe's history ran out before reaching its own message total in ${String(unexplained.length)} group(s): ` + + `${String(unexplainedTotal)} message(s) are included in GroupMe's totals but were not served to us. ` + + "GroupMe's total counts a group's whole lifetime, while the messages it will hand back can stop earlier — " + + "most often because the oldest messages are no longer stored on its side. Everything collected is saved; " + + "these group(s) are simply not claimed as fully read.", + diagnostics: { + unexplained_group_count: unexplained.length, + unexplained_message_total: unexplainedTotal, + groups: visible.map((s) => ({ group_id: s.groupId, provider_count: s.providerCount, walked: s.walked })), + truncated: visible.length < unexplained.length, + }, + // Retryable, and deliberately so. The most likely cause is retention + // (the oldest messages are gone from the provider), which retrying + // will NOT recover — but a transient refusal produces the identical + // response, and the connector cannot tell them apart per-group. The + // honest hint is the one that leaves the door open; claiming + // `not_retriable` would assert a certainty the evidence does not + // support. + recovery_hint: { action: "retry_by_runtime", retryable: true }, + }); + } + if (outcome.unanchoredGroupIds.length > 0) { + const visible = outcome.unanchoredGroupIds.slice(0, MAX_ANCHOR_IDS_IN_DIAGNOSTIC); + await emit({ + type: "SKIP_RESULT", + stream: "group_messages", + reason: "group_message_count_unanchored", + message: + `GroupMe reported no per-group message count for ${String(outcome.unanchoredGroupIds.length)} group(s), so their ` + + "walk has no external completeness anchor. Coverage for these groups is unproven, not proven complete.", + diagnostics: { + unanchored_group_count: outcome.unanchoredGroupIds.length, + unanchored_group_ids: visible, + truncated: visible.length < outcome.unanchoredGroupIds.length, + }, + recovery_hint: { action: "retry_by_runtime", retryable: true }, + }); + } +} + export async function collectGroupMessages( token: string, cursor: ReturnType, @@ -1907,6 +2254,8 @@ export async function collectGroupMessages( // cursor. The cursor MAP is still rebuilt from what this full walk // observes, so the next ordinary run resumes forward-incrementally again. const bypassCursor = collectionMode === "full_refresh"; + const shortfalls: GroupMessageShortfall[] = []; + const unanchoredGroupIds: string[] = []; const outcome = await runCollectionPass( "group_messages", "group messages", @@ -1921,6 +2270,7 @@ export async function collectGroupMessages( ); for (const group of groups) { const priorCursor = priorCursors[group.id]; + const walkedWholeGroup = priorCursor === undefined || bypassCursor; const groupResult = await collectOneGroupMessages( token, group, @@ -1934,6 +2284,21 @@ export async function collectGroupMessages( sinceEpochSeconds ); considered += groupResult.totalSeen; + // The provider count describes the group's WHOLE history, so it can + // only be compared against a walk that covered the whole history: a + // cold start or an explicit full_refresh. An incremental forward + // resume deliberately sees only new messages, and comparing a total + // against that window would report a false shortfall on every + // healthy incremental run. A `since`-scoped walk is excluded for the + // same reason. + if (walkedWholeGroup && sinceEpochSeconds === null) { + const verdict = groupMessageShortfall(group, groupResult.totalSeen, groupResult.unprovenBoundary); + if (verdict.kind === "short") { + shortfalls.push(verdict.shortfall); + } else if (verdict.kind === "unanchored") { + unanchoredGroupIds.push(group.id); + } + } if (groupResult.newestMessageId !== undefined) { nextCursors[group.id] = groupResult.newestMessageId; } @@ -1942,7 +2307,14 @@ export async function collectGroupMessages( }, reportStreamFailure ); - return { ...outcome, nextCursors: outcome.failed ? {} : nextCursors }; + return { + ...outcome, + nextCursors: outcome.failed ? {} : nextCursors, + // A failed pass proves nothing about completeness — withhold both + // findings exactly as the cursor map is withheld. + shortfalls: outcome.failed ? [] : shortfalls, + unanchoredGroupIds: outcome.failed ? [] : unanchoredGroupIds, + }; } /** @@ -2027,6 +2399,7 @@ export async function collect( effectiveCollectionMode, reportStreamFailure ); + await emitGroupMessageAnchorEvidence(emit, groupMessagesOutcome); } let directChatsOutcome: CollectionOutcome | undefined; if (requested.has("direct_messages")) { diff --git a/packages/polyfill-connectors/connectors/groupme/stream-coverage.test.ts b/packages/polyfill-connectors/connectors/groupme/stream-coverage.test.ts index 60d08c629..e3b65da17 100644 --- a/packages/polyfill-connectors/connectors/groupme/stream-coverage.test.ts +++ b/packages/polyfill-connectors/connectors/groupme/stream-coverage.test.ts @@ -373,6 +373,18 @@ test("collectGroupMessages: cold start (no prior cursor) walks backward, clean p considered: 3, failed: false, nextCursors: { "group-1": "m1", "group-2": "m3" }, + // The `group()` fixture declares `messages_count: 10` but the stubbed + // pages supply only 2 and 1 messages, so the provider-count anchor + // correctly reports both groups short. This is the anchor doing its + // job against the fixture's own numbers, not a regression. + // `unprovenBoundary: false` on both: each walk ended on a page the + // provider actually served, so the boundary evidence is coherent and + // the shortfall is an ordinary one, not an ambiguous empty-page case. + shortfalls: [ + { groupId: "group-1", providerCount: 10, unprovenBoundary: false, walked: 2 }, + { groupId: "group-2", providerCount: 10, unprovenBoundary: false, walked: 1 }, + ], + unanchoredGroupIds: [], }); assert.equal(emitted.filter((r) => r.stream === "group_messages").length, 3); } finally { @@ -474,7 +486,7 @@ test("collectGroupMessages: genuine zero groups reports failed: false, considere assert.deepEqual( outcome, - { considered: 0, failed: false, nextCursors: {} }, + { considered: 0, failed: false, nextCursors: {}, shortfalls: [], unanchoredGroupIds: [] }, "no groups means no messages — a proven-empty walk" ); assert.equal(emitted.length, 0); @@ -1254,3 +1266,180 @@ test("collect(): failed direct_chat_messages reports failure while preserving su restore(); } }); + +// ─── throttle-blindness: an empty page against a non-zero count ─────────── +// +// GroupMe answers with HTTP 200 + `messages: []` both when it has nothing to +// serve and when it is declining to serve content it still counts. Measured +// live, those two responses are identical apart from `content-length` — same +// status, same `meta.code`, no `Retry-After`, no rate-limit header — so the +// status-based retry governor cannot tell them apart. +// +// The connector must therefore refuse to claim a PROVEN walk in that case, +// and must not assert the gap is unrecoverable either. These tests pin both +// halves at the `collectGroupMessages` boundary. + +test("collectGroupMessages: an empty page short of the provider count marks the boundary unproven", async () => { + const restore = stubFetchSequence([ + { body: { response: [group({ id: "group-1" })] } }, // /groups + // The provider counts 10 messages and serves none — the exact live shape. + { body: { response: { count: 10, messages: [] } } }, + ]); + try { + const cursor = openFingerprintCursor(new Map()); + const { emitRecord } = makeHarness(); + const outcome = await collectGroupMessages(TOKEN, cursor, undefined, undefined, noopProgress, emitRecord); + + assert.equal(outcome.failed, false, "nothing errored — only the completeness claim is withheld"); + assert.equal(outcome.shortfalls.length, 1); + assert.equal( + outcome.shortfalls[0]?.unprovenBoundary, + true, + "an empty page short of the provider total must never pass for a proven walk" + ); + } finally { + restore(); + } +}); + +test("collectGroupMessages: an empty page whose count AGREES at zero stays a proven walk", async () => { + const restore = stubFetchSequence([ + { body: { response: [group({ id: "group-1", messages_count: 0 })] } }, // /groups + // Provider says zero and serves zero: coherent, an ordinary natural end. + { body: { response: { count: 0, messages: [] } } }, + ]); + try { + const cursor = openFingerprintCursor(new Map()); + const { emitRecord } = makeHarness(); + const outcome = await collectGroupMessages(TOKEN, cursor, undefined, undefined, noopProgress, emitRecord); + + assert.equal(outcome.failed, false); + assert.deepEqual(outcome.shortfalls, [], "a coherent zero is a real anchor, not a gap"); + } finally { + restore(); + } +}); + +test("collectGroupMessages: GroupMe's documented 304 end-of-history is a PROVEN walk, not a shortfall", async () => { + // GroupMe documents: "If no messages are found (e.g. when filtering with + // `before_id`) we return code 304." That is the ordinary, correct way a + // fully-collected group signals it has nothing left — it must NEVER be + // reported as the provider withholding data. + // + // MUTATION GUARD. `fetchMessagesPage` normalizes the 304 into a SYNTHETIC + // `{count: 0, messages: []}`; GroupMe sends no body with a 304, so that + // zero is ours, not the provider's. If it is ever synthesized as non-zero, + // this group — which served its whole history and then said "nothing more" + // — would be accused of a gap it does not have. The `messages_count: 1` + // below is load-bearing: it makes the provider total non-zero, so only the + // synthesized count decides the verdict. + // Page 1 must be FULL (PAGE_SIZE), or the walk exits on the short-page + // natural end and never requests the page that 304s. + const fullPage = Array.from({ length: PAGE_SIZE }, (_, i) => + groupMessage({ id: `m${String(i)}`, created_at: 1_700_000_100 - i }) + ); + // `new Response(..., { status: 304 })` throws (undici forbids constructing a + // null-body status), so the 304 is stubbed as a minimal response-shaped + // object rather than through `stubFetchSequence`. + const original = globalThis.fetch; + const bodies: unknown[] = [ + // The provider total EXCEEDS the page we walked, so only the count + // synthesized for the 304 decides whether this reads as a shortfall. + { response: [group({ id: "group-1", messages_count: PAGE_SIZE + 5 })] }, // /groups + { response: { count: PAGE_SIZE, messages: fullPage } }, // page 1: full + ]; + let call = 0; + globalThis.fetch = ((): Promise => { + const index = call; + call += 1; + const body = bodies[index]; + if (body === undefined) { + // Page 2 and beyond: GroupMe's documented end-of-history signal. + return Promise.resolve({ status: 304, text: () => Promise.resolve(""), headers: new Headers() } as Response); + } + return Promise.resolve(new Response(JSON.stringify(body), { status: 200 })); + }) as typeof globalThis.fetch; + const restore = (): void => { + globalThis.fetch = original; + }; + try { + const cursor = openFingerprintCursor(new Map()); + const { emitRecord } = makeHarness(); + const outcome = await collectGroupMessages(TOKEN, cursor, undefined, undefined, noopProgress, emitRecord); + + assert.equal(outcome.failed, false, "a 304 terminal page is a clean end, not a failure"); + // The provider total is higher than the walk, so a shortfall IS expected. + // What matters is which KIND: the 304 is a boundary GroupMe actually + // served, so it must be a plain `partial`, never an unproven boundary. + assert.equal(outcome.shortfalls.length, 1); + assert.equal( + outcome.shortfalls[0]?.unprovenBoundary, + false, + "GroupMe's documented 304 end-of-history is a PROVEN boundary — reporting it as unproven would accuse a group that served everything it had" + ); + } finally { + restore(); + } +}); + +test("collect(): an ambiguous empty page is reported as unexplained AND retryable, never as proven-unrecoverable", async () => { + const restore = stubFetchSequence([ + { body: { response: [group()] } }, // /groups + // The short-of-total page: provider total is 10, serves none. + { body: { response: { count: 10, messages: [] } } }, + { body: { response: [] } }, // /chats + ]); + try { + const messages: EmittedMessage[] = []; + await collect({ + state: {}, + requested: new Map([["group_messages", { name: "group_messages" }]]), + credentials: { GROUPME_ACCESS_TOKEN: TOKEN }, + emit: (message: EmittedMessage) => { + messages.push(message); + return Promise.resolve(); + }, + emitRecord: async () => { + await Promise.resolve(); + }, + progress: async () => { + await Promise.resolve(); + }, + assist: async () => "", + capture: null, + completeAssistance: async () => { + await Promise.resolve(); + }, + detailGaps: [], + emittedAt: new Date().toISOString(), + requestDetailGapPage: async () => [], + scope: { streams: [{ name: "group_messages" }] }, + sendInteraction: async () => ({}) as never, + } satisfies CollectContext); + + const skips = messages.filter( + (m): m is Extract => + m.type === "SKIP_RESULT" && m.stream === "group_messages" + ); + const ambiguous = skips.find((s) => s.reason === "history_ended_before_provider_count"); + + assert.ok(ambiguous, "the ambiguous gap must be reported under its own reason"); + // The load-bearing assertion. Claiming `not_retriable` here would assert a + // certainty the response cannot support: being throttled produces this + // exact same body, so the honest hint leaves the door open. + const hint = ambiguous.recovery_hint; + assert.ok(typeof hint === "object" && hint !== null, "recovery_hint must be the structured form"); + assert.equal(hint.action, "retry_by_runtime"); + assert.equal(hint.retryable, true); + assert.equal( + skips.some((s) => s.reason === "provider_serves_no_messages_for_group"), + false, + "the retired proven-unrecoverable verdict must not come back" + ); + // Never subtracted: the counted-but-unserved messages stay reported missing. + const diagnostics = ambiguous.diagnostics as { unexplained_message_total?: number } | undefined; + assert.equal(diagnostics?.unexplained_message_total, 10); + } finally { + restore(); + } +}); diff --git a/packages/polyfill-connectors/connectors/heb/__fixtures__/email-first-login-page.html b/packages/polyfill-connectors/connectors/heb/__fixtures__/email-first-login-page.html new file mode 100644 index 000000000..d483a6e67 --- /dev/null +++ b/packages/polyfill-connectors/connectors/heb/__fixtures__/email-first-login-page.html @@ -0,0 +1,141 @@ + + + + +
+ H-E-B logo + + + +
+

Log in

+

+
+
+
+ +
+ +
+ +
+ + +
+
+ Don’t have an account? + Create one +
+ +
+

Copyright © 2026 H‑E‑B, LP

+ + diff --git a/packages/polyfill-connectors/connectors/heb/__fixtures__/login-method-chooser-page.html b/packages/polyfill-connectors/connectors/heb/__fixtures__/login-method-chooser-page.html new file mode 100644 index 000000000..a9df46471 --- /dev/null +++ b/packages/polyfill-connectors/connectors/heb/__fixtures__/login-method-chooser-page.html @@ -0,0 +1,88 @@ + + + + +
+ H-E-B logo +
+
+ + +
+

Choose how you log in

+

+
+ +
+ + +
+

+ I agree to the H‑E‑B + Terms & Conditions + and acknowledge the + Privacy Policy. +

+
+ + +
+
+
+
+

Copyright © 2026 H‑E‑B, LP

+ + diff --git a/packages/polyfill-connectors/connectors/heb/__fixtures__/orders-list-no-past-orders.html b/packages/polyfill-connectors/connectors/heb/__fixtures__/orders-list-no-past-orders.html new file mode 100644 index 000000000..ce14e3d65 --- /dev/null +++ b/packages/polyfill-connectors/connectors/heb/__fixtures__/orders-list-no-past-orders.html @@ -0,0 +1,25 @@ + + + +Your orders | HEB.com + +
+
+

No past orders

Once you place a curbside or delivery order, you’ll find it here.

Start shopping
+
+ + diff --git a/packages/polyfill-connectors/connectors/heb/__fixtures__/passkey-enrollment-page.html b/packages/polyfill-connectors/connectors/heb/__fixtures__/passkey-enrollment-page.html new file mode 100644 index 000000000..c5219100e --- /dev/null +++ b/packages/polyfill-connectors/connectors/heb/__fixtures__/passkey-enrollment-page.html @@ -0,0 +1,30 @@ + + + + +
+

Skip the password

+

You can now use passkeys to log in

+
    +
  • Set up takes seconds
  • +
  • Faster log in on all your devices
  • +
  • Your account gets added protection
  • +
+ + +

Keep using password or passcode for now

+ Learn more about passkeys +
+ + diff --git a/packages/polyfill-connectors/connectors/heb/__fixtures__/whats-new-modal-over-orders.html b/packages/polyfill-connectors/connectors/heb/__fixtures__/whats-new-modal-over-orders.html new file mode 100644 index 000000000..b0b1d941a --- /dev/null +++ b/packages/polyfill-connectors/connectors/heb/__fixtures__/whats-new-modal-over-orders.html @@ -0,0 +1,97 @@ + + + + +
+
+

Your orders

+
+ +
+
+
+ + + +
+
+
+ +
+
+
+ + diff --git a/packages/polyfill-connectors/connectors/heb/index.test.ts b/packages/polyfill-connectors/connectors/heb/index.test.ts index 2acb3e40e..158af17d6 100644 --- a/packages/polyfill-connectors/connectors/heb/index.test.ts +++ b/packages/polyfill-connectors/connectors/heb/index.test.ts @@ -22,12 +22,16 @@ */ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import test from "node:test"; +import { fileURLToPath } from "node:url"; import type { EmittedMessage } from "@pdpp/connector-protocol"; import type { Page } from "playwright"; import type { BrowserCollectContext } from "../../src/connector-runtime.ts"; import { makeRecordingEmit } from "../../src/test-harness.ts"; import { + classifyEmptyListPage, classifyHebDetailFailure, type EmitDeps, emitOrderItemsCoverage, @@ -42,6 +46,7 @@ import { newOrdersCoverage, type OrderItemsCoverage, type OrdersCoverage, + priorOrdersEvidenceFromState, processListOrder, type RepairDeps, type RunFlags, @@ -53,7 +58,9 @@ import { runForwardScan, } from "./index.ts"; import { validateRecord } from "./schemas.ts"; -import type { ListPageOrder } from "./types.ts"; +import type { ListPageDiagnostics, ListPageOrder } from "./types.ts"; + +const FIXTURES_DIR = join(dirname(fileURLToPath(import.meta.url)), "__fixtures__"); type DetailGap = Extract; type DetailGapRecovered = Extract; @@ -1206,6 +1213,239 @@ test("runForwardScan: an empty page whose OWN pagination nav still agrees maxPag ); }); +// ─── Source-authored empty state ────────────────────────────────────────── + +test("runForwardScan: H-E-B's real 'No past orders' page completes the scan instead of aborting as selector_drift", async () => { + // Fail-before/pass-after oracle for the real defect. `orders-list-no-past- + // orders.html` is a live capture (2026-08-21, in-container, connector's own + // authenticated profile): a 272 KB served page titled "Your orders | + // HEB.com" with no Imperva markers, showing H-E-B's own "No past orders" + // empty state. + // + // Without the empty_state branch this page aborts as `selector_drift`, + // because the empty-state component's own CSS-module class names supply all + // four `class*="order"` matches (`order_cards: 0, any_card: 4`). That + // diagnosis blames H-E-B's markup and sends recovery at a selector rewrite + // that cannot succeed. Note this page also has no pagination nav, so it + // would otherwise fall to `pagination_metadata_absent` — the assertion below + // is that it aborts for NEITHER reason. + const ordersCoverage = newOrdersCoverage(); + const { deps, protocolMessages } = makeRecordingDeps({ ordersCoverage, wantsItems: false, wantsOrders: true }); + const html = readFileSync(join(FIXTURES_DIR, "orders-list-no-past-orders.html"), "utf8"); + const page = makePageStub({ content: html, url: "https://www.heb.com/my-account/your-orders?page=1" }); + + const newest = await runForwardScan(page, deps, makeRunFlags(), null); + + assert.equal(newest, null, "an empty history yields no newest order date"); + // The scan must reach a clean terminal, not throw. Before the fix this call + // rejected with heb_empty_list_page_selector_drift. + const skips = protocolMessages.filter((m) => m.type === "SKIP_RESULT"); + assert.deepEqual( + skips.map((m) => (m as { reason: string }).reason), + [], + "a source-reported empty history is proven terminal and must emit no SKIP_RESULT" + ); + + // Completing the scan is what lets coverage be emitted at all: the throwing + // path left the `orders` stream permanently unmeasured. Zero considered / + // zero covered is an honest proven-empty claim here, because H-E-B scopes + // order history to the ACCOUNT, not the selected store (verified against + // stored records: one scrape of a single connection returned orders from + // four different H-E-B stores). + await emitOrdersCoverage(deps, ordersCoverage); + const coverage = findDetailCoverage(protocolMessages); + assert.ok(coverage, "the orders stream must still report coverage on an empty run"); + assert.equal(coverage.considered, 0); + assert.equal(coverage.covered, 0); +}); + +test("runForwardScan: an empty page WITHOUT the empty-state marker still aborts as selector_drift", async () => { + // The fail-closed half. Genuine drift — order cards gone but other + // `class*="order"` elements still present, and no empty-state component to + // vouch for it — must keep aborting exactly as before. This is what stops + // the new branch from becoming a blanket "zero orders is fine" escape. + const { deps } = makeRecordingDeps({ wantsItems: false }); + const html = `
+
+
`; + const page = makePageStub({ content: html, url: "https://www.heb.com/my-account/your-orders?page=1" }); + + await assert.rejects( + () => runForwardScan(page, deps, makeRunFlags(), null), + /heb_empty_list_page_selector_drift/, + "an empty page with no source-authored empty state is still unproven" + ); +}); + +test("runForwardScan: an Imperva block is never laundered into a proven-empty result", async () => { + // Ordering guard. The block check runs before the empty_state check, so a + // challenge page can never be reported as a proven-empty history even if a + // future block shape were to carry empty-state-looking markup. + const { deps } = makeRecordingDeps({ wantsItems: false }); + const blockWithEmptyStateMarkup = `{ "incidentId" : "0-0", "hostName" : "www.heb.com", "errorCode" : "15" }
`; + const page = makePageStub({ + content: blockWithEmptyStateMarkup, + url: "https://www.heb.com/my-account/your-orders?page=1", + }); + + await assert.rejects( + () => runForwardScan(page, deps, makeRunFlags(), null), + /heb_empty_list_page_source_auth_or_challenge/, + "bot protection must outrank the empty-state marker" + ); +}); + +// ─── Proven-empty regression guard (prior-orders evidence) ──────────────── +// +// A connection that has already collected orders must never be able to +// complete a run as "proven empty". The source-authored empty state is +// trustworthy for an account that never had orders; for an account we have +// already measured, the same page is a contradiction, not a result. + +const EMPTY_STATE_DIAG: ListPageDiagnostics = { + any_card: 4, + body_preview: "", + empty_state: true, + incapsula_block: false, + order_cards: 0, + password_form: false, + title: "", + url: "", +}; + +const RESOLVED_MAX_PAGE = { kind: "resolved", source: "dom", value: 1 } as const; + +test("classifyEmptyListPage: source-reported empty on a connection with NO prior orders stays proven-empty", () => { + // Preserves efc601bb7. A first-ever run on a genuinely empty account is the + // one case where zero coverage is an honest measurement. + assert.deepEqual(classifyEmptyListPage(EMPTY_STATE_DIAG, 1, RESOLVED_MAX_PAGE, { hasPriorOrders: false }), { + action: "terminal", + reason: "source_reported_empty", + }); +}); + +test("classifyEmptyListPage: the prior-orders argument defaults to absent, so callers cannot silently opt in", () => { + assert.deepEqual(classifyEmptyListPage(EMPTY_STATE_DIAG, 1, RESOLVED_MAX_PAGE), { + action: "terminal", + reason: "source_reported_empty", + }); +}); + +test("classifyEmptyListPage: source-reported empty on a connection WITH prior orders aborts instead of proving zero", () => { + // The defect this guard closes: without it, this exact input returned + // {action:"terminal", reason:"source_reported_empty"}, letting a connection + // holding 41 orders commit covered:0/considered:0 as a measured result. + assert.deepEqual(classifyEmptyListPage(EMPTY_STATE_DIAG, 1, RESOLVED_MAX_PAGE, { hasPriorOrders: true }), { + action: "abort", + reason: "heb_empty_history_after_prior_orders", + }); +}); + +test("classifyEmptyListPage: an auth/challenge page keeps its own reason even when prior orders exist", () => { + // Ordering guard, upper half. The block check stays ABOVE the new branch: + // when a challenge is actually established, that is the more specific and + // more actionable diagnosis, and it must not be relabelled. + assert.deepEqual( + classifyEmptyListPage({ ...EMPTY_STATE_DIAG, incapsula_block: true }, 1, RESOLVED_MAX_PAGE, { + hasPriorOrders: true, + }), + { action: "abort", reason: "source_auth_or_challenge" } + ); + assert.deepEqual( + classifyEmptyListPage({ ...EMPTY_STATE_DIAG, password_form: true }, 1, RESOLVED_MAX_PAGE, { + hasPriorOrders: true, + }), + { action: "abort", reason: "source_auth_or_challenge" } + ); +}); + +test("classifyEmptyListPage: prior orders do not relabel a page that never claimed to be empty", () => { + // Ordering guard, lower half. The new branch is gated on `empty_state`, so + // real selector drift keeps reporting as drift regardless of prior orders — + // the guard adds a failure mode, it does not swallow the existing ones. + assert.deepEqual( + classifyEmptyListPage({ ...EMPTY_STATE_DIAG, empty_state: false }, 1, RESOLVED_MAX_PAGE, { + hasPriorOrders: true, + }), + { action: "abort", reason: "selector_drift" } + ); +}); + +test("priorOrdersEvidenceFromState: a committed orders checkpoint is what arms the guard", () => { + // Pins the checkpoint-to-evidence link that collect() depends on. Without + // this test, hardcoding `hasPriorOrders: false` in collect() would disarm + // the guard for every connection while every other test still passed. + assert.deepEqual(priorOrdersEvidenceFromState({ checkpoint: "2026-08-17" }), { hasPriorOrders: true }); + // A never-collected connection stores no checkpoint at all. `exactOptional + // PropertyTypes` makes an explicitly-undefined checkpoint unrepresentable, + // so the absent-property case is the only "no prior orders" shape there is. + assert.deepEqual(priorOrdersEvidenceFromState({}), { hasPriorOrders: false }); +}); + +test("runForwardScan: H-E-B's real 'No past orders' page aborts when this connection already collected orders", async () => { + // Integration half, through the same live 2026-08-21 capture the + // proven-empty test uses. Same page, same markup, opposite verdict — the + // only difference is that this connection has a prior orders checkpoint. + const ordersCoverage = newOrdersCoverage(); + const { deps, protocolMessages } = makeRecordingDeps({ ordersCoverage, wantsItems: false, wantsOrders: true }); + const html = readFileSync(join(FIXTURES_DIR, "orders-list-no-past-orders.html"), "utf8"); + const page = makePageStub({ content: html, url: "https://www.heb.com/my-account/your-orders?page=1" }); + + await assert.rejects( + () => runForwardScan(page, deps, makeRunFlags(), "2026-08-01", { hasPriorOrders: true }), + /heb_empty_list_page_heb_empty_history_after_prior_orders/, + "an account with prior orders cannot be proven empty by a page render" + ); + + // The failure must be legible to the owner, and must not blame anything the + // page does not establish. + const skip = protocolMessages.find( + (m) => m.type === "SKIP_RESULT" && (m as { reason: string }).reason === "heb_empty_history_after_prior_orders" + ) as { message: string; diagnostics: Record } | undefined; + assert.ok(skip, "the abort must surface a SKIP_RESULT the owner can read"); + assert.match(skip.message, /previously collected orders/); + assert.match(skip.message, /retained and untouched/); + assert.doesNotMatch(skip.message, /selector|drift/i, "selector drift is not established and must not be blamed"); + assert.doesNotMatch(skip.message, /block|bot|captcha/i, "a bot block is not established and must not be blamed"); + assert.equal(skip.diagnostics.empty_state, true); + assert.equal(skip.diagnostics.has_prior_orders, true); + + // Nothing may be recorded as covered, and no proven-zero coverage may be + // claimed: the scan threw before any coverage accounting ran. + assert.equal(ordersCoverage.considered.length, 0); + assert.equal(ordersCoverage.covered.length, 0); + assert.equal(findDetailCoverage(protocolMessages), undefined, "an aborted run must claim no coverage at all"); +}); + +test("runForwardScan: the empty-history abort emits no records and no STATE cursor", async () => { + // Requirement: the guard must not delete, tombstone, or overwrite stored + // records, and must not advance or clear the orders cursor. The connector's + // only durable writes are protocol messages, so proving it emitted no + // RECORD and no STATE proves the stored copy and the prior checkpoint are + // both untouched. + const { deps, emitted, protocolMessages } = makeRecordingDeps({ wantsItems: false, wantsOrders: true }); + const html = readFileSync(join(FIXTURES_DIR, "orders-list-no-past-orders.html"), "utf8"); + const page = makePageStub({ content: html, url: "https://www.heb.com/my-account/your-orders?page=1" }); + + await assert.rejects(() => runForwardScan(page, deps, makeRunFlags(), "2026-08-01", { hasPriorOrders: true })); + + assert.equal(emitted.length, 0, "no records may be written on a contradicted-empty run"); + assert.equal( + protocolMessages.filter((m) => m.type === "STATE").length, + 0, + "the orders cursor must not be advanced or cleared" + ); + // Deletion needs no assertion: the connector protocol has no delete or + // tombstone message, so a connector cannot remove a stored record even in + // principle. The only durable writes available here are RECORD and STATE, + // and both are asserted absent above. + assert.deepEqual( + [...new Set(protocolMessages.map((m) => m.type))].sort(), + ["SKIP_RESULT"], + "the abort's only durable output is the owner-facing SKIP_RESULT" + ); +}); + // ─── #3: old-gap recovery lane ───────────────────────────────────────────── function makeGap(orderId: string, orderDate = "2026-07-01"): BrowserCollectContext["detailGaps"][number] { diff --git a/packages/polyfill-connectors/connectors/heb/index.ts b/packages/polyfill-connectors/connectors/heb/index.ts index 89109acad..c3856b62c 100755 --- a/packages/polyfill-connectors/connectors/heb/index.ts +++ b/packages/polyfill-connectors/connectors/heb/index.ts @@ -35,6 +35,7 @@ import { runConnector, } from "../../src/connector-runtime.ts"; import { type FingerprintCursor, openFingerprintCursor } from "../../src/fingerprint-cursor.ts"; +import { type OrderItemTally, summarizeItemCounts } from "./item-count-anchor.ts"; import { buildOrderItemRecord, buildOrderRecord, @@ -296,6 +297,29 @@ interface EmptyListPageClassification { reason: string; } +/** + * What this connection already knows about its own order history, threaded + * explicitly into the otherwise-pure empty-page classifier. + * + * `hasPriorOrders` is true when a prior run committed an `orders` checkpoint — + * the in-connector proof that H-E-B once listed orders for this account. It is + * durable evidence about the CONNECTION, which no single page render can + * retract. + */ +export interface PriorOrdersEvidence { + hasPriorOrders: boolean; +} + +/** Owner-facing message for the one classification whose whole point is to be + * read by a person. Says exactly what was observed and what was NOT concluded: + * neither selector drift nor a bot block is established, so neither is named. + * Stored records are untouched — this connector never deletes, tombstones, or + * overwrites on an empty page; it only declines to advance. */ +export const HEB_EMPTY_AFTER_PRIOR_ORDERS_MESSAGE = + "H-E-B reported no order history, but PDPP previously collected orders for this account. " + + "Your stored orders are retained and untouched. This run was stopped instead of recording " + + "an empty history, because a page showing no orders cannot prove the history is gone."; + /** * Classify a zero-order list page: distinguish a genuine end-of-list from * selector drift, an auth/challenge block, or missing/contradictory @@ -316,11 +340,54 @@ interface EmptyListPageClassification { export function classifyEmptyListPage( diag: ListPageDiagnostics, pageNum: number, - maxPageResolution: MaxPageResolution + maxPageResolution: MaxPageResolution, + priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false } ): EmptyListPageClassification { if (diag.incapsula_block || diag.password_form) { return { action: "abort", reason: "source_auth_or_challenge" }; } + // A connection that has already collected orders can never prove itself + // empty. `hasPriorOrders` is this connection's own prior `orders` checkpoint + // — durable evidence that H-E-B previously listed orders for this account — + // threaded in explicitly by `collect()` rather than read from ambient state, + // so this branch stays pure and unit-testable. + // + // Without this check, a connection holding 41 orders could complete a run as + // "succeeded, considered:0, covered:0, checkpoint committed", replacing a + // measured coverage claim with a fabricated proven-zero. The two causes are + // indistinguishable from the page alone — H-E-B may have purged the history + // upstream (making our stored copy the only copy), or the page may render + // empty for a degraded session — so the run fails loudly and lets a human + // decide, rather than guessing. + // + // Ordering: BELOW the block/auth check (an established block is the more + // specific diagnosis and keeps its own reason), and ABOVE the empty_state + // branch, so a source-authored empty state cannot short-circuit past it. + if (diag.empty_state && priorOrdersEvidence.hasPriorOrders) { + return { action: "abort", reason: "heb_empty_history_after_prior_orders" }; + } + // H-E-B's own empty-state component, rendered inside the order-results + // container, is the source asserting the history is empty. Trust it as + // terminal proof: it is positive evidence, unlike every check below, which + // can only infer emptiness from things being absent. + // + // Ordering is load-bearing in both directions. It must stay BELOW the + // block/auth check, so a challenge page can never be laundered into a proven + // empty result. It must stay ABOVE the `selector_drift` check, because a + // genuinely empty page trips that check: the empty-state component's own + // CSS-module class names match `[class*="order" i]`, producing + // `order_cards: 0, any_card: 4` — the drift signature. Before this branch + // existed, every zero-order run aborted as `selector_drift`, which reads as + // "H-E-B changed their markup" and sends recovery at a selector rewrite that + // could never succeed, because the markup is fine and the history is empty. + // + // Terminal here means "stop paginating, and count this as proven-empty + // coverage" — honest because order history is account-wide (verified: a + // single scrape of one connection returned orders from four different H-E-B + // stores, so the selected store context does not scope what is listed). + if (diag.empty_state) { + return { action: "terminal", reason: "source_reported_empty" }; + } if (diag.order_cards === 0 && diag.any_card > 0) { return { action: "abort", reason: "selector_drift" }; } @@ -339,12 +406,13 @@ export function classifyEmptyListPage( async function reportEmptyPageDiagnostics( page: Page, pageNum: number, - emit: BrowserCollectContext["emit"] + emit: BrowserCollectContext["emit"], + priorOrdersEvidence: PriorOrdersEvidence ): Promise { const html = await page.content().catch((): string => ""); const diag = diagnoseEmptyListPage(html, page.url()); const maxPageResolution = resolveMaxPage(html); - const classification = classifyEmptyListPage(diag, pageNum, maxPageResolution); + const classification = classifyEmptyListPage(diag, pageNum, maxPageResolution, priorOrdersEvidence); if (classification.action === "terminal") { return classification; } @@ -352,10 +420,15 @@ async function reportEmptyPageDiagnostics( type: "SKIP_RESULT", stream: "orders", reason: classification.reason, - message: `H-E-B list page ${pageNum}: empty page is not a proven terminal page (${classification.reason}).`, + message: + classification.reason === "heb_empty_history_after_prior_orders" + ? HEB_EMPTY_AFTER_PRIOR_ORDERS_MESSAGE + : `H-E-B list page ${pageNum}: empty page is not a proven terminal page (${classification.reason}).`, diagnostics: { any_card: diag.any_card, body_preview: "", + empty_state: diag.empty_state, + has_prior_orders: priorOrdersEvidence.hasPriorOrders, incapsula_block: diag.incapsula_block, max_page_resolution: maxPageResolution, order_cards: diag.order_cards, @@ -462,6 +535,11 @@ export interface EmitDeps extends HydrationDeps { emit: BrowserCollectContext["emit"]; emitRecord: BrowserCollectContext["emitRecord"]; emittedAt: string; + /** Per-order declared-vs-collected item counts, accumulated across the run + * and rolled up into the `order_items` completeness anchor. Optional so + * existing callers and tests that do not exercise the anchor need no + * change. */ + itemCountTallies?: OrderItemTally[] | undefined; orderItemsCoverage: OrderItemsCoverage | undefined; ordersCoverage: OrdersCoverage | undefined; ordersFingerprintCursor: FingerprintCursor | undefined; @@ -746,6 +824,18 @@ async function emitOrderAndItems( buildOrderItemRecord(listOrder.orderId, orderDate, item, itemIndex, deps.emittedAt) ); } + // Completeness anchor: H-E-B's own list card declared how many items + // this order has. Recording the pair here — declared (list page) vs + // collected (detail page) — lets the run compare two independent source + // surfaces instead of trusting the detail page alone. Only orders whose + // detail actually hydrated are tallied; a gapped order is already + // reported as a DETAIL_GAP and must not also be counted as an item + // shortfall. + deps.itemCountTallies?.push({ + orderId: listOrder.orderId, + declaredItemCount: listOrder.itemCount, + collectedItemCount: detail.items.length, + }); } } @@ -829,7 +919,8 @@ export async function runForwardScan( page: Page, deps: EmitDeps, flags: RunFlags, - boundary: string | null + boundary: string | null, + priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false } ): Promise { let newestOrderDate: string | null = null; let pageNum = 1; @@ -839,7 +930,7 @@ export async function runForwardScan( // list/item records and two DETAIL_GAPs for one logical order (S5). const seenOrderIds = new Set(); while (pageNum <= MAX_LIST_PAGES) { - const listPage = await loadListPage(page, pageNum, deps.emit, deps.waitForHydration); + const listPage = await loadListPage(page, pageNum, deps.emit, priorOrdersEvidence, deps.waitForHydration); if (listPage === "terminal") { break; } @@ -907,6 +998,7 @@ async function loadListPage( page: Page, pageNum: number, emit: BrowserCollectContext["emit"], + priorOrdersEvidence: PriorOrdersEvidence, waitForHydration?: () => Promise ): Promise { const url = `https://www.heb.com/my-account/your-orders?page=${pageNum}`; @@ -950,7 +1042,7 @@ async function loadListPage( } return { maxPage: maxPageResolution.value, orders }; } - const classification = await reportEmptyPageDiagnostics(page, pageNum, emit); + const classification = await reportEmptyPageDiagnostics(page, pageNum, emit, priorOrdersEvidence); if (classification.action === "terminal") { return "terminal"; } @@ -1014,6 +1106,22 @@ interface OrdersStateShape { fingerprints?: Record; } +/** + * Derive the prior-orders evidence from this connection's stored `orders` + * state. Exported and pure because `collect()` lives inside the + * `isMainModule` block and cannot be driven from a test — without this seam + * the checkpoint-to-evidence link would be the one untested link in the + * chain, and a mutation that hardcodes `false` here (silently disarming the + * guard for every connection) would go unnoticed. + * + * Any committed checkpoint counts, including one recorded by a run that + * emitted no new records: the checkpoint's existence is the claim that H-E-B + * once listed orders for this account. + */ +export function priorOrdersEvidenceFromState(ordersState: { checkpoint?: string }): PriorOrdersEvidence { + return { hasPriorOrders: Boolean(ordersState.checkpoint) }; +} + /** * H-E-B's order list is globally reverse-chronological (not year-partitioned * like Amazon). Given the prior run's checkpoint date, compute the resume @@ -1080,6 +1188,12 @@ if (isMainModule(import.meta.url)) { const ordersState = (state.orders ?? {}) as OrdersStateShape; const boundary = resumeBoundary(ordersState.checkpoint); + // A committed `orders` checkpoint is this connection's own record that + // H-E-B has listed orders for this account before. It is what makes a + // later "no order history" page a contradiction to escalate rather than + // a result to trust. Read here, next to the checkpoint it derives from, + // and passed down explicitly. + const priorOrdersEvidence = priorOrdersEvidenceFromState(ordersState); const ordersFingerprintCursor = wantsOrders ? openFingerprintCursor(state.orders, { excludeFromFingerprint: ["fetched_at"] }) @@ -1088,6 +1202,9 @@ if (isMainModule(import.meta.url)) { // `orders` list-stream coverage is only meaningful when `orders` itself // is in scope — mirrors the `wantsItems`-gated accumulator above. const ordersCoverage = wantsOrders ? newOrdersCoverage() : undefined; + // Declared-vs-collected item tallies for the `order_items` anchor. + // Only meaningful when items are in scope. + const itemCountTallies: OrderItemTally[] | undefined = wantsItems ? [] : undefined; const flags: RunFlags = { detailAttempts: 0, @@ -1100,6 +1217,7 @@ if (isMainModule(import.meta.url)) { emit, emitRecord, emittedAt, + itemCountTallies, orderItemsCoverage, ordersCoverage, ordersFingerprintCursor, @@ -1134,7 +1252,7 @@ if (isMainModule(import.meta.url)) { await progress("H-E-B session verified; scanning order history"); - const newestOrderDate = await runForwardScan(page, deps, flags, boundary); + const newestOrderDate = await runForwardScan(page, deps, flags, boundary, priorOrdersEvidence); if (wantsOrders) { const cursor = buildOrdersStateCursor(newestOrderDate, ordersState, ordersFingerprintCursor); @@ -1144,6 +1262,31 @@ if (isMainModule(import.meta.url)) { if (orderItemsCoverage) { await emitOrderItemsCoverage(deps, orderItemsCoverage); } + // The `order_items` completeness anchor: every hydrated order's item + // count as H-E-B declared it on the list card, against what the detail + // page actually yielded. Reported only when the provider's own numbers + // say something is missing — a run where every order reconciles needs + // no notice, and an order with no declared count is silently + // unanchored rather than falsely clean. + if (itemCountTallies && itemCountTallies.length > 0) { + const summary = summarizeItemCounts(itemCountTallies); + if (summary.short > 0) { + await emit({ + type: "SKIP_RESULT", + stream: "order_items", + reason: "item_count_short", + message: "Some orders hold fewer items than H-E-B says they contain", + diagnostics: { + short_orders: summary.short, + complete_orders: summary.complete, + unanchored_orders: summary.unavailable, + declared_items: summary.declaredItems, + collected_items: summary.collectedItems, + short_order_ids: summary.shortOrderIds, + }, + }); + } + } // Same honesty posture as order_items: emit once the forward scan // completes, including the zero-considered steady-state case, so the // `orders` list stream is never left permanently unmeasured. diff --git a/packages/polyfill-connectors/connectors/heb/item-count-anchor.test.ts b/packages/polyfill-connectors/connectors/heb/item-count-anchor.test.ts new file mode 100644 index 000000000..80de61071 --- /dev/null +++ b/packages/polyfill-connectors/connectors/heb/item-count-anchor.test.ts @@ -0,0 +1,143 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Completeness-anchor tests for the H-E-B `order_items` stream. + * + * The anchor is H-E-B's own declared item count, printed on each order card + * ("$382.67 · 85 items") and already stored on the `orders` record as + * `item_count`. It was recorded and never checked. + * + * The live numbers below are real: one instance holds two orders declaring + * 59 and 85 items but only 35 and 54 `order_items` records — 89 of 144. + * Nothing reported a problem because nothing compared the two. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + MAX_SHORT_ORDER_IDS_IN_DIAGNOSTIC, + type OrderItemTally, + summarizeItemCounts, + tallyOrderItems, + validateDeclaredItemCount, +} from "./item-count-anchor.ts"; + +// ─── validateDeclaredItemCount: fail closed, never fabricate ───────────── + +test("validateDeclaredItemCount accepts a genuine count", () => { + assert.equal(validateDeclaredItemCount(85), 85); +}); + +test("validateDeclaredItemCount accepts a genuine zero from the provider", () => { + // The provider SAYING zero is a fact; it is not the same as saying nothing. + assert.equal(validateDeclaredItemCount(0), 0); +}); + +test("validateDeclaredItemCount refuses a missing count rather than calling it zero", () => { + // A zero denominator would make every order trivially complete. + assert.equal(validateDeclaredItemCount(null), null); + assert.equal(validateDeclaredItemCount(undefined), null); +}); + +test("validateDeclaredItemCount refuses malformed counts", () => { + assert.equal(validateDeclaredItemCount(-1), null); + assert.equal(validateDeclaredItemCount(1.5), null); + assert.equal(validateDeclaredItemCount(Number.NaN), null); + assert.equal(validateDeclaredItemCount(Number.POSITIVE_INFINITY), null); + assert.equal(validateDeclaredItemCount("85"), null); +}); + +// ─── tallyOrderItems ───────────────────────────────────────────────────── + +test("tallyOrderItems confirms a fully-collected order", () => { + const verdict = tallyOrderItems({ orderId: "o1", declaredItemCount: 12, collectedItemCount: 12 }); + assert.equal(verdict.status, "complete"); + assert.equal(verdict.status === "complete" && verdict.covered, 12); +}); + +test("tallyOrderItems catches the real live shortfall", () => { + // HEB20607368035: declared 85, collected 54. + const verdict = tallyOrderItems({ orderId: "HEB20607368035", declaredItemCount: 85, collectedItemCount: 54 }); + assert.equal(verdict.status, "short"); + assert.equal(verdict.status === "short" && verdict.missing, 31); + assert.equal(verdict.status === "short" && verdict.considered, 85); + assert.equal(verdict.status === "short" && verdict.covered, 54); +}); + +test("tallyOrderItems catches the second real live shortfall", () => { + // HEB20169324473: declared 59, collected 35. + const verdict = tallyOrderItems({ orderId: "HEB20169324473", declaredItemCount: 59, collectedItemCount: 35 }); + assert.equal(verdict.status === "short" && verdict.missing, 24); +}); + +test("tallyOrderItems is deletion-safe: MORE held than declared is not a gap", () => { + // H-E-B restates an order's count downward after a refund. PDPP preserves + // the items it already captured, so holding more is correct behaviour — + // flagging it would report preservation as loss. + const verdict = tallyOrderItems({ orderId: "o1", declaredItemCount: 10, collectedItemCount: 14 }); + assert.equal(verdict.status, "complete"); + assert.equal(verdict.status === "complete" && verdict.covered, 10); +}); + +test("tallyOrderItems reports unavailable when the provider declared nothing", () => { + const verdict = tallyOrderItems({ orderId: "o1", declaredItemCount: null, collectedItemCount: 7 }); + assert.equal(verdict.status, "unavailable"); +}); + +test("tallyOrderItems honours a declared zero as a real anchor", () => { + const verdict = tallyOrderItems({ orderId: "o1", declaredItemCount: 0, collectedItemCount: 0 }); + assert.equal(verdict.status, "complete"); +}); + +// ─── summarizeItemCounts ───────────────────────────────────────────────── + +const LIVE_TALLIES: OrderItemTally[] = [ + { orderId: "HEB20169324473", declaredItemCount: 59, collectedItemCount: 35 }, + { orderId: "HEB20607368035", declaredItemCount: 85, collectedItemCount: 54 }, +]; + +test("summarizeItemCounts reproduces the live 89-of-144 shortfall", () => { + const summary = summarizeItemCounts(LIVE_TALLIES); + assert.equal(summary.short, 2); + assert.equal(summary.complete, 0); + assert.equal(summary.declaredItems, 144); + assert.equal(summary.collectedItems, 89); + assert.deepEqual(summary.shortOrderIds, ["HEB20169324473", "HEB20607368035"]); +}); + +test("summarizeItemCounts reports a fully-reconciled run cleanly", () => { + const summary = summarizeItemCounts([{ orderId: "o1", declaredItemCount: 5, collectedItemCount: 5 }]); + assert.equal(summary.short, 0); + assert.equal(summary.complete, 1); + assert.deepEqual(summary.shortOrderIds, []); +}); + +test("summarizeItemCounts excludes unanchored orders from both totals", () => { + // Counting an order with no declared count would treat its collected + // items as if they had been verified against something. + const summary = summarizeItemCounts([ + { orderId: "o1", declaredItemCount: null, collectedItemCount: 9 }, + { orderId: "o2", declaredItemCount: 4, collectedItemCount: 4 }, + ]); + assert.equal(summary.unavailable, 1); + assert.equal(summary.declaredItems, 4); + assert.equal(summary.collectedItems, 4); +}); + +test("summarizeItemCounts bounds the id sample but keeps the count exact", () => { + const many: OrderItemTally[] = Array.from({ length: MAX_SHORT_ORDER_IDS_IN_DIAGNOSTIC + 20 }, (_, i) => ({ + orderId: `o${i}`, + declaredItemCount: 2, + collectedItemCount: 1, + })); + const summary = summarizeItemCounts(many); + assert.equal(summary.short, MAX_SHORT_ORDER_IDS_IN_DIAGNOSTIC + 20, "the COUNT must stay exact"); + assert.equal(summary.shortOrderIds.length, MAX_SHORT_ORDER_IDS_IN_DIAGNOSTIC, "only the id sample is bounded"); +}); + +test("summarizeItemCounts handles a run with no orders", () => { + const summary = summarizeItemCounts([]); + assert.equal(summary.short, 0); + assert.equal(summary.declaredItems, 0); +}); diff --git a/packages/polyfill-connectors/connectors/heb/item-count-anchor.ts b/packages/polyfill-connectors/connectors/heb/item-count-anchor.ts new file mode 100644 index 000000000..fe38d0842 --- /dev/null +++ b/packages/polyfill-connectors/connectors/heb/item-count-anchor.ts @@ -0,0 +1,160 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// The `order_items` completeness anchor for H-E-B. +// +// WHY THIS IS A REAL ANCHOR +// ------------------------- +// Every H-E-B order card on the list page states its own item count — the +// "$123.45 · 59 items" line the `LIST_TOTAL_COUNT_RE` parser already reads +// into `ListPageOrder.itemCount` and stores on the `orders` record as +// `item_count`. That number is computed by H-E-B, not by this connector, and +// it is read from the LIST page while the items themselves come from a +// separate DETAIL page. So comparing them compares two independent source +// surfaces rather than checking the connector's output against itself. +// +// Until now `item_count` was recorded and never checked. Live evidence for +// why that matters: one instance holds two orders declaring 59 and 85 items, +// but only 35 and 54 `order_items` records — 89 of 144. Both orders are +// partially hydrated, and nothing anywhere reported a problem, because +// nothing compared the two numbers. +// +// DELETION-SAFE, ONE-DIRECTIONAL +// ------------------------------ +// Only "the provider says MORE than we hold" is a gap. Holding more items +// than the current card states is not loss: H-E-B restates an order's count +// when items are refunded or removed after fulfilment, and PDPP deliberately +// preserves the items it already captured. A two-way equality would flag +// that correct preservation as a defect — the same trap the fleet's other +// anchors avoid. +// +// REFUSES RATHER THAN FABRICATES +// ------------------------------ +// A missing or malformed `item_count` yields `unavailable`, never zero. A +// zero denominator would make every order trivially "complete" and would be +// a denominator invented from the very data it is meant to verify. + +/** One order's declared count against what was actually collected. */ +export interface OrderItemTally { + /** `order_items` records collected for this order this run. */ + collectedItemCount: number; + /** The count H-E-B printed on the list card, or null when it did not. */ + declaredItemCount: number | null; + orderId: string; +} + +export type OrderItemVerdict = + /** No sound anchor: the provider stated no usable count. */ + | { status: "unavailable"; orderId: string; reason: "no_declared_count" } + /** Everything the provider declared is accounted for. */ + | { status: "complete"; orderId: string; considered: number; covered: number } + /** The provider declared more items than were collected. */ + | { status: "short"; orderId: string; considered: number; covered: number; missing: number }; + +/** + * Validate a provider-declared item count. + * + * Fails closed to `null` — a malformed count is NOT zero and NOT complete. + * Mirrors jellyfin's `validateTotalRecordCount` discipline, minus its + * monotonicity rule: H-E-B legitimately restates an order's count downward + * after a refund, so a decrease is ordinary source behaviour here rather + * than the anomaly it is for a Jellyfin library. + */ +export function validateDeclaredItemCount(value: unknown): number | null { + if (typeof value !== "number") { + return null; + } + if (!Number.isFinite(value)) { + return null; + } + if (!Number.isInteger(value)) { + return null; + } + if (value < 0) { + return null; + } + return value; +} + +/** + * Compare one order's declared count against what was collected. + * + * A declared zero is a real fact (an order genuinely holding no line items), + * so it is honoured rather than treated as "unknown" — this is the same + * distinction the fleet draws elsewhere between the provider SAYING zero and + * the provider saying nothing. + */ +export function tallyOrderItems(tally: OrderItemTally): OrderItemVerdict { + const declared = validateDeclaredItemCount(tally.declaredItemCount); + if (declared === null) { + return { status: "unavailable", orderId: tally.orderId, reason: "no_declared_count" }; + } + const covered = Math.min(tally.collectedItemCount, declared); + if (covered < declared) { + return { + status: "short", + orderId: tally.orderId, + considered: declared, + covered, + missing: declared - covered, + }; + } + return { status: "complete", orderId: tally.orderId, considered: declared, covered }; +} + +/** The run-level roll-up of every per-order verdict. */ +export interface ItemCountAnchorSummary { + /** Total items collected across those same orders. */ + collectedItems: number; + /** Orders whose declared count was fully accounted for. */ + complete: number; + /** Total items the provider declared across anchorable orders. */ + declaredItems: number; + /** Orders holding fewer items than the provider declared. */ + short: number; + /** Order ids that came up short, for a bounded diagnostic. */ + shortOrderIds: string[]; + /** Orders offering no sound anchor. */ + unavailable: number; +} + +/** Cap on ids listed in the diagnostic. The COUNT is always exact; only the + * id sample is bounded, so a large shortfall stays legible without an + * unbounded diagnostic. Mirrors signal's and slack's id-sample caps. */ +export const MAX_SHORT_ORDER_IDS_IN_DIAGNOSTIC = 50; + +/** + * Roll per-order verdicts into one run-level summary. + * + * `unavailable` orders contribute to neither total: including an order with + * no declared count would silently treat its collected items as if they had + * been verified against something. + */ +export function summarizeItemCounts(tallies: readonly OrderItemTally[]): ItemCountAnchorSummary { + const summary: ItemCountAnchorSummary = { + complete: 0, + short: 0, + unavailable: 0, + declaredItems: 0, + collectedItems: 0, + shortOrderIds: [], + }; + for (const tally of tallies) { + const verdict = tallyOrderItems(tally); + if (verdict.status === "unavailable") { + summary.unavailable += 1; + continue; + } + summary.declaredItems += verdict.considered; + summary.collectedItems += verdict.covered; + if (verdict.status === "short") { + summary.short += 1; + if (summary.shortOrderIds.length < MAX_SHORT_ORDER_IDS_IN_DIAGNOSTIC) { + summary.shortOrderIds.push(verdict.orderId); + } + } else { + summary.complete += 1; + } + } + return summary; +} diff --git a/packages/polyfill-connectors/connectors/heb/parsers.test.ts b/packages/polyfill-connectors/connectors/heb/parsers.test.ts index d9487d801..3904914f7 100644 --- a/packages/polyfill-connectors/connectors/heb/parsers.test.ts +++ b/packages/polyfill-connectors/connectors/heb/parsers.test.ts @@ -23,6 +23,7 @@ import { buildOrderItemRecord, buildOrderRecord, diagnoseEmptyListPage, + hasOrdersEmptyState, isIncapsulaBlocked, looksLoggedOut, mergeOrdersListPage, @@ -609,6 +610,35 @@ test("isIncapsulaBlocked is false for a normal populated page", () => { assert.equal(isIncapsulaBlocked(fixture("orders-list.html")), false); }); +// Captured live from /my-account/order-history on 2026-08-20, served with +// HTTP 200. The iframe-only heuristic returned false here, so the block fell +// through to the zero-order branch and was reported as `selector_drift` — +// sending every retry at a selector rewrite when the real fault was bot +// protection. Verbatim shape (ids scrubbed); the run that exposed it saw +// any_card=4, order_cards=0. +const IMPERVA_INCIDENT_PAGE = `{ "incidentId" : "000000000000000000-000000000000000000", "hostName" : "www.heb.com", "errorCode" : "15", "description" : "This page could not load. It looks like you may be using a web browser version that we do not support." }`; + +test("isIncapsulaBlocked detects Imperva's iframe-free JSON incident report served with HTTP 200", () => { + assert.equal(isIncapsulaBlocked(IMPERVA_INCIDENT_PAGE), true); +}); + +// `classifyEmptyListPage` (connectors/heb/index.ts) branches on this flag +// BEFORE it reaches the selector-drift check, so proving the flag here is what +// makes the block win over `selector_drift`. That ordering is asserted in +// index.test.ts, which owns the classifier. +test("an Imperva incident report sets the block flag the classifier branches on first", () => { + const diag = diagnoseEmptyListPage(IMPERVA_INCIDENT_PAGE, "https://www.heb.com/my-account/order-history"); + assert.equal(diag.incapsula_block, true); + assert.equal(diag.order_cards, 0); +}); + +test("isIncapsulaBlocked does not fire on a real page that merely mentions incidentId", () => { + // H-E-B's own API error envelopes use `errorCode`; only Imperva's report + // carries the incidentId+hostName pair, and a real page is never this small. + const realPage = `

Order history

${"x".repeat(5000)}"errorCode" : "15"
`; + assert.equal(isIncapsulaBlocked(realPage), false); +}); + test("isIncapsulaBlocked is false for a legitimate empty terminal page (has h3/breadcrumb/testid)", () => { assert.equal(isIncapsulaBlocked(fixture("orders-list-empty.html")), false); }); @@ -618,6 +648,62 @@ test("isIncapsulaBlocked is false for an iframe-free shallow page (no false posi assert.equal(isIncapsulaBlocked(html), false); }); +// ─── Source-authored empty state ────────────────────────────────────────── +// `orders-list-no-past-orders.html` is a real capture, taken in-container on +// 2026-08-21 with the connector's own authenticated profile. The full page was +// 272 KB, titled "Your orders | HEB.com", carried zero Imperva markers, and +// rendered the signed-in header — it is a served page, not a block. + +test("hasOrdersEmptyState is true for H-E-B's real 'No past orders' page", () => { + assert.equal(hasOrdersEmptyState(fixture("orders-list-no-past-orders.html")), true); +}); + +test("hasOrdersEmptyState is false for a populated order list", () => { + assert.equal(hasOrdersEmptyState(fixture("orders-list.html")), false); + assert.equal(hasOrdersEmptyState(fixture("orders-list-nextdata.html")), false); +}); + +test("hasOrdersEmptyState is false for an Incapsula block page", () => { + // A block must never be mistaken for a source-authored empty result; the + // classifier's ordering depends on this staying false. + assert.equal(hasOrdersEmptyState(fixture("incapsula-block.html")), false); +}); + +test("hasOrdersEmptyState requires the marker to be INSIDE the order-results container", () => { + // The container is present on populated pages too, so nesting is the whole + // discriminator. An empty-state component elsewhere on a page whose results + // region holds real orders must not read as "no orders". + const emptyStateOutsideResults = ` +

No past orders

+ + `; + assert.equal(hasOrdersEmptyState(emptyStateOutsideResults), false); +}); + +test("hasOrdersEmptyState is false when the order-results container is absent entirely", () => { + // A page that never rendered the results region proves nothing about the + // history; only the container can speak for it. + assert.equal(hasOrdersEmptyState(`
`), false); +}); + +test("diagnoseEmptyListPage reports the real empty page as empty_state with the drift signature", () => { + // This is the exact confusion the empty_state flag exists to resolve: all + // four `class*="order"` matches come from the empty-state component's OWN + // class names, so counting alone reads `order_cards: 0, any_card: 4` — + // indistinguishable from selector drift. + const diag = diagnoseEmptyListPage( + fixture("orders-list-no-past-orders.html"), + "https://www.heb.com/my-account/your-orders?page=1" + ); + assert.equal(diag.empty_state, true); + assert.equal(diag.order_cards, 0); + assert.equal(diag.any_card, 4, "the empty state's own class names are what made this look like drift"); + assert.equal(diag.incapsula_block, false, "the captured page was served, not blocked"); + assert.equal(diag.password_form, false); +}); + // ─── Session probe (deep check) ────────────────────────────────────────── test("looksLoggedOut is true for a sign-in URL", () => { @@ -716,3 +802,29 @@ test("buildOrderItemRecord maps a parsed detail item into the emitted order_item assert.equal(record.line_total_cents, 429); assert.equal(record.order_date, "2026-07-14"); }); + +// ─── Promotional interstitial vs the empty state (run_1787343993082) ────── +// The promo overlay that covered a live orders page arrives on the SAME route +// as the source-authored empty state and, like it, leaves no order cards +// legible. These pin the two apart in both directions, because mistaking either +// for the other would either fabricate an empty history for an account that has +// orders or suppress a genuine "no past orders" claim behind a marketing popup. +// +// They cannot collide by construction, not merely by ordering: the empty-state +// marker must sit INSIDE `[data-qe-id="orderResults"]`, whereas the modal is +// portaled to , outside the Next.js root entirely. + +test("the promotional interstitial is not read as a source-authored empty state", () => { + assert.equal(hasOrdersEmptyState(fixture("whats-new-modal-over-orders.html")), false); +}); + +test("the promotional interstitial is not read as an Incapsula block", () => { + assert.equal(isIncapsulaBlocked(fixture("whats-new-modal-over-orders.html")), false); +}); + +test("the source-authored empty state is still detected with an overlay-capable page shape", () => { + // Guards the reverse direction: teaching the connector about modals must not + // make a genuinely empty history stop reporting itself as empty. + assert.equal(hasOrdersEmptyState(fixture("orders-list-no-past-orders.html")), true); + assert.equal(isIncapsulaBlocked(fixture("orders-list-no-past-orders.html")), false); +}); diff --git a/packages/polyfill-connectors/connectors/heb/parsers.ts b/packages/polyfill-connectors/connectors/heb/parsers.ts index e566d8ef6..e19fc5fb2 100644 --- a/packages/polyfill-connectors/connectors/heb/parsers.ts +++ b/packages/polyfill-connectors/connectors/heb/parsers.ts @@ -661,8 +661,39 @@ export function parseOrderDetailDom(html: string): OrderDetail | null { // block/challenge renders as an empty shell — no h3, no breadcrumb nav, no // [data-testid], document.body.children.length <= 2, and at least one iframe. // `_Incapsula_Resource` alone (present on every page) must NOT count as a block. +// +// Imperva also serves a SECOND, iframe-less block shape: a ~650-byte document +// whose entire body is a JSON incident report — `{"incidentId": "...", +// "hostName": "www.heb.com", "errorCode": "15", "description": "This page +// could not load..."}` — returned with HTTP **200**, not 403. Observed live +// against /my-account/order-history on 2026-08-20. +// +// That shape defeated the iframe-only detector: `hasIframe` is false, so a +// block fell through to the zero-order classification and was reported as +// `selector_drift` ("4 cards on the page, 0 matching") — a markup-drift +// diagnosis for what is really bot protection. The recovery for those two is +// opposite (rewrite a selector vs. back off and re-establish trust), so the +// misclassification sent every retry down the wrong path. +// +// Matched structurally on the incident-report field set rather than on the +// prose, which is localized and has changed before. `errorCode` alone is too +// generic to key on: H-E-B's own API errors use that name too, so require the +// `incidentId` + `hostName` pair that only Imperva's report carries. +const IMPERVA_INCIDENT_ID_RE = /"incidentId"\s*:/; +const IMPERVA_HOST_NAME_RE = /"hostName"\s*:/; +const IMPERVA_INCIDENT_MAX_BYTES = 4000; + +function isImpervaIncidentReport(html: string): boolean { + if (html.length > IMPERVA_INCIDENT_MAX_BYTES) { + return false; + } + return IMPERVA_INCIDENT_ID_RE.test(html) && IMPERVA_HOST_NAME_RE.test(html); +} export function isIncapsulaBlocked(html: string): boolean { + if (isImpervaIncidentReport(html)) { + return true; + } const { document } = parseHTML(html); const { body } = document; if (!body) { @@ -692,6 +723,43 @@ export function looksLoggedOut(landedUrl: string, html: string): boolean { // ─── Empty-list-page diagnostics ────────────────────────────────────────── +// ─── Source-authored empty state ────────────────────────────────────────── +// When the account has no orders in H-E-B's retention window, the order-results +// container renders a dedicated empty-state component instead of order cards: +// +//
+// … +//

No past orders

+// +// This is H-E-B telling us the history is empty, not us inferring it from the +// absence of something. Detect the component, not the prose: the copy ("No past +// orders") is user-facing and localizable, whereas `OrderEmpty_`/`Empty_box` are +// CSS-module class prefixes on the component itself. +// +// Require the marker to sit INSIDE `[data-qe-id="orderResults"]`. That container +// is present on populated pages too, so nesting is what separates "the results +// region says it is empty" from an empty-state component reused elsewhere on a +// page that does have orders — a bare document-wide search for `Empty_box` would +// conflate the two. +// +// This matters because the empty state is otherwise indistinguishable from +// selector drift by counting alone: the component's own class names +// (OrderHistoryPage_headerContainer, OrderHistoryPage_resultsContainer, +// OrderHistoryPage_messagingContainer, OrderEmpty_orderSvg) all match +// `[class*="order" i]`, so a genuinely empty page reports `any_card: 4, +// order_cards: 0` — exactly the signature the classifier reads as drift. +const ORDER_RESULTS_SELECTOR = '[data-qe-id="orderResults"]'; +const EMPTY_STATE_SELECTOR = '[class*="OrderEmpty_"], [class*="Empty_box"]'; + +export function hasOrdersEmptyState(html: string): boolean { + const { document } = parseHTML(html); + const results = document.querySelector(ORDER_RESULTS_SELECTOR); + if (!results) { + return false; + } + return Boolean(results.querySelector(EMPTY_STATE_SELECTOR)); +} + export function diagnoseEmptyListPage(html: string, url: string): ListPageDiagnostics { const { document } = parseHTML(html); return { @@ -701,6 +769,7 @@ export function diagnoseEmptyListPage(html: string, url: string): ListPageDiagno any_card: document.querySelectorAll('[class*="order" i]').length, password_form: Boolean(document.querySelector('input[type="password"]')), incapsula_block: isIncapsulaBlocked(html), + empty_state: hasOrdersEmptyState(html), body_preview: normText(document.body).slice(0, 240), }; } @@ -711,6 +780,7 @@ export function redactHebListPageDiagnostics(diag: ListPageDiagnostics): ListPag return { any_card: diag.any_card, body_preview: "", + empty_state: diag.empty_state, incapsula_block: diag.incapsula_block, order_cards: diag.order_cards, password_form: diag.password_form, diff --git a/packages/polyfill-connectors/connectors/heb/types.ts b/packages/polyfill-connectors/connectors/heb/types.ts index 41f04c42d..32d0b6315 100644 --- a/packages/polyfill-connectors/connectors/heb/types.ts +++ b/packages/polyfill-connectors/connectors/heb/types.ts @@ -91,6 +91,11 @@ export interface OrderItemRecord { export interface ListPageDiagnostics { any_card: number; body_preview: string; + /** True when the order-results container renders H-E-B's own "no past + * orders" empty-state component. A source-authored assertion that the + * account's order history is empty, which is different from — and must + * outrank — the structural guesswork the other fields support. */ + empty_state: boolean; incapsula_block: boolean; order_cards: number; password_form: boolean; diff --git a/packages/polyfill-connectors/connectors/jellyfin/manifest-honesty.test.ts b/packages/polyfill-connectors/connectors/jellyfin/manifest-honesty.test.ts index 4ea63ca79..29bf94eef 100644 --- a/packages/polyfill-connectors/connectors/jellyfin/manifest-honesty.test.ts +++ b/packages/polyfill-connectors/connectors/jellyfin/manifest-honesty.test.ts @@ -17,8 +17,21 @@ const manifest = JSON.parse( ) as Manifest; test("Jellyfin remains Preview until live version capability is proven", () => { + // `public_listing.tier` is what actually withholds Jellyfin from + // auto-enrollment and from the dashboard catalog, so it is the assertion + // that carries this test's intent. + // + // This test used to ALSO pin `recommended_mode: "manual"` and + // `background_safe: false`. Those encoded "not proven yet" as a claim + // about background SAFETY, which is a different fact: Jellyfin is a + // self-hosted API-key connector with no interactive login, so there is + // nothing about it that makes unattended refresh unsafe. Maturity belongs + // to the tier; capability belongs to the refresh policy. Mode is now + // derived from capability (reference-implementation/runtime/ + // refresh-mode-derivation.ts), so pinning it here would re-introduce the + // contradiction the derivation exists to prevent. assert.equal(manifest.capabilities.public_listing.tier, "preview"); - assert.equal(manifest.capabilities.refresh_policy.recommended_mode, "manual"); - assert.equal(manifest.capabilities.refresh_policy.background_safe, false); - assert.match(manifest.capabilities.refresh_policy.rationale, /version compatibility.*unproven/i); + assert.match(manifest.capabilities.refresh_policy.rationale, /unproven/i); + // The unproven-ness must still be stated in owner-readable terms. + assert.match(manifest.capabilities.refresh_policy.rationale, /version and credentialed-deployment/i); }); diff --git a/packages/polyfill-connectors/connectors/notion/index.ts b/packages/polyfill-connectors/connectors/notion/index.ts index 10346147b..4339bcf2e 100755 --- a/packages/polyfill-connectors/connectors/notion/index.ts +++ b/packages/polyfill-connectors/connectors/notion/index.ts @@ -16,7 +16,7 @@ import type { EmittedMessage } from "@pdpp/connector-protocol"; import { createConnectorHttpGovernor } from "../../src/connector-http-governor.ts"; -import { buildFullScanCoverageMessage, politeDelay, runConnector } from "../../src/connector-runtime.ts"; +import { buildDetailCoverageMessage, politeDelay, runConnector } from "../../src/connector-runtime.ts"; import { notionPacingProfile } from "../../src/provider-profile.ts"; import { validateRecord } from "./schemas.ts"; @@ -267,11 +267,25 @@ async function runStream(args: RunStreamArgs): Promise { const streamState = state[streamName] as { last_edited_time?: string } | undefined; const prior = streamState?.last_edited_time; let latest = prior; + // Rows this run objectively accounted for — emitted-and-valid, or correctly + // suppressed as unchanged. NEVER aliased to `items.length`: the schema is + // strict (title ≤ 4000 chars, url ≤ 4096, safe-text), so a real page CAN be + // rejected, and a rejected page must not be claimed as covered just because + // it was enumerated. Tallied from the same `validateRecord` verdict the + // runtime's emitRecord applies. + let covered = 0; for (const item of items) { if (prior && item.last_edited_time && item.last_edited_time <= prior) { + // Suppressed-unchanged: an earlier run's identical content already passed + // the real emitRecord shape-check, so this is genuinely accounted for. + covered += 1; continue; } - await emitRecord(streamName, toRecord(item)); + const record = toRecord(item); + if (validateRecord(streamName, record).ok) { + covered += 1; + } + await emitRecord(streamName, record); if (item.last_edited_time && (!latest || item.last_edited_time > latest)) { latest = item.last_edited_time; } @@ -286,7 +300,30 @@ async function runStream(args: RunStreamArgs): Promise { // The search result is the complete enumeration boundary for this stream. // Emit measured full-scan evidence even when it is empty; record counts alone // cannot distinguish a verified empty workspace from missing coverage proof. - await emit(buildFullScanCoverageMessage(streamName, items.length)); + // + // `considered` is that boundary (`items.length`, measured at the enumeration + // site); `covered` is the independently tallied count above, so a page the + // schema rejected reads a real `partial` instead of being silently absorbed. + // `buildFullScanCoverageMessage` is deliberately NOT used: it forces + // `covered === considered`, which cannot express a dropped page. + // + // Deletion-safe by construction: Notion reports a deleted page IN-BAND as + // `archived: true` rather than omitting it, so a deletion stays inside the + // boundary, is validated like any other row, and counts as covered. The + // runtime turns it into a tombstone (`isTombstone`, below). An upstream + // deletion is therefore a covered fact, never a coverage gap — which is what + // keeps this proof from firing on PDPP's deliberate retention of records the + // provider has since removed. + await emit( + buildDetailCoverageMessage({ + stream: streamName, + stateStream: streamName, + requiredKeys: [], + hydratedKeys: [], + considered: items.length, + covered, + }) + ); await emit({ type: "STATE", stream: streamName, diff --git a/packages/polyfill-connectors/connectors/notion/rejected-page-coverage.test.ts b/packages/polyfill-connectors/connectors/notion/rejected-page-coverage.test.ts new file mode 100644 index 000000000..574ce1529 --- /dev/null +++ b/packages/polyfill-connectors/connectors/notion/rejected-page-coverage.test.ts @@ -0,0 +1,143 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * A page the schema rejects is considered-but-NOT-covered. + * + * `runStream` previously declared coverage with `buildFullScanCoverageMessage`, + * which forces `covered === considered`. That shape cannot express a dropped + * page: the enumeration boundary absorbed the rejected row and the stream read + * `complete` while a real page was silently missing. Notion's schema is strict + * (title ≤ 4000 chars, url ≤ 4096, safe text), so rejection is genuinely + * reachable, not theoretical. + * + * `covered` is now tallied per record from the same `validateRecord` verdict + * the runtime's emitRecord applies, so a rejected page reads a real `partial`. + * + * The companion property — an ARCHIVED (deleted) page must still count as + * covered — is asserted here too. Notion reports deletions in-band as + * `archived: true`, and PDPP deliberately retains records the provider has + * removed, so treating a deletion as a gap would make correct preservation look + * like loss on every subsequent run. + */ + +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import type { EmittedMessage } from "../../src/connector-runtime.ts"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; + +const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "notion", "index.ts"); +const GOOD_PAGE_ID = "11111111-1111-4111-8111-111111111111"; +const BAD_PAGE_ID = "44444444-4444-4444-8444-444444444444"; +const ARCHIVED_PAGE_ID = "55555555-5555-4555-8555-555555555555"; +const ACTOR_ID = "33333333-3333-4333-8333-333333333333"; + +/** Runs the real connector against a bounded in-process Notion stub. */ +async function runWithPages(pagesJson: string, state: Record = {}): Promise { + const harnessDir = await mkdtemp(join(tmpdir(), "pdpp-notion-rejected-")); + const wrapperPath = join(harnessDir, "notion-wrapper.mjs"); + await writeFile( + wrapperPath, + ` +globalThis.fetch = async (_input, init) => { + const body = JSON.parse(init?.body ?? "{}"); + const isPage = body.filter?.value === "page"; + return new Response(JSON.stringify({ + has_more: false, + next_cursor: null, + results: isPage ? ${pagesJson} : [] + }), { status: 200, headers: { "content-type": "application/json" } }); +}; +await import(${JSON.stringify(pathToFileURL(ENTRYPOINT).href)}); +`, + "utf8" + ); + try { + const result = await runConnectorProtocolSubprocess({ + cwd: PACKAGE_ROOT, + entrypoint: wrapperPath, + env: { NOTION_API_TOKEN: "bounded-test-token" }, + start: { scope: { streams: [{ name: "pages" }] }, state, type: "START" }, + }); + return result.messages; + } finally { + await rm(harnessDir, { force: true, recursive: true }); + } +} + +function pagesCoverage(messages: readonly EmittedMessage[]): { considered?: number; covered?: number } { + const coverage = messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "pages"); + assert.ok(coverage, "expected a pages DETAIL_COVERAGE"); + return coverage as { considered?: number; covered?: number }; +} + +/** A well-formed page. `titleText` is the only variable of interest. */ +function page(id: string, titleText: string, archived = false): string { + return JSON.stringify({ + id, + object: "page", + parent: { type: "workspace", workspace: true }, + properties: { Name: { type: "title", title: [{ plain_text: titleText }] } }, + url: `https://www.notion.so/${id.replace(/-/g, "")}`, + archived, + created_time: "2026-08-12T00:00:00.000Z", + last_edited_time: "2026-08-12T01:00:00.000Z", + created_by: { id: ACTOR_ID }, + last_edited_by: { id: ACTOR_ID }, + }); +} + +test("a page rejected by the schema is considered but not covered", async () => { + // The second page's title exceeds the schema's 4000-char bound, so + // `validateRecord` rejects it and it can never be committed. + const messages = await runWithPages(`[${page(GOOD_PAGE_ID, "Fine")}, ${page(BAD_PAGE_ID, "x".repeat(4001))}]`); + + const coverage = pagesCoverage(messages); + assert.equal(coverage.considered, 2, "both enumerated pages are in the denominator"); + // The load-bearing assertion. Under `buildFullScanCoverageMessage` this was 2. + assert.equal(coverage.covered, 1, "a page the schema rejected must not be claimed as covered"); +}); + +test("an archived page still counts as covered, so deletion is not read as loss", async () => { + // Notion reports a deleted page in-band as `archived: true`. It is a valid + // record and the runtime tombstones it; it must NOT depress coverage, or + // every upstream deletion would make a correct run look incomplete forever. + const messages = await runWithPages(`[${page(GOOD_PAGE_ID, "Fine")}, ${page(ARCHIVED_PAGE_ID, "Gone", true)}]`); + + const coverage = pagesCoverage(messages); + assert.equal(coverage.considered, 2); + assert.equal(coverage.covered, 2, "an archived (deleted) page is a covered fact, not a coverage gap"); +}); + +test("a clean enumeration still reads fully covered", async () => { + // Guards the opposite failure: a covered-tally that under-counts would make + // every healthy run read a false `partial`. + const messages = await runWithPages(`[${page(GOOD_PAGE_ID, "Fine")}, ${page(BAD_PAGE_ID, "Also fine")}]`); + + const coverage = pagesCoverage(messages); + assert.equal(coverage.considered, 2); + assert.equal(coverage.covered, 2); +}); + +test("a steady-state run whose cursor suppresses every page still reads fully covered", async () => { + // The incremental path: a cursor newer than every page means nothing is + // emitted. Those pages were still enumerated and accounted for — an earlier + // run's identical content already passed the real shape-check — so they must + // count as covered. If suppressed-unchanged rows were dropped from `covered`, + // every healthy steady-state re-scan would report a false `partial`. + const messages = await runWithPages(`[${page(GOOD_PAGE_ID, "Fine")}, ${page(BAD_PAGE_ID, "Also fine")}]`, { + pages: { last_edited_time: "2030-01-01T00:00:00.000Z" }, + }); + + const emitted = messages.filter((m) => m.type === "RECORD" && m.stream === "pages").length; + assert.equal(emitted, 0, "the cursor should suppress every page"); + + const coverage = pagesCoverage(messages); + assert.equal(coverage.considered, 2, "suppressed pages are still enumerated"); + assert.equal(coverage.covered, 2, "suppressed-unchanged pages are accounted for, not lost"); +}); diff --git a/packages/polyfill-connectors/connectors/reddit/index.ts b/packages/polyfill-connectors/connectors/reddit/index.ts index 659019c43..a49553176 100755 --- a/packages/polyfill-connectors/connectors/reddit/index.ts +++ b/packages/polyfill-connectors/connectors/reddit/index.ts @@ -25,9 +25,14 @@ * connector over the public API — they capture preference signal * (upvoted/downvoted history) no third party can see. * - * Pagination: opaque `after` cursor, newest-first. Incremental sync stops - * once we cross the earliest `created_utc` from the prior run — same - * pattern the original API-based connector used. + * Pagination: opaque `after` cursor. The stop rule is per-stream, because + * Reddit does not order every listing the same way: + * submitted/comments — ordered by the item's own `created_utc`, so an + * incremental run stops once it crosses the prior run's high-water mark. + * saved/upvoted/downvoted/hidden — ordered by OWNER ACTION time. An old + * post upvoted today sits at rank 1 with an old `created_utc`, so the + * created-based stop is invalid here: these walk the full listing and + * dedupe by fullname. See `RedditListingOrder` in parsers.ts. * * Rate limit: Reddit's logged-in web JSON allows ~100 req/min before 429. * We page at limit=100 and use a conservative 500ms politeDelay between @@ -50,7 +55,12 @@ import { isMainModule } from "@pdpp/connector-protocol"; import type { Page } from "playwright"; -import { ensureRedditSession, isSessionLive } from "../../src/auto-login/reddit.ts"; +import { + ensureRedditJsonOrigin, + ensureRedditSession, + isSessionLive, + REDDIT_JSON_ORIGIN, +} from "../../src/auto-login/reddit.ts"; import { type BrowserCollectContext, buildDetailCoverageMessage, @@ -67,10 +77,12 @@ import { appendNewChildren, classifyListingStatus, commentRecord, + dedupeByFullname, MAX_PAGES, maxCreatedEpoch, nextAfter, pagePath, + type RedditListingOrder, savedRecord, sinceFromState, submittedRecord, @@ -152,11 +164,30 @@ interface ProgressExtra { // ─── Fetch through the page (preserves session cookie + anti-bot) ─────── +/** + * Every listing fetch is issued from the page, and Reddit grants NO + * cross-origin access to its listing JSON — so the page must already be on + * {@link REDDIT_JSON_ORIGIN} or the browser blocks the request before it + * reaches the network, surfacing as `TypeError: Failed to fetch` (mapped to + * `status: 0` below, then to `reddit_http_0`). + * + * `ensureSession` normally leaves the page on the right origin, but collect + * runs after an arbitrary amount of navigation and the reauth path can move it + * again, so this does not assume — it re-establishes the origin on the first + * fetch and then no-ops (a URL check, no navigation) for every page after it. + * This is the same defect that broke the liveness probe in + * `run_1787164349370`; see `src/auto-login/reddit.ts`'s REDDIT_JSON_ORIGIN. + */ async function redditFetch(page: Page, path: string): Promise { + if (!(await ensureRedditJsonOrigin(page))) { + // Reported as a transport-shaped failure so the existing retry + // classification handles it, rather than a bare throw from collect. + return { status: 0, json: { error: "reddit_json_origin_unavailable" } as never }; + } return (await page.evaluate( - async ({ path: evalPath, userAgent }) => { + async ({ origin, path: evalPath, userAgent }) => { try { - const res = await fetch(`https://old.reddit.com${evalPath}`, { + const res = await fetch(`${origin}${evalPath}`, { credentials: "include", headers: { accept: "application/json", @@ -175,7 +206,7 @@ async function redditFetch(page: Page, path: string): Promise return { status: 0, json: { error: String(err) } }; } }, - { path, userAgent: USER_AGENT } + { origin: REDDIT_JSON_ORIGIN, path, userAgent: USER_AGENT } )) as RedditFetchResult; } @@ -289,7 +320,8 @@ export async function paginate( progress?: (message: string, extra?: ProgressExtra) => Promise, streamName?: string, onAuthFailed?: RedditReauthFn, - repairBudget: ReturnType = createRepairBudget() + repairBudget: ReturnType = createRepairBudget(), + order: RedditListingOrder = "created" ): Promise { const all: RedditChild[] = []; let after: string | null = null; @@ -335,7 +367,7 @@ export async function paginate( if (children.length === 0) { break; } - if (appendNewChildren(children, sinceEpochUtc, all)) { + if (appendNewChildren(children, sinceEpochUtc, all, order)) { break; } @@ -346,7 +378,13 @@ export async function paginate( await delay(PAGE_DELAY_MS); } - return all; + // `action`-ordered streams walk the whole listing every run, so the same + // item recurs across runs and (rarely) within one walk when Reddit shifts + // items between pages mid-walk. Dedupe by fullname before the caller counts + // `considered`/`covered`, so coverage reflects distinct items rather than + // repeat sightings. `created`-ordered streams stop at the cursor and are + // already distinct, so this is a no-op for them. + return order === "action" ? dedupeByFullname(all) : all; } // ─── Stream runner ────────────────────────────────────────────────────── @@ -357,6 +395,10 @@ export async function paginate( export interface RedditStreamConfig { endpoint: string; name: string; + /** How Reddit sorts this listing. Drives the pagination stop rule — see + * {@link RedditListingOrder}. Omitted means `created` (authorship + * timeline), the only ordering for which an early stop is sound. */ + order?: RedditListingOrder; progressMessage: string; toRecord: (c: RedditChild) => RecordData; } @@ -401,7 +443,8 @@ export async function collectStream(args: CollectStreamArgs): Promise commentRecord(c.data, emittedAt), }, + // The four streams below are ordered by OWNER ACTION time, not by the + // item's `created_utc` — acting on an old item puts an old `created_utc` + // at the top of the listing. They must walk the full listing and dedupe; + // see `RedditListingOrder`. { name: "saved", endpoint: `${userPath}/saved.json`, + order: "action", progressMessage: "Fetching saved items", toRecord: (c) => savedRecord(c, emittedAt), }, { name: "upvoted", endpoint: `${userPath}/upvoted.json`, + order: "action", progressMessage: "Fetching upvoted items", toRecord: (c) => voteRecord(c, emittedAt), }, { name: "downvoted", endpoint: `${userPath}/downvoted.json`, + order: "action", progressMessage: "Fetching downvoted items", toRecord: (c) => voteRecord(c, emittedAt), }, { name: "hidden", endpoint: `${userPath}/hidden.json`, + order: "action", progressMessage: "Fetching hidden items", toRecord: (c) => voteRecord(c, emittedAt), }, @@ -479,8 +530,12 @@ export function buildStreamTable(userPath: string, emittedAt: string): RedditStr } /** Build a RedditListingFetch bound to a Playwright page. Extracted - * so tests can substitute a non-browser fetch. */ -function makePageFetch(page: Page): RedditListingFetch { + * so tests can substitute a non-browser fetch. + * + * Exported so `redditFetch`'s origin guard is directly testable: reaching it + * only through `collectAllStreams` means every listing/parsing fixture has to + * model navigation, which would bury the one behavior under test. */ +export function makePageFetch(page: Page): RedditListingFetch { return (path) => redditFetch(page, path); } @@ -595,12 +650,18 @@ export async function collectAllStreams(ctx: BrowserCollectContext): Promise { - await ensureRedditSession({ capture, context, onCredentialSubmit, page, sendInteraction }); + // Forwarding `checkpoint` is the point of production run_1787109028586's + // fix: without it the watchdog's no-progress message could only name the + // runtime's own `session-establish:begin`, so a 120s stall inside the + // first liveness probe was indistinguishable from a stall anywhere else in + // session establishment. + await ensureRedditSession({ capture, checkpoint, context, onCredentialSubmit, page, sendInteraction }); } if (isMainModule(import.meta.url)) { diff --git a/packages/polyfill-connectors/connectors/reddit/integration.test.ts b/packages/polyfill-connectors/connectors/reddit/integration.test.ts index 05370b82e..faa55ed4e 100644 --- a/packages/polyfill-connectors/connectors/reddit/integration.test.ts +++ b/packages/polyfill-connectors/connectors/reddit/integration.test.ts @@ -26,6 +26,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import type { Page } from "playwright"; +import { REDDIT_JSON_ORIGIN } from "../../src/auto-login/reddit.ts"; import type { BrowserCollectContext } from "../../src/connector-runtime.ts"; import { createRepairBudget } from "../../src/repair-budget.ts"; import { makeRecordingEmit } from "../../src/test-harness.ts"; @@ -33,6 +34,7 @@ import { buildStreamTable, collectAllStreams, collectStream, + makePageFetch, makeReauth, normalizeRedditTerminalError, paginate, @@ -325,17 +327,67 @@ for (const stream of buildStreamTable(USER_PATH, EMITTED_AT)) { delay: NO_DELAY, }); - assert.deepEqual( - secondHarness.emitted.map((record) => record.data.id), - [stream.name === "comments" ? `t1_${stream.name}new` : `t3_${stream.name}new`], - `${stream.name}: a restart must not re-emit the cursor boundary` - ); + const prefix = stream.name === "comments" ? "t1_" : "t3_"; + const emittedIds = secondHarness.emitted.map((record) => record.data.id); + if (stream.order === "action") { + // Action-ordered listings (saved/upvoted/downvoted/hidden) are sorted by + // when the OWNER acted, not by created_utc, so an item below the cursor + // says nothing about what follows it. These streams must walk the whole + // listing and re-see the boundary item; suppressing it is exactly the + // defect that froze `upvoted` at a 2026-04-28 cursor while real history + // ran back to 2011. Re-emitting is safe — records are keyed by fullname. + assert.deepEqual( + emittedIds, + [`${prefix}${stream.name}new`, `${prefix}${stream.name}old`], + `${stream.name}: an action-ordered restart must walk past the cursor boundary` + ); + } else { + assert.deepEqual( + emittedIds, + [`${prefix}${stream.name}new`], + `${stream.name}: a created-ordered restart must not re-emit the cursor boundary` + ); + } const secondState = secondHarness.protocolMessages.find((message) => message.type === "STATE"); assert.ok(secondState && secondState.type === "STATE"); assert.deepEqual(secondState.cursor, { last_created_utc: 300 }); }); } +// A restart on an action-ordered stream must recover history BELOW the stored +// cursor — the real-world shape of the defect, where `upvoted`'s cursor sat at +// a 2026 timestamp and every older upvote had become unreachable. +test("collectStream: action-ordered restart recovers items far below the cursor", async () => { + const stream = buildStreamTable(USER_PATH, EMITTED_AT).find((s) => s.name === "upvoted"); + assert.ok(stream); + const harness = makeRecordingEmit(validateRecord); + const { fetch, calls } = makeScriptedFetch({ + [stream.endpoint]: [ + // Rank 1 is a 2011-era post upvoted moments ago, far below the cursor. + okResult(listing([makePost("t3_upvotedancient", 100)], "t3_upvotedancient")), + okResult(listing([makePost("t3_upvotedolder", 90)], null)), + ], + }); + + await collectStream({ + stream, + fetchPath: fetch, + state: { upvoted: { last_created_utc: 1_777_366_297 } }, + emit: harness.emit, + emitRecord: harness.emitRecord, + progress: async () => undefined, + capture: null, + delay: NO_DELAY, + }); + + assert.equal(calls.length, 2, "must page past an old item instead of halting on it"); + assert.deepEqual( + harness.emitted.map((r) => r.data.id), + ["t3_upvotedancient", "t3_upvotedolder"], + "history below the cursor must be recovered, not skipped" + ); +}); + // ─── Invariant 4: multi-page pagination threads the 'after' cursor ────── test("paginate: follows 'after' through multiple pages until exhausted", async () => { @@ -822,41 +874,46 @@ test("collectAllStreams: credential-less env gate stops the FIRST requested stre // login count is exactly 1 regardless of how many streams 401. /** A fake Playwright Page that (a) always reports a live session to - * `isSessionLive`'s `goto` + `locator(...).count()` probe — so every reauth - * attempt genuinely succeeds, never masked by a scripted-sequence - * exhaustion — while counting each `goto` call (the real navigation - * `isSessionLive` performs) as one login-repair attempt, and (b) answers - * `page.evaluate(fetch, ...)` — the real production shape `makePageFetch` - * builds — by routing to a scripted `RedditListingFetch`. One object plays - * both roles because production `collectAllStreams` drives both `fetchPath` - * (via `makePageFetch(page)`) and `makeReauth(ctx)` (via - * `isSessionLive(ctx.page)`) off the SAME `ctx.page`. Counting real `goto` - * navigations (rather than a bounded scripted-answer sequence) is what - * makes this discriminate a per-stream-budget regression: with a real - * budget shared across the run, `goto` fires at most twice total (the two - * `isSessionLive` probes inside ONE repair); a regressed per-call budget - * would let every one of the 6 streams attempt its own repair, each firing - * two more `goto` calls, so the count would climb unboundedly instead of + * `isSessionLive`'s `/saved.json` JSON probe — so every reauth attempt + * genuinely succeeds, never masked by a scripted-sequence exhaustion — + * while counting each such probe call as one login-repair attempt, and + * (b) answers `page.evaluate(fetch, ...)` for listing pages — the real + * production shape `makePageFetch` builds — by routing to a scripted + * `RedditListingFetch`. One object plays both roles because production + * `collectAllStreams` drives both `fetchPath` (via `makePageFetch(page)`) + * and `makeReauth(ctx)` (via `isSessionLive(ctx.page)`) off the SAME + * `ctx.page`, and both go through `page.evaluate`. Counting real + * liveness-probe calls (rather than a bounded scripted-answer sequence) is + * what makes this discriminate a per-stream-budget regression: with a real + * budget shared across the run, the probe fires at most twice total (the + * two `isSessionLive` calls inside ONE repair — `ensureRedditSession`'s own + * fast-path check, then `makeReauth`'s follow-up); a regressed per-call + * budget would let every one of the 6 streams attempt its own repair, each + * firing two more probes, so the count would climb unboundedly instead of * capping at 2 — a scripted-sequence approach would instead just run out * and silently report "not live" for the extra attempts, hiding the defect. */ -function makeReauthCapablePage(fetch: RedditListingFetch): { gotoCalls: number; page: Page } { - const state = { gotoCalls: 0 }; +function makeReauthCapablePage(fetch: RedditListingFetch): { probeCalls: number; page: Page } { + const state = { probeCalls: 0 }; const page = { evaluate: (_fn: unknown, args: unknown): Promise => { const { path } = args as { path: string }; + if (path === `${USER_PATH}/saved.json`) { + state.probeCalls += 1; + return Promise.resolve({ status: 200 }); // isSessionLive's liveness probe: always live + } return fetch(path); }, - goto: () => { - state.gotoCalls += 1; - return Promise.resolve(null); - }, + goto: () => Promise.resolve(null), locator: () => ({ - count: async () => 1, // always reports the logout link present: session reads live + count: async () => 1, // credential-less fallback path only; unused once REDDIT_USERNAME is set }), + // Already on the JSON origin, as a real run is by collect time — the + // origin guard in `redditFetch`/`isSessionLive` then no-ops. + url: () => `${REDDIT_JSON_ORIGIN}/`, } as any; return { - get gotoCalls() { - return state.gotoCalls; + get probeCalls() { + return state.probeCalls; }, page, }; @@ -925,15 +982,14 @@ test("collectAllStreams: 6 credentialed streams each 401 on their first page — // regression by reading "not live" for extra attempts) — every reauth // attempted, whether 1 or 6, would genuinely succeed if attempted. The // discriminating signal is therefore how many times a repair is - // attempted at all, measured by counting `page.goto` calls: `isSessionLive` - // navigates once per probe, and one successful repair costs exactly two - // navigations (ensureRedditSession's own fast-path probe, then - // makeReauth's follow-up probe). A run-scoped budget spends this ONCE for - // the whole run: 2 navigations total, no matter how many of the 6 - // streams 401. A regressed per-call/per-stream budget would let every - // 401'ing stream attempt its own repair, each costing 2 more - // navigations — the count would grow with stream count instead of - // staying flat at 2. + // attempted at all, measured by counting `isSessionLive`'s `/saved.json` + // probe calls: one successful repair costs exactly two probes + // (ensureRedditSession's own fast-path check, then makeReauth's + // follow-up check). A run-scoped budget spends this ONCE for the whole + // run: 2 probes total, no matter how many of the 6 streams 401. A + // regressed per-call/per-stream budget would let every 401'ing stream + // attempt its own repair, each costing 2 more probes — the count would + // grow with stream count instead of staying flat at 2. const pageHandle = makeReauthCapablePage(fetch); const { page } = pageHandle; const ctx = createCredentialedMockBrowserContext( @@ -954,9 +1010,9 @@ test("collectAllStreams: 6 credentialed streams each 401 on their first page — ); assert.equal( - pageHandle.gotoCalls, + pageHandle.probeCalls, 2, - "exactly one repair's worth of session-live navigation (2 goto calls) for the WHOLE run, regardless of " + + "exactly one repair's worth of session-live probing (2 /saved.json calls) for the WHOLE run, regardless of " + "how many of the 6 streams 401 — a per-stream/per-call budget would let every 401'ing stream repair " + "independently and this count would climb with stream count instead of staying at 2" ); @@ -1268,13 +1324,24 @@ test("collectStream: schema-invalid item not counted in covered, still emitted f // ─── Invariant 11: real collectAllStreams emits DETAIL_COVERAGE ─────────── -/** Create a mock page that redirects evaluate calls to a scripted fetch */ +/** + * Create a mock page that redirects evaluate calls to a scripted fetch. + * + * `url()` reports the JSON origin because that is where a real run's page + * already sits by the time collect runs: `redditFetch`'s origin guard is a + * URL check that no-ops in that state. Modeling it keeps these listing/parsing + * oracles about listings and parsing. The guard's own behavior — including a + * page on the WRONG origin — is proven separately below and in + * `src/auto-login/reddit.test.ts`, not accidentally by every test here. + */ function createMockPageForFetch(fetch: RedditListingFetch) { return { evaluate: (_fn: (args: unknown) => Promise, args: unknown): Promise => { const { path } = args as { path: string }; return fetch(path); }, + goto: (): Promise => Promise.resolve(null), + url: (): string => `${REDDIT_JSON_ORIGIN}/`, }; } @@ -1402,3 +1469,52 @@ test("collectAllStreams: one valid + one invalid child emits DETAIL_COVERAGE wit assert.equal(harness.emitted.length, 1, "runtime emits only valid record"); assert.equal(harness.skipped.length, 1, "runtime SKIP_RESULT logs invalid record"); }); + +// ─── Invariant 12: the collect path's in-page fetch is same-origin ──────── +// +// `redditFetch` has the SAME cross-origin defect the liveness probe had +// (production `run_1787164349370`): Reddit sends no +// `Access-Control-Allow-Origin`, so a credentialed fetch issued from a page on +// the wrong origin is blocked by the browser before it reaches the network and +// surfaces as `TypeError: Failed to fetch` -> `status: 0` -> `reddit_http_0`. +// A live session would fail every listing with an opaque HTTP error. + +/** A page that models the browser's CORS rule: the in-page fetch only succeeds + * when the page itself is already on the JSON origin. */ +function makeCorsAwarePage(startUrl: string): { gotoCalls: () => number; page: Page } { + let url = startUrl; + let gotoCalls = 0; + const page = { + evaluate: (_fn: unknown, args: unknown): Promise => { + const { origin } = args as { origin: string }; + if (new URL(url).origin !== origin) { + // Blocked before the network — exactly what the real callback catches. + return Promise.resolve({ status: 0, json: { error: "TypeError: Failed to fetch" } }); + } + return Promise.resolve({ status: 200, json: listing([], null) }); + }, + goto: (target: string): Promise => { + gotoCalls += 1; + url = target; + return Promise.resolve(null); + }, + url: (): string => url, + } as unknown as Page; + return { gotoCalls: () => gotoCalls, page }; +} + +test("redditFetch establishes the JSON origin, so a listing fetched from a www.reddit.com page succeeds instead of failing CORS", async () => { + const { gotoCalls, page } = makeCorsAwarePage("https://www.reddit.com/"); + const result = await makePageFetch(page)(`${USER_PATH}/saved.json`); + + assert.equal(result.status, 200, "the collect fetch must not be blocked by the page's origin"); + assert.equal(gotoCalls(), 1, "the wrong origin must be corrected exactly once"); +}); + +test("redditFetch does not re-navigate when the page is already on the JSON origin (COUNTERWEIGHT)", async () => { + const { gotoCalls, page } = makeCorsAwarePage(`${REDDIT_JSON_ORIGIN}/`); + const result = await makePageFetch(page)(`${USER_PATH}/saved.json`); + + assert.equal(result.status, 200); + assert.equal(gotoCalls(), 0, "an already-correct origin must not be re-navigated on every listing page"); +}); diff --git a/packages/polyfill-connectors/connectors/reddit/parsers.test.ts b/packages/polyfill-connectors/connectors/reddit/parsers.test.ts index 0fa6638fd..6db61f116 100644 --- a/packages/polyfill-connectors/connectors/reddit/parsers.test.ts +++ b/packages/polyfill-connectors/connectors/reddit/parsers.test.ts @@ -7,6 +7,7 @@ import { appendNewChildren, classifyListingStatus, commentRecord, + dedupeByFullname, domainOf, isoFromUnix, isTopLevelComment, @@ -374,6 +375,78 @@ test("appendNewChildren: stops once an item is at or below cursor", () => { assert.equal(out[0]?.data.name, "t3_a"); }); +// ─── Action-ordered listings must not stop on created_utc ─────────────── +// +// Regression guard for the observed defect: `upvoted` held a stored cursor of +// 2026-04-28 while the account's real upvote history reached back to 2011. +// Upvoting one old post puts an old `created_utc` at rank 1, and the +// created-based stop then halted the walk on item one, freezing the stream. + +test("appendNewChildren: action-ordered listing does not stop on an old item at the top", () => { + const out: RedditChild[] = []; + // Rank 1 is a 2015 post the owner upvoted TODAY — far below the cursor. + // A created-ordered stop would halt here and lose the two newer items. + const children: RedditChild[] = [ + { kind: "t3", data: { name: "t3_old", created_utc: 100 } }, + { kind: "t3", data: { name: "t3_mid", created_utc: 5000 } }, + { kind: "t3", data: { name: "t3_new", created_utc: 9000 } }, + ]; + const stop = appendNewChildren(children, 4000, out, "action"); + assert.equal(stop, false, "action-ordered listings must keep paging"); + assert.equal(out.length, 3, "every child must be kept regardless of created_utc"); + assert.deepEqual( + out.map((c) => c.data.name), + ["t3_old", "t3_mid", "t3_new"] + ); +}); + +test("appendNewChildren: created-ordered listing still stops at the cursor", () => { + // The incremental optimization must survive for submitted/comments, where + // created_utc IS the sort key. Losing this would make every run a full walk. + const out: RedditChild[] = []; + const children: RedditChild[] = [ + { kind: "t3", data: { name: "t3_a", created_utc: 300 } }, + { kind: "t3", data: { name: "t3_b", created_utc: 200 } }, + ]; + const stop = appendNewChildren(children, 200, out, "created"); + assert.equal(stop, true, "created-ordered listings must stop at the cursor"); + assert.equal(out.length, 1); +}); + +test("appendNewChildren: defaults to created-ordered when no order is given", () => { + // Guards the default parameter: an omitted order must be the SAFE-for- + // submitted/comments behavior, and callers must opt IN to the full walk. + const out: RedditChild[] = []; + const children: RedditChild[] = [{ kind: "t3", data: { name: "t3_a", created_utc: 100 } }]; + assert.equal(appendNewChildren(children, 200, out), true); + assert.equal(out.length, 0); +}); + +test("dedupeByFullname: collapses repeat sightings, keeping first-seen order", () => { + const children: RedditChild[] = [ + { kind: "t3", data: { name: "t3_a", created_utc: 300 } }, + { kind: "t1", data: { name: "t1_b", created_utc: 200 } }, + { kind: "t3", data: { name: "t3_a", created_utc: 300 } }, + ]; + const out = dedupeByFullname(children); + assert.equal(out.length, 2, "a repeated fullname must collapse to one item"); + assert.deepEqual( + out.map((c) => c.data.name), + ["t3_a", "t1_b"], + "first-seen order must be preserved" + ); +}); + +test("dedupeByFullname: keeps children lacking a fullname rather than collapsing them", () => { + // Two distinct nameless children must NOT collapse into one — treating a + // missing name as a single shared identity would silently drop real data. + const children: RedditChild[] = [ + { kind: "t3", data: { created_utc: 300 } }, + { kind: "t3", data: { created_utc: 200 } }, + ]; + assert.equal(dedupeByFullname(children).length, 2); +}); + test("maxCreatedEpoch: returns max across batch, clamped to current", () => { const children: RedditChild[] = [ { kind: "t3", data: { name: "t3_a", created_utc: 200 } }, diff --git a/packages/polyfill-connectors/connectors/reddit/parsers.ts b/packages/polyfill-connectors/connectors/reddit/parsers.ts index 3ba6d7e52..bbd1fdad7 100644 --- a/packages/polyfill-connectors/connectors/reddit/parsers.ts +++ b/packages/polyfill-connectors/connectors/reddit/parsers.ts @@ -197,17 +197,53 @@ export function voteRecord(c: RedditChild, fetchedAt: string): VoteRecord { // ─── Pagination helpers ───────────────────────────────────────────────── -/** Append children newer than the cursor into `out`. Reddit listings are - * newest-first, so once we hit a child at or before the cursor we're - * done with the stream — return true to signal "stop paging". */ +/** + * How a listing endpoint orders its children. This is a per-stream property + * of Reddit, not a global one, and getting it wrong silently truncates + * history — see `appendNewChildren`. + * + * "created" — ordered by the item's own `created_utc`, newest first. + * True for `submitted` and `comments`: the listing is the + * user's authorship timeline, so creation order IS listing + * order and an early stop at the cursor is sound. + * + * "action" — ordered by when the OWNER acted on the item (upvoted, + * saved, downvoted, hid), newest action first. True for + * `upvoted`/`saved`/`downvoted`/`hidden`. The item's + * `created_utc` is unrelated to its position: upvoting a + * 2015 post today puts a 2015 `created_utc` at rank 1. + */ +export type RedditListingOrder = "action" | "created"; + +/** + * Append children newer than the cursor into `out`, returning true to signal + * "stop paging". + * + * The stop rule depends on {@link RedditListingOrder}: + * + * For `created`-ordered listings the first child at or below the cursor + * proves every later child is older too (the listing is sorted by the very + * field being compared), so stopping there is correct and keeps incremental + * runs cheap. + * + * For `action`-ordered listings that inference is invalid, and acting on it + * loses data. `created_utc` is NOT the sort key, so a single old item near + * the top says nothing about what follows it. The pre-fix code stopped on the + * first such item, which for `upvoted` meant halting on item one as soon as + * the owner upvoted anything older than the stored high-water mark — the + * observed defect. These streams therefore always walk the full listing and + * rely on the caller deduping by fullname (`t3_`/`t1_` ids are stable), which + * is cheap because the walk is bounded by Reddit's own end-of-listing. + */ export function appendNewChildren( children: readonly RedditChild[], sinceEpochUtc: number | null, - out: RedditChild[] + out: RedditChild[], + order: RedditListingOrder = "created" ): boolean { for (const c of children) { const created = Number(c?.data?.created_utc ?? 0); - if (sinceEpochUtc && created <= sinceEpochUtc) { + if (order === "created" && sinceEpochUtc && created <= sinceEpochUtc) { return true; } out.push(c); @@ -215,6 +251,33 @@ export function appendNewChildren( return false; } +/** + * Drop children already present by Reddit fullname (`t3_…`/`t1_…`), keeping + * first-seen order. Fullnames are stable per item, so this is a safe identity + * for the full-walk `action`-ordered streams, where the same item can appear + * across runs and must not be double-counted. Children without a usable + * fullname are kept rather than collapsed — dropping them would silently lose + * data on a shape Reddit has not been observed to emit, and downstream + * validation is the right place to judge them. + */ +export function dedupeByFullname(children: readonly RedditChild[]): RedditChild[] { + const seen = new Set(); + const out: RedditChild[] = []; + for (const c of children) { + const name = typeof c?.data?.name === "string" ? c.data.name : ""; + if (!name) { + out.push(c); + continue; + } + if (seen.has(name)) { + continue; + } + seen.add(name); + out.push(c); + } + return out; +} + /** Build the `?after=…&limit=100` path segment given an endpoint and * the current pagination cursor. */ export function pagePath(endpoint: string, after: string | null, limit = PAGE_LIMIT): string { diff --git a/packages/polyfill-connectors/connectors/signal/collector-definition.ts b/packages/polyfill-connectors/connectors/signal/collector-definition.ts new file mode 100644 index 000000000..ee917f4fb --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/collector-definition.ts @@ -0,0 +1,52 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Signal connector's local-collector definition. + * + * The connector's own declaration of how it participates in local + * collection: its stable id, the runtime bindings it needs, and the default + * stream set an unscoped `run` should request. See + * `packages/polyfill-connectors/src/collector-definition.ts` for the + * contract this satisfies. + * + * Pure data only — no Node built-ins, no connector runtime imports — so it + * stays safe for the publishable `@pdpp/local-collector` build to re-export. + * + * This connector carries no native COMPILED dependency in the published + * npm package itself — it spawns the external `sigtop` binary + * (github.com/tbvdm/sigtop, ISC license) as an arms-length subprocess, the + * same shape this repo's Slack connector already uses for slackdump and + * Google Messages already uses for gmcli. `sigtop` is not bundled/installed + * by `@pdpp/local-collector`; it is a separate operator-installed + * prerequisite (`go install github.com/tbvdm/sigtop@latest`), resolved via + * the `SIGTOP_BIN` env var (default `"sigtop"` on PATH — see index.ts's + * `resolveSigtopBin`), documented in the manifest's + * `runtime_requirements.external_tools` entry. + * + * Unlike claude_code/codex/google_messages, this connector declares no + * `coverage_diagnostics` stream: each stream carries its own manifest + * `coverage_strategy` (`snapshot_import_receipt` / `parent_detail_accounting`), + * the same per-stream coverage mechanism imessage uses for an identical + * reason (see imessage/collector-definition.ts's module doc). + */ + +import type { LocalCollectorDefinition } from "@pdpp/connector-protocol/collector-definition"; + +export const SIGNAL_DEFAULT_STREAMS = ["messages", "conversations", "reactions", "attachments"] as const; + +/** + * Only `messages` declares a `consent_time_field` (`sent_at`). Conversations + * are standing entities (full resnapshot every run), and reactions/ + * attachments carry no owner-moment of their own — all three are collected + * whole, same as imessage's `participants`/`attachments`. + */ +export const SIGNAL_TIME_SCOPABLE_STREAMS = ["messages"] as const; + +export const signalCollectorDefinition: LocalCollectorDefinition = { + connector_id: "signal", + entry: "signal", + bindings: { filesystem: { required: true } }, + streams: SIGNAL_DEFAULT_STREAMS, + time_scopable_streams: SIGNAL_TIME_SCOPABLE_STREAMS, +}; diff --git a/packages/polyfill-connectors/connectors/signal/completeness-anchor.test.ts b/packages/polyfill-connectors/connectors/signal/completeness-anchor.test.ts new file mode 100644 index 000000000..8c5dc2bcb --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/completeness-anchor.test.ts @@ -0,0 +1,156 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Completeness-anchor tests for the Signal connector. + * + * Two concerns, deliberately tested apart: + * + * 1. `validateSourceTotal` — the fail-closed guard on the source-measured + * row total. A missing or malformed count must THROW, never silently + * become zero (which would read as "proven empty"). + * + * 2. `parseEmittedIds` / `mergeEmittedIds` — the durable emitted-id cursor + * that makes the below-watermark backfill check a SET comparison. The + * set is the whole point: a scalar count of the same facts is a + * tautology (`sourceTotal - belowWatermark` IS the in-window row count + * when both are read from one database in one instant), and it also + * cannot tell an upstream deletion from a real gap. PDPP retains + * records the source deletes, so held-but-gone-upstream must never + * alarm. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { mergeEmittedIds, parseEmittedIds, validateSourceTotal } from "./index.ts"; + +// ─── validateSourceTotal: fail closed ──────────────────────────────────── + +test("validateSourceTotal accepts a normal non-negative integer", () => { + assert.equal(validateSourceTotal(4739, "messages"), 4739); +}); + +test("validateSourceTotal accepts a proven-empty zero", () => { + assert.equal(validateSourceTotal(0, "messages"), 0); +}); + +test("validateSourceTotal throws on undefined rather than defaulting to zero", () => { + assert.throws(() => validateSourceTotal(undefined, "messages"), /signal_source_total_not_number/); +}); + +test("validateSourceTotal throws on null rather than defaulting to zero", () => { + assert.throws(() => validateSourceTotal(null, "messages"), /signal_source_total_not_number/); +}); + +test("validateSourceTotal throws on a string count", () => { + assert.throws(() => validateSourceTotal("4739", "messages"), /signal_source_total_not_number/); +}); + +test("validateSourceTotal throws on NaN", () => { + assert.throws(() => validateSourceTotal(Number.NaN, "messages"), /signal_source_total_not_finite/); +}); + +test("validateSourceTotal throws on Infinity", () => { + assert.throws(() => validateSourceTotal(Number.POSITIVE_INFINITY, "messages"), /signal_source_total_not_finite/); +}); + +test("validateSourceTotal throws on a fractional count", () => { + assert.throws(() => validateSourceTotal(12.5, "messages"), /signal_source_total_not_integer/); +}); + +test("validateSourceTotal throws on a negative count", () => { + assert.throws(() => validateSourceTotal(-1, "messages"), /signal_source_total_negative/); +}); + +test("validateSourceTotal names the failing measurement in the error", () => { + assert.throws(() => validateSourceTotal(undefined, "messages_below_watermark"), /messages_below_watermark/); +}); + +// ─── parseEmittedIds: tolerant read, safe default ──────────────────────── + +test("parseEmittedIds reads a normal id array", () => { + const parsed = parseEmittedIds(["a", "b", "c"]); + assert.deepEqual([...parsed].sort(), ["a", "b", "c"]); +}); + +test("parseEmittedIds returns an empty set for a legacy cursor with no id list", () => { + assert.equal(parseEmittedIds(undefined).size, 0); +}); + +test("parseEmittedIds returns an empty set for a malformed (non-array) value", () => { + assert.equal(parseEmittedIds({ nope: true }).size, 0); + assert.equal(parseEmittedIds("a,b,c").size, 0); +}); + +test("parseEmittedIds drops non-string and empty entries rather than trusting them", () => { + const parsed = parseEmittedIds(["a", 7, null, "", "b"]); + assert.deepEqual([...parsed].sort(), ["a", "b"]); +}); + +test("parseEmittedIds dedupes repeated ids", () => { + assert.equal(parseEmittedIds(["a", "a", "a"]).size, 1); +}); + +// ─── mergeEmittedIds: carry forward, bounded ───────────────────────────── + +test("mergeEmittedIds carries prior ids forward alongside this run's", () => { + const merged = mergeEmittedIds(new Set(["old1", "old2"]), ["new1"]); + assert.deepEqual([...merged].sort(), ["new1", "old1", "old2"]); +}); + +test("mergeEmittedIds keeps an empty prior set intact on a cold start", () => { + assert.deepEqual(mergeEmittedIds(new Set(), ["a", "b"]), ["a", "b"]); +}); + +test("mergeEmittedIds truncates to the newest ids when the cursor cap binds", () => { + // 200_000 is the cap; build one over it and assert the OLDEST are dropped, + // never the newest — the watermark advances, so the newest ids are the + // ones a backfill check still needs. + const prior = new Set(); + for (let i = 0; i < 200_000; i += 1) { + prior.add(`old${String(i)}`); + } + const merged = mergeEmittedIds(prior, ["newest1", "newest2"]); + assert.equal(merged.length, 200_000); + assert.equal(merged.at(-1), "newest2"); + assert.equal(merged.at(-2), "newest1"); + assert.equal(merged.includes("old0"), false, "oldest id must be the one dropped"); + assert.equal(merged.includes("old1"), false, "second-oldest id must be dropped"); +}); + +test("mergeEmittedIds does not truncate below the cap", () => { + const merged = mergeEmittedIds(new Set(["a"]), ["b", "c"]); + assert.equal(merged.length, 3); +}); + +// ─── The set-vs-count property this design exists for ──────────────────── + +test("a set difference reports a below-watermark backfill that a count cannot see", () => { + // Source ids at or below the watermark after a re-link backfill added + // three OLD messages. A count of the same instant is blind to this: the + // backfill raises both the source total and the below-watermark count by + // three, so `total - below` is unchanged. + const sourceBelowWatermark = ["m1", "m2", "m3", "backfilled1", "backfilled2", "backfilled3"]; + const priorEmitted = new Set(["m1", "m2", "m3"]); + + const unreachable = sourceBelowWatermark.filter((id) => !priorEmitted.has(id)); + assert.deepEqual(unreachable, ["backfilled1", "backfilled2", "backfilled3"]); + + // The count view of the very same facts, showing it cannot fire. + const totalBefore = 3 + 10; // 3 below-watermark + 10 in-window + const belowBefore = 3; + const totalAfter = 6 + 10; + const belowAfter = 6; + assert.equal(totalBefore - belowBefore, totalAfter - belowAfter, "the count check is blind to the backfill"); +}); + +test("an id present in holdings but deleted upstream produces no finding", () => { + // Preservation: PDPP keeps records the source later deletes. The check is + // one-directional by construction — it only asks which SOURCE ids were + // never emitted, so a vanished id simply is not in the source list. + const sourceBelowWatermark = ["m1", "m3"]; // m2 deleted from Signal Desktop + const priorEmitted = new Set(["m1", "m2", "m3"]); + + const unreachable = sourceBelowWatermark.filter((id) => !priorEmitted.has(id)); + assert.deepEqual(unreachable, [], "an upstream deletion must never be reported as a gap"); +}); diff --git a/packages/polyfill-connectors/connectors/signal/fixtures.ts b/packages/polyfill-connectors/connectors/signal/fixtures.ts new file mode 100644 index 000000000..ba41bf0e9 --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/fixtures.ts @@ -0,0 +1,275 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Test fixtures for the Signal connector: + * + * - `buildSignalExportFixture` builds a bounded, schema-accurate SQLite + * database mirroring the real Signal Desktop schema this connector + * reads (verified directly against sigtop's Go source — see + * parsers.ts's module doc) — never a copy of, or generated from, a + * real db.sqlite. No PII: all ids/text are synthetic fixture data. + * - `writeMockSigtopScript` writes a small Node script that stands in + * for the real `sigtop` binary (pointed at via `SIGTOP_BIN` in + * integration.test.ts): `check-database` succeeds, `export-database + * ` copies a pre-built fixture SQLite file to `` (mirroring + * sigtop's own real O_EXCL-target-must-not-exist behavior), and + * `export-attachments -i ` copies pre-seeded fake attachment + * files into ``. This lets integration.test.ts exercise the real + * subprocess-spawn seam (the actual entrypoint, via + * `runConnectorProtocolSubprocess`) without requiring a real `sigtop` + * install or a real Signal account — no real sigtop binary is + * available in this environment, so this is the only way to prove the + * spawn/parse/emit wiring end-to-end. + */ + +import { chmodSync, copyFileSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import Database from "better-sqlite3"; + +export interface FixtureMessage { + body: string | null; + conversationId: string; + id: string; + json?: string | null; + receivedAtMs?: number | null; + sentAt: number | null; + sourceServiceId?: string | null; + type?: string | null; +} + +export interface FixtureConversation { + e164?: string | null; + groupId?: string | null; + id: string; + name?: string | null; + serviceId?: string | null; + type: "private" | "group"; +} + +export interface FixtureMessageAttachment { + contentType: string | null; + fileName: string | null; + messageId: string; + size: number | null; +} + +export interface SignalExportFixtureOptions { + conversations?: FixtureConversation[]; + /** Omit the `message_attachments` table entirely (simulates a pre-1360 schema). */ + includeMessageAttachmentsTable?: boolean; + messageAttachments?: FixtureMessageAttachment[]; + messages: FixtureMessage[]; +} + +/** + * Builds a SQLite file at `dbPath` mirroring the real, minimal shape of + * Signal Desktop's `messages`/`conversations`/`message_attachments` tables + * this connector actually queries (column names verified against sigtop's + * signal/{message,recipient,attachment}.go — see this file's module doc). + * This is the file `sigtop export-database` would produce; the mock sigtop + * script (`writeMockSigtopScript`) copies it verbatim in place of a real + * export. + */ +export function buildSignalExportFixture(dbPath: string, opts: SignalExportFixtureOptions): void { + const db = new Database(dbPath); + try { + db.exec(` + CREATE TABLE conversations ( + id TEXT PRIMARY KEY, + type TEXT, + name TEXT, + e164 TEXT, + serviceId TEXT, + groupId TEXT + ); + CREATE TABLE messages ( + id TEXT PRIMARY KEY, + conversationId TEXT, + sourceServiceId TEXT, + type TEXT, + body TEXT, + sent_at INTEGER, + received_at_ms INTEGER, + json TEXT + ); + `); + if (opts.includeMessageAttachmentsTable !== false) { + db.exec(` + CREATE TABLE message_attachments ( + messageId TEXT, + contentType TEXT, + path TEXT, + fileName TEXT, + localKey TEXT, + size INTEGER + ); + `); + } + + const insertConversation = db.prepare( + "INSERT INTO conversations (id, type, name, e164, serviceId, groupId) VALUES (@id, @type, @name, @e164, @serviceId, @groupId)" + ); + for (const c of opts.conversations ?? []) { + insertConversation.run({ + e164: c.e164 ?? null, + groupId: c.groupId ?? null, + id: c.id, + name: c.name ?? null, + serviceId: c.serviceId ?? null, + type: c.type, + }); + } + + const insertMessage = db.prepare( + "INSERT INTO messages (id, conversationId, sourceServiceId, type, body, sent_at, received_at_ms, json) VALUES (@id, @conversationId, @sourceServiceId, @type, @body, @sentAt, @receivedAtMs, @json)" + ); + for (const m of opts.messages) { + insertMessage.run({ + body: m.body, + conversationId: m.conversationId, + id: m.id, + json: m.json ?? null, + receivedAtMs: m.receivedAtMs ?? null, + sentAt: m.sentAt, + sourceServiceId: m.sourceServiceId ?? null, + type: m.type ?? "incoming", + }); + } + + if (opts.includeMessageAttachmentsTable !== false) { + const insertAttachment = db.prepare( + "INSERT INTO message_attachments (messageId, contentType, path, fileName, localKey, size) VALUES (@messageId, @contentType, @path, @fileName, @localKey, @size)" + ); + for (const a of opts.messageAttachments ?? []) { + insertAttachment.run({ + contentType: a.contentType, + fileName: a.fileName, + localKey: "fixture-key", + messageId: a.messageId, + path: a.fileName, + size: a.size, + }); + } + } + } finally { + db.close(); + } +} + +export interface MockSigtopAttachmentFile { + bytes: Buffer; + /** Conversation subdirectory name sigtop's own export would create. */ + conversationDir: string; + filename: string; +} + +export interface MockSigtopOptions { + attachments?: MockSigtopAttachmentFile[]; + checkDatabaseExitCode?: number; + checkDatabaseStdout?: string; + exportDatabaseExitCode?: number; + fixtureDbPath: string; +} + +/** + * Writes a Node script at `scriptPath` that stands in for the real + * `sigtop` binary. Understands exactly the three subcommands+flag shapes + * this connector actually invokes: + * + * check-database -> exits 0, or checkDatabaseExitCode + * export-database -> copies fixtureDbPath to + * (fails if already exists, + * matching real sigtop's O_EXCL) + * export-attachments -i -> writes each configured + * attachment file under + * // + * + * Any other invocation exits non-zero with a message on stderr, so a test + * asserting on an unexpected sigtop call fails loudly instead of silently + * no-op'ing. + */ +export function writeMockSigtopScript(scriptPath: string, opts: MockSigtopOptions): void { + const payload = { + attachments: (opts.attachments ?? []).map((a) => ({ + bytesBase64: a.bytes.toString("base64"), + conversationDir: a.conversationDir, + filename: a.filename, + })), + checkDatabaseExitCode: opts.checkDatabaseExitCode ?? 0, + checkDatabaseStdout: opts.checkDatabaseStdout ?? "", + exportDatabaseExitCode: opts.exportDatabaseExitCode ?? 0, + fixtureDbPath: opts.fixtureDbPath, + }; + // The payload is inlined directly into the generated script's source + // text (rather than written to a sidecar file the script re-reads at + // spawn time) so this fixture builder never itself performs a whole-file + // read — that pattern is exactly what + // src/local-source-bounded-read-guard.ts's mechanical scan flags for + // filesystem/local-DB connector directories, and this file lives in + // exactly such a directory (connectors/signal/). + const script = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const data = ${JSON.stringify(payload)}; +const args = process.argv.slice(2); +const cmd = args[0]; + +if (cmd === "check-database") { + if (data.checkDatabaseStdout) { + process.stdout.write(data.checkDatabaseStdout + "\\n"); + } + process.exit(data.checkDatabaseExitCode); +} else if (cmd === "export-database") { + const target = args[1]; + if (!target) { + process.stderr.write("mock sigtop: export-database requires a target file\\n"); + process.exit(1); + } + if (fs.existsSync(target)) { + process.stderr.write("mock sigtop: target already exists (O_EXCL)\\n"); + process.exit(1); + } + if (data.exportDatabaseExitCode !== 0) { + process.exit(data.exportDatabaseExitCode); + } + fs.copyFileSync(data.fixtureDbPath, target); + process.exit(0); +} else if (cmd === "export-attachments") { + const dirArgIndex = args.findIndex((a, i) => i > 0 && a !== "-i" && !a.startsWith("-")); + const dir = dirArgIndex === -1 ? "." : args[dirArgIndex]; + fs.mkdirSync(dir, { recursive: true }); + for (const att of data.attachments) { + const convDir = path.join(dir, att.conversationDir); + fs.mkdirSync(convDir, { recursive: true }); + fs.writeFileSync(path.join(convDir, att.filename), Buffer.from(att.bytesBase64, "base64")); + } + process.exit(0); +} else { + process.stderr.write("mock sigtop: unrecognized invocation: " + args.join(" ") + "\\n"); + process.exit(1); +} +`; + writeFileSync(scriptPath, script, "utf8"); + chmodSync(scriptPath, 0o755); +} + +/** Convenience: build the fixture DB + mock script together under `dir`. */ +export function setupMockSigtop( + dir: string, + dbOpts: SignalExportFixtureOptions, + mockOpts: Omit = {} +): string { + mkdirSync(dir, { recursive: true }); + const fixtureDbPath = join(dir, "fixture-export.sqlite"); + buildSignalExportFixture(fixtureDbPath, dbOpts); + const scriptPath = join(dir, "mock-sigtop.cjs"); + writeMockSigtopScript(scriptPath, { ...mockOpts, fixtureDbPath }); + return scriptPath; +} + +// Re-exported for tests that want to copy an existing file the way the +// mock script's export-database handler does (parity check helper). +export function copyForFixture(from: string, to: string): void { + copyFileSync(from, to); +} diff --git a/packages/polyfill-connectors/connectors/signal/index.ts b/packages/polyfill-connectors/connectors/signal/index.ts new file mode 100644 index 000000000..7082e2a45 --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/index.ts @@ -0,0 +1,1318 @@ +#!/usr/bin/env node +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * PDPP Signal Desktop Connector (v0.1.0) + * + * Reads Signal Desktop's local encrypted `db.sqlite` through the `sigtop` + * sidecar CLI (github.com/tbvdm/sigtop, ISC license) rather than a native + * SQLCipher binding: Signal's DB key is protected by Electron's + * `safeStorage` (OS-keychain-backed AES/PBKDF2 unwrap — KWallet/libsecret + * on Linux, Keychain on macOS, DPAPI on Windows), and sigtop already + * implements that unwrap correctly across all three platforms. This is the + * same "arms-length subprocess" pattern the `slack` connector uses for its + * own Go sidecar tool — a Go binary kept at a distance rather than + * reimplemented or natively bound. + * + * `sigtop` binary resolution: `SIGTOP_BIN` env var, default `"sigtop"` on + * PATH — mirrors the analogous `*_BIN`/`resolve*Bin` pattern the `slack` + * connector uses for its own sidecar. Install: + * `go install github.com/tbvdm/sigtop@latest` (Linux additionally needs + * `libsecret-1-dev` + `pkg-config` at build time; see sigtop's README). + * + * ## sigtop CLI mechanics (verified directly against sigtop's Go source, + * github.com/tbvdm/sigtop — a prior design pass assumed `query-database` + * writes a SQLite file; it does not, and that assumption is corrected + * here): + * + * - `sigtop check-database` — fast-fail health check (SQLCipher + * cipher_integrity_check / integrity_check / foreign_key_check + * pragmas). Non-zero exit + failure lines on stdout when the DB fails + * a check. + * - `sigtop query-database [-o outfile] sql` — writes each result row as + * UNESCAPED pipe-delimited text (`strings.Join(columns, "|")`, one + * line per row) to `outfile` or stdout. There is no quoting: a column + * value containing a literal `|` (e.g. a message body) makes the row + * ambiguous to re-split. This connector therefore does NOT use + * query-database for anything containing free text. + * - `sigtop export-database [-B] [-d dir] file` — decrypts the FULL + * database to `file` as a real, regular plaintext SQLite database. + * `file` is a positional argument, not `-o`, and sigtop opens it with + * `O_EXCL` (fails if it already exists) — this connector always + * targets a freshly `mkdtemp`'d path it never pre-creates. This is + * the connector's PRIMARY data-access mechanism for `messages`, + * `conversations`, and `reactions`: opened read-only with + * `node:sqlite`'s `DatabaseSync`, queried with real bound `?` + * parameters (no manual SQL-string interpolation of the cursor value + * — `node:sqlite` supports positional bound parameters the same way + * `imessage`/`slack` already use them). One export, shared across all + * three streams, deleted at the end of the run. + * - `sigtop export-attachments -i ` — real incremental support + * native to the tool: a `.incremental` marker file in `` skips + * attachments already exported by a prior run. Attachment BYTES are + * encrypted at rest by Signal Desktop itself (a per-attachment + * `localKey`); only sigtop knows how to decrypt them, so this + * connector never reads Signal's raw `attachments.noindex/` tree + * directly — it always goes through this export step first, then + * hydrates from sigtop's own (connector-controlled) decrypted output + * directory using the exact same TOCTOU-safe O_NOFOLLOW read + * primitive (`readAttachmentFileSync`, `resolveSafeAttachmentPath`) + * `imessage/index.ts` uses against its own Attachments tree — same + * threat model (untrusted path from a local tool's output, trusted + * root directory, check-then-read race), so it is imported rather + * than re-derived. + * + * Signal Desktop's own schema exposes `hasAttachments`/edit-history/ + * reactions only inside each message row's `json` TEXT column, not as + * flat SQL columns (verified against sigtop's own + * signal/{message,reaction,attachment}.go, which parses that same JSON + * envelope for its own text/JSON export formats). `parsers.ts`'s + * `parseMessageJson`/`extractReactionsFromMessageJson` decode exactly that + * shape; this connector has no CLI subcommand that emits per-message + * reaction or attachment-presence data as a flat row, so both are derived + * here rather than queried directly. + * + * Per-attachment metadata (content_type, size, owning message id) comes + * from Signal Desktop's `message_attachments` table (schema version >= + * 1360 only — see sigtop's signal/attachment.go), read from the same + * export-database SQLite file already opened for messages/conversations/ + * reactions. On an older schema lacking that table, attachments still + * hydrate (bytes + hash) but message_id/content_type degrade to + * best-effort/null rather than the run failing — the same + * `tableExists`-gated schema-drift tolerance imessage applies to its own + * optional tables. + * + * Complexity budget: this connector deliberately does NOT copy slack's + * stall-vs-runtime-timeout budgets, sqlite-lock-race retry counts, scoped- + * archive reconciliation, or 3-way resumed/failed/throttled outcome + * machinery — all of that was earned by specific production incidents + * that have no analog here yet (see slack/index.ts's own module doc for + * that history). This connector starts at imessage's complexity budget: + * one subprocess call per concern, a plain incremental cursor, + * schema-drift tolerance via `tableExists`. Additional complexity is + * warranted only by a real observed failure, not preemptively. + * + * Reachability probe and mock-mutation check are permanently and + * correctly UNKNOWN for this connector: it has no network surface at all + * (a local subprocess reading a local file), the same shape as + * imessage/claude_code/whatsapp. That is not a gap the checklist expects + * closed — see CONNECTOR-CHECKLIST.md's exemption rule and this repo's + * docs/inbox/report-connector-coverage.md classification table. + * + * Target evidence level: Development — real collector, unverified against + * a live account (CONNECTOR-CHECKLIST.md). No cross-platform testing claim + * is made anywhere in this connector or its tests: sigtop's own docs and + * source state the same subcommands/flags behave identically across + * Linux/macOS/Windows, but only a Linux run (unit tests + a mocked + * subprocess integration test) has actually been exercised while building + * this — no real sigtop binary, no real Signal account, no macOS/Windows + * run. + */ + +import { spawn } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import type { Dirent } from "node:fs"; +import { + closeSync, + constants as fsConstants, + fstatSync, + mkdtempSync, + openSync, + readSync, + realpathSync, + rmSync, +} from "node:fs"; +import { mkdir, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, sep } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { isMainModule } from "@pdpp/connector-protocol"; +import { + buildDetailCoverageMessage, + type CollectContext, + type RecordData, + runConnector, +} from "../../src/connector-runtime.ts"; +import { + makeReferenceBlobUploader, + type ReferenceBlobRef, + runtimeBlobUploadAvailable, +} from "../../src/reference-blob-uploader.ts"; +import { + buildConversationRecord, + buildMessageRecord, + buildReactionRecord, + extractReactionsFromMessageJson, + parseMessageJson, + type SignalConversationRow, + type SignalMessageRow, +} from "./parsers.ts"; +import { validateRecord } from "./schemas.ts"; + +const PROGRESS_INTERVAL_ROWS = 10_000; +const ATTACHMENT_PROGRESS_INTERVAL = 25; +/** + * Cap on ids listed in the unreachable-backfill diagnostic. The full count + * is always reported; only the id sample is bounded, so a large hole stays + * legible without unbounded diagnostic growth. Mirrors slack's + * `MAX_MISSING_CHANNEL_IDS_IN_DIAGNOSTIC`. + */ +const MAX_UNREACHABLE_IDS_IN_DIAGNOSTIC = 50; +/** + * Cap on the durable emitted-id cursor. A Signal Desktop replica is a + * bounded local store (this owner's is ~4.7k messages), so this holds the + * whole history for realistic accounts while still refusing to grow without + * limit. When it binds, the backfill check reports "cannot prove" rather + * than a clean bill — see `mergeEmittedIds`. + */ +const MAX_EMITTED_ID_CURSOR = 200_000; + +// Conservative default cap for local attachment reads, matching imessage's +// / Gmail's documented-default pattern (25 MiB). Operators can raise/lower +// with PDPP_SIGNAL_MAX_ATTACHMENT_BYTES; non-positive/non-numeric overrides +// are ignored so a misconfigured env var can never silently disable the cap. +export const DEFAULT_MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024; +const MAX_ATTACHMENT_BYTES_ENV = "PDPP_SIGNAL_MAX_ATTACHMENT_BYTES"; +const POSITIVE_INTEGER_PATTERN = /^\d+$/; + +export function resolveMaxAttachmentBytes(env: NodeJS.ProcessEnv = process.env): number { + const raw = env[MAX_ATTACHMENT_BYTES_ENV]; + if (!(raw && POSITIVE_INTEGER_PATTERN.test(raw))) { + return DEFAULT_MAX_ATTACHMENT_BYTES; + } + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return DEFAULT_MAX_ATTACHMENT_BYTES; + } + return parsed; +} + +export function resolveSigtopBin(env: NodeJS.ProcessEnv = process.env): string { + return env.SIGTOP_BIN || "sigtop"; +} + +export function formatSigtopMissingError(bin: string): string { + return [ + `sigtop binary not found: ${bin}`, + "Install sigtop (github.com/tbvdm/sigtop) and either put it on PATH or set SIGTOP_BIN to its absolute path.", + "Linux builds additionally require libsecret-1-dev and pkg-config at build time (apt install libsecret-1-dev pkg-config) — see sigtop's README.", + ].join(" "); +} + +interface SigtopRunResult { + code: number | null; + stderr: string; + stdout: string; +} + +const DEFAULT_SIGTOP_TIMEOUT_MS = 5 * 60 * 1000; + +/** + * Spawn `sigtop ` and collect its output. Deliberately simple: a + * single total-runtime timeout, no stall-detection budget, no retry + * machinery — see this file's module doc for why slack's heavier + * subprocess runtime is not copied here. `check-database`/ + * `export-database`/`export-attachments` are all bounded, single-shot + * operations against a local file, not a multi-hour network archive dump. + */ +export function runSigtop( + args: string[], + { timeoutMs = DEFAULT_SIGTOP_TIMEOUT_MS }: { timeoutMs?: number } = {} +): Promise { + const bin = resolveSigtopBin(); + return new Promise((resolve, reject) => { + let child: ReturnType; + try { + child = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] }); + } catch (err) { + reject(err instanceof Error ? err : new Error(String(err))); + return; + } + let stdout = ""; + let stderr = ""; + let settled = false; + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + child.kill("SIGKILL"); + reject(new Error(`sigtop_timeout: '${args[0] ?? ""}' did not complete within ${timeoutMs}ms`)); + }, timeoutMs); + timer.unref?.(); + + child.stdout?.on("data", (d: Buffer) => { + stdout += d.toString(); + }); + child.stderr?.on("data", (d: Buffer) => { + stderr += d.toString(); + }); + child.on("error", (err: NodeJS.ErrnoException) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + if (err.code === "ENOENT") { + reject(new Error(`sigtop_not_found: ${formatSigtopMissingError(bin)}`, { cause: err })); + return; + } + reject(err); + }); + child.on("close", (code) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resolve({ code, stderr, stdout }); + }); + }); +} + +/** + * Validate + coerce a STATE-supplied cursor value to a non-negative + * integer. Even though this connector now queries via bound `?` + * parameters (no manual SQL-string interpolation — see + * `openExportedDatabase`'s callers), an untrusted STATE cursor is still + * coerced defensively before use: anything that is not a finite + * non-negative integer (NaN, Infinity, negative, non-numeric) coerces to + * 0, matching imessage's since-defaults-to-0 behavior on first run or a + * malformed cursor. + */ +export function parseCursorMs(value: unknown): number { + const n = Number(value); + if (!Number.isFinite(n) || n < 0) { + return 0; + } + return Math.floor(n); +} + +// Returns true when `table` exists in the opened database. Signal +// Desktop's schema has changed columns/tables across app versions (e.g. +// `message_attachments` only exists from schema version 1360 onward — see +// sigtop's signal/attachment.go). A stream built on an absent table +// degrades to a best-effort/SKIP_RESULT outcome rather than crashing the +// whole run, mirroring imessage's identical `tableExists` gate. +function tableExists(db: DatabaseSync, table: string): boolean { + const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table); + return row !== undefined; +} + +function columnExists(db: DatabaseSync, table: string, column: string): boolean { + const rows = db.prepare(`PRAGMA table_info(${table})`).iterate() as IterableIterator<{ name: string }>; + for (const r of rows) { + if (r.name === column) { + return true; + } + } + return false; +} + +/** + * Decrypts Signal's full database to a fresh temp file via + * `sigtop export-database` and opens it read-only with `node:sqlite`. + * Caller owns closing the DB and removing the temp directory (see + * `withExportedDatabase`) — kept separate so tests can drive the open step + * without spawning a real sigtop process. + */ +async function exportAndOpenDatabase(): Promise<{ db: DatabaseSync; tmpDir: string }> { + const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-signal-")); + // sigtop's export-database opens its target with O_EXCL and fails if it + // already exists — this path must be reserved (a fresh dir) but never + // pre-created as a file. + const dbFile = join(tmpDir, "export.sqlite"); + const result = await runSigtop(["export-database", dbFile]); + if (result.code !== 0) { + rmSync(tmpDir, { force: true, recursive: true }); + throw new Error(`sigtop_export_database_failed: exit code ${String(result.code)}: ${result.stderr.trim()}`); + } + const db = new DatabaseSync(dbFile, { readOnly: true }); + return { db, tmpDir }; +} + +/** Runs `fn` against a freshly exported+opened database, always cleaning up. */ +async function withExportedDatabase(fn: (db: DatabaseSync) => Promise): Promise { + const { db, tmpDir } = await exportAndOpenDatabase(); + try { + return await fn(db); + } finally { + db.close(); + rmSync(tmpDir, { force: true, recursive: true }); + } +} + +/** + * `sourceServiceId` is not a flat column on `messages` in any schema + * version sigtop supports (verified against signal/message.go) — it is + * obtained by joining the message's own sender identifier against + * `conversations.serviceId`. Older schema versions use `source`/ + * `sourceUuid` instead of `sourceServiceId` on the message row itself; + * this connector targets current (schema >= 88) Signal Desktop databases, + * consistent with sigtop's own newest-first column preference, and + * degrades `sender` to null on a database old enough to lack the + * `sourceServiceId` column rather than failing the whole stream. + * + * `sender` resolves through `LEFT JOIN conversations AS c ON + * m.sourceServiceId = c.serviceId`, selecting `c.id` rather than the raw + * `sourceServiceId` — matching sigtop's own sender resolution (verified + * against signal/message.go's schema>=88 query, which performs the + * identical join and returns the same `conversations.id`). This is a + * deliberate identity choice, not an incidental copy of sigtop's shape: + * Signal Desktop's schema gives every contact exactly one 1:1 + * `conversations` row (a "recipient" and its "conversation" are the same + * row), so `c.id` is that contact's own canonical identity row — the same + * id this connector's own `conversations` stream emits as `id`. Joining + * lets `messages.sender` actually foreign-key against `conversations.id` + * (what a consumer joining sender -> conversations would need); the raw + * `sourceServiceId` ACI/PNI UUID is a different identifier space that + * matches nothing else this connector emits. On a database old enough to + * lack `conversations.serviceId`, the join degrades to `NULL` the same way + * a missing `messages.sourceServiceId` already does. + */ +function messagesSelect(db: DatabaseSync): string { + const hasSourceServiceId = columnExists(db, "messages", "sourceServiceId"); + const hasReceivedAtMs = columnExists(db, "messages", "received_at_ms"); + const hasConversationServiceId = columnExists(db, "conversations", "serviceId"); + let senderExpr = "NULL"; + if (hasSourceServiceId) { + // With both columns present the join resolves the sender to a conversation + // id; without `conversations.serviceId` the raw service id is the best the + // schema can offer. Neither column means no sender at all. + senderExpr = hasConversationServiceId ? "c.id" : "m.sourceServiceId"; + } + const receivedExpr = hasReceivedAtMs ? "m.received_at_ms" : "NULL"; + const joinClause = + hasSourceServiceId && hasConversationServiceId + ? "LEFT JOIN conversations AS c ON m.sourceServiceId = c.serviceId" + : ""; + return ` + SELECT m.id AS id, m.conversationId AS conversationId, ${senderExpr} AS sourceServiceId, + m.sent_at AS sentAt, ${receivedExpr} AS receivedAtMs, m.body AS body, m.type AS type, + m.json AS json + FROM messages AS m + ${joinClause} + WHERE (m.sent_at > ? OR m.sent_at IS NULL) + ORDER BY m.sent_at ASC + `; +} + +function conversationsSelect(): string { + return ` + SELECT id, type, name, e164, serviceId, groupId + FROM conversations + ORDER BY id ASC + `; +} + +/** + * Validate a source-measured row total: must be a finite, non-negative + * integer. Fail closed — a missing or malformed count is NOT zero and NOT + * "complete". Mirrors jellyfin's `validateTotalRecordCount` discipline + * (connectors/jellyfin/index.ts), minus the monotonicity rule: Signal + * Desktop legitimately DELETES rows (disappearing messages expire, the + * owner deletes a thread), so a decreasing total here is ordinary source + * behavior rather than the provider-side anomaly it is for Jellyfin. + */ +export function validateSourceTotal(value: unknown, label: string): number { + if (typeof value !== "number") { + throw new Error(`signal_source_total_not_number: ${label}`); + } + if (!Number.isFinite(value)) { + throw new Error(`signal_source_total_not_finite: ${label}`); + } + if (!Number.isInteger(value)) { + throw new Error(`signal_source_total_not_integer: ${label}`); + } + if (value < 0) { + throw new Error(`signal_source_total_negative: ${label}`); + } + return value; +} + +/** + * The `messages` completeness anchor: `SELECT COUNT(*)` over Signal + * Desktop's own `messages` table, measured at the SOURCE boundary (the + * decrypted database) and independent of what this run enumerated or + * emitted. + * + * This is a genuine external anchor — it is the source's own count, not a + * number this connector derived from its own output. It is what makes the + * `messages` stream's `considered` an objectively-measured denominator + * rather than a tautology. + * + * CEILING, stated honestly: Signal Desktop is a linked-device REPLICA, not + * the account of record. Messages that Signal Desktop never received (sent + * before this device was linked, or expired before it synced) are absent + * from this table and therefore absent from this denominator too. This + * anchor proves "we hold everything the local replica holds"; it cannot + * prove "we hold everything the Signal account ever had." No local anchor + * can, and this connector does not claim otherwise. + * + * Fails closed: a query error or a malformed count throws rather than + * defaulting to zero, because an unmeasurable boundary is not an empty one. + */ +function countSourceMessages(db: DatabaseSync): number { + let row: unknown; + try { + row = db.prepare("SELECT COUNT(*) AS total FROM messages").get(); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`signal_source_total_query_failed: messages: ${msg}`, { cause: err }); + } + const total = (row as { total?: unknown } | undefined)?.total; + return validateSourceTotal(typeof total === "bigint" ? Number(total) : total, "messages"); +} + +/** + * Read the durable emitted-id cursor tolerantly. A missing, malformed, or + * legacy (pre-cursor) value yields an EMPTY set, which is safe: an empty + * prior set combined with a `since` of 0 (the only state a cold start can + * be in) puts nothing below the watermark, so no false gap can be reported. + * A legacy cursor that DOES carry `last_sent_at_ms` but no `emitted_ids` + * would report every below-watermark id as unreachable on its first + * post-deploy run — see `readPriorEmittedIds`'s caller, which suppresses + * the finding in exactly that case. + */ +export function parseEmittedIds(value: unknown): Set { + if (!Array.isArray(value)) { + return new Set(); + } + const out = new Set(); + for (const entry of value) { + if (typeof entry === "string" && entry !== "") { + out.add(entry); + } + } + return out; +} + +/** + * Merge the prior emitted-id set with this run's, newest-last, bounded. + * + * The bound matters: this set is a durable cursor, and an unbounded one + * grows without limit on a large account. Keeping the NEWEST ids is the + * right truncation because the check only ever asks about ids at or below + * the watermark — and the watermark advances, so the oldest ids are the + * ones least likely to be re-offered by a backfill. When truncation is in + * force the check degrades to "cannot prove", never to a false clean bill: + * `reconcileMessageAnchor`'s caller reports the truncation explicitly. + */ +export function mergeEmittedIds(prior: ReadonlySet, current: readonly string[]): string[] { + const merged = [...prior, ...current]; + return merged.length > MAX_EMITTED_ID_CURSOR ? merged.slice(merged.length - MAX_EMITTED_ID_CURSOR) : merged; +} + +interface QueriedMessageRows { + /** Ids this run actually emitted, for the durable emitted-id cursor. */ + emittedIds: string[]; + latestMs: number; + reactionSourceRows: Array<{ id: string; json: string | null }>; + skippedNullDate: number; + /** Rows this run's cursor window enumerated (the in-window denominator). */ + windowConsidered: number; + /** Rows this run's cursor window accounted for (emitted or deliberately skipped). */ + windowCovered: number; +} + +/** + * How this run's cursor window relates to the source's own holdings. + * + * `sourceTotal` is the objective anchor (every row Signal Desktop holds), + * reported as a stream-level fact and NOT substituted for the per-window + * `considered`: `isHealthyBoundedContinuation` + * (reference-implementation/server/continuation-proof.ts) admits a bounded + * window only when `considered === covered`, so folding a stream-level + * total into the window denominator would pin every incremental run to a + * permanent false `partial`. + * + * `unreachableCount` is the load-bearing part: source ids sitting at-or-below + * the cursor watermark that this connector has NOT previously emitted. A + * forward-only `sent_at > ?` filter can never revisit them. + * + * WHY A SET, NOT A COUNT. A scalar comparison cannot work here, in both + * directions: + * + * - `sourceTotal - belowWatermark` is identically the in-window row count + * when both are measured from the same database in the same instant. + * It is a tautology: it cannot fire, not even on the backfill hole it + * would be written to catch. (Verified numerically against the live + * 4,739-row database before this design was chosen.) + * - Counts also conflate "missing upstream", "surplus", and "duplicated". + * PDPP is a preservation product: it deliberately RETAINS records the + * source later deletes, so held-but-absent-upstream is expected correct + * behavior, never a defect. A two-way count check flags successful + * preservation as loss — backwards. + * + * A set difference distinguishes the three cases. Only `upstream-present + * AND never-emitted` is a real gap; `held-but-gone-upstream` is preservation + * working and is deliberately not reported here. + */ +interface MessageAnchorReconciliation { + belowWatermark: number; + sourceTotal: number; + /** + * How many below-watermark source ids were never emitted. This is the full + * count, independent of how many ids `unreachableIdSample` retained. + */ + unreachableCount: number; + /** + * At most `MAX_UNREACHABLE_IDS_IN_DIAGNOSTIC` ids, kept for the diagnostic. + * The full id list is deliberately NOT materialized: the below-watermark + * row set is unbounded (it grows with the whole Signal history), while the + * only consumers are a count and a capped sample. + */ + unreachableIdSample: string[]; +} + +/** + * Reconcile the source's own rows against the cursor watermark and the set + * of ids prior runs already emitted. + * + * Every input is measured at the SOURCE boundary (the decrypted database) + * except `priorEmittedIds`, which is this connector's own durable cursor. + * Fails closed: a malformed count or an unreadable query throws rather than + * defaulting to "complete". + * + * Deletion-safe by construction: it only ever asks "which source ids have + * we never emitted", never "do our holdings match the source count". A row + * deleted from Signal Desktop simply stops appearing in the source set; it + * produces no finding. + */ +function reconcileMessageAnchor( + db: DatabaseSync, + since: number, + priorEmittedIds: ReadonlySet +): MessageAnchorReconciliation { + const sourceTotal = countSourceMessages(db); + // Streamed with `iterate()`, not `.all()`: the below-watermark set is + // bounded only by the size of the owner's whole Signal history, and the + // two facts derived from it (a count, and a capped id sample) both fold + // row-by-row. Materializing the full list would put the entire message + // history in memory to compute a number. + // + // A cold start (no prior cursor) has emitted nothing yet, so every + // below-watermark id would look "unreachable". But a cold start has + // `since === 0`, which puts nothing below the watermark — the set is + // empty and no false finding is possible. + let belowWatermarkRows = 0; + let unreachableCount = 0; + const unreachableIdSample: string[] = []; + try { + const iter = db + .prepare("SELECT id AS id FROM messages WHERE sent_at IS NOT NULL AND sent_at <= ?") + .iterate(since) as IterableIterator<{ id?: unknown }>; + for (const row of iter) { + belowWatermarkRows += 1; + const { id } = row; + if (typeof id === "string" && !priorEmittedIds.has(id)) { + unreachableCount += 1; + if (unreachableIdSample.length < MAX_UNREACHABLE_IDS_IN_DIAGNOSTIC) { + unreachableIdSample.push(id); + } + } + } + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`signal_source_total_query_failed: messages_below_watermark: ${msg}`, { cause: err }); + } + const belowWatermark = validateSourceTotal(belowWatermarkRows, "messages_below_watermark"); + if (belowWatermark > sourceTotal) { + throw new Error( + `signal_source_total_inconsistent: below-watermark ${String(belowWatermark)} exceeds source total ${String(sourceTotal)}` + ); + } + return { belowWatermark, sourceTotal, unreachableCount, unreachableIdSample }; +} + +/** + * Runs the messages query and builds every row's record + cursor + * contribution, WITHOUT emitting anything — pure row-shaping over an + * already-open database. Kept separate from `emitMessageRowsAndReactions` + * so a `reactions`-only request (no `messages` in scope) can walk the same + * rows to derive reactions without also emitting `messages` RECORD/ + * SKIP_RESULT traffic for a stream nobody asked for. + */ +function queryMessageRows( + db: DatabaseSync, + since: number +): { latestMs: number; rows: Array<{ built: ReturnType; raw: SignalMessageRow }> } { + const iter = db.prepare(messagesSelect(db)).iterate(since) as IterableIterator; + let latestMs = since; + const rows: Array<{ built: ReturnType; raw: SignalMessageRow }> = []; + for (const r of iter) { + const built = buildMessageRecord(r); + if (built.sentAtMs !== null && built.sentAtMs > latestMs) { + latestMs = built.sentAtMs; + } + rows.push({ built, raw: r }); + } + return { latestMs, rows }; +} + +async function emitMessageRowsAndReactions({ + db, + emitRecord, + emitReactions, + progress, + since, +}: { + db: DatabaseSync; + emitRecord: (stream: string, data: RecordData) => Promise; + emitReactions: boolean; + progress: (message: string, extra?: Record) => Promise; + since: number; +}): Promise { + const { latestMs, rows } = queryMessageRows(db, since); + let itemOrdinal = 0; + let skippedNullDate = 0; + // Measured at the enumeration site from the rows the source handed back — + // never aliased to the emitted count. A row skipped for an unusable date + // raises `windowConsidered` without raising `windowCovered`, so it reads + // `partial` exactly as it should. + const windowConsidered = rows.length; + let windowCovered = 0; + const emittedIds: string[] = []; + const reactionSourceRows: Array<{ id: string; json: string | null }> = []; + for (const { built, raw } of rows) { + itemOrdinal += 1; + if (built.sentAtMs === null) { + // No usable timestamp on this row (neither sent_at nor + // received_at_ms) — no honest cursor position. Skip with a + // diagnostic rather than fabricating the run's wall clock (see + // imessage's identical null-date-skip-not-fabricate rule). + skippedNullDate += 1; + continue; + } + await emitRecord("messages", built.record); + windowCovered += 1; + emittedIds.push(raw.id); + if (emitReactions) { + reactionSourceRows.push({ id: raw.id, json: raw.json }); + } + if (itemOrdinal % PROGRESS_INTERVAL_ROWS === 0) { + await progress(`Signal phase=emit pass=emit stream=messages item=${itemOrdinal}`, { stream: "messages" }); + } + } + return { emittedIds, latestMs, reactionSourceRows, skippedNullDate, windowConsidered, windowCovered }; +} + +async function emitReactionRowsFromMessages( + reactionSourceRows: ReadonlyArray<{ id: string; json: string | null }>, + emitRecord: (stream: string, data: RecordData) => Promise +): Promise { + let emitted = 0; + for (const row of reactionSourceRows) { + const json = parseMessageJson(row.json); + for (const reaction of extractReactionsFromMessageJson(row.id, json)) { + await emitRecord("reactions", buildReactionRecord(reaction)); + emitted += 1; + } + } + return emitted; +} + +async function emitConversationRows( + db: DatabaseSync, + emitRecord: (stream: string, data: RecordData) => Promise +): Promise { + const rows = db.prepare(conversationsSelect()).iterate() as IterableIterator; + let emitted = 0; + for (const r of rows) { + await emitRecord("conversations", buildConversationRecord(r)); + emitted += 1; + } + return emitted; +} + +// ─── Attachment hydration (reused pattern from imessage/index.ts) ─────── + +interface SafeAttachmentPathResult { + ok: boolean; + path: string | null; +} + +/** + * Resolves `rawPath` (sigtop's own exported-attachment path) against + * `root` (sigtop's export directory) and verifies the result genuinely + * resolves inside `root` before returning it. Identical logic to + * imessage's `resolveSafeAttachmentPath` — the threat model is the same + * (untrusted path from a local tool's output, trusted root directory, + * TOCTOU race between check and read) even though sigtop's export + * directory is connector-controlled and arguably a stronger boundary than + * imessage's user-owned Attachments tree. + */ +function resolveSafeAttachmentPath(rawPath: string, root: string): SafeAttachmentPathResult { + let realRoot: string; + try { + realRoot = realpathSync(root); + } catch { + return { ok: false, path: null }; + } + let realCandidate: string; + try { + realCandidate = realpathSync(rawPath); + } catch { + return { ok: false, path: null }; + } + const withinRoot = realCandidate === realRoot || realCandidate.startsWith(realRoot + sep); + if (!withinRoot) { + return { ok: false, path: null }; + } + return { ok: true, path: realCandidate }; +} + +export interface AttachmentHydrationResult { + blobRef: ReferenceBlobRef | null; + bytes: Buffer | null; + contentSha256: string | null; + hydrationError: string | null; + hydrationStatus: "deferred" | "hydrated" | "failed" | "too_large" | "missing"; + sizeBytes: number | null; +} + +function missingAttachmentResult(): AttachmentHydrationResult { + return { + blobRef: null, + bytes: null, + contentSha256: null, + hydrationError: "attachment file is missing, unreadable, or was replaced with a symlink.", + hydrationStatus: "missing", + sizeBytes: null, + }; +} + +/** + * Bounded local read with the check-then-read (TOCTOU) window closed — + * verbatim port of imessage/index.ts's `readAttachmentFileSync`. See that + * file's doc comment for the full O_NOFOLLOW rationale; it applies + * unchanged here since the threat model (canonicalize-then-verify + * followed by a separate read syscall) is identical. Linux (this + * connector's development/test platform) supports O_NOFOLLOW + * unconditionally, and so do macOS/Windows (sigtop's other supported + * platforms per Node's fs module) — untested here, see module doc. + */ +export function readAttachmentFileSync(localPath: string, maxBytes: number): AttachmentHydrationResult { + let fd: number; + try { + // biome-ignore lint/suspicious/noBitwiseOperators: composing POSIX open() flags requires a bitmask OR, not logical OR. + fd = openSync(localPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + } catch { + return missingAttachmentResult(); + } + try { + let size: number; + try { + ({ size } = fstatSync(fd)); + } catch (err) { + return { + blobRef: null, + bytes: null, + contentSha256: null, + hydrationError: err instanceof Error ? err.message : "Failed to stat attachment file.", + hydrationStatus: "failed", + sizeBytes: null, + }; + } + if (size > maxBytes) { + return { + blobRef: null, + bytes: null, + contentSha256: null, + hydrationError: `attachment exceeds max size: ${size} > ${maxBytes} bytes`, + hydrationStatus: "too_large", + sizeBytes: size, + }; + } + const buffer = Buffer.alloc(size); + let offset = 0; + try { + while (offset < size) { + const bytesRead = readSync(fd, buffer, offset, size - offset, offset); + if (bytesRead === 0) { + break; + } + offset += bytesRead; + } + } catch (err) { + return { + blobRef: null, + bytes: null, + contentSha256: null, + hydrationError: err instanceof Error ? err.message : "Failed to read attachment file.", + hydrationStatus: "failed", + sizeBytes: size, + }; + } + const bytes = offset === size ? buffer : buffer.subarray(0, offset); + const contentSha256 = createHash("sha256").update(bytes).digest("hex"); + return { + blobRef: null, + bytes, + contentSha256, + hydrationError: null, + hydrationStatus: "deferred", + sizeBytes: bytes.byteLength, + }; + } finally { + try { + closeSync(fd); + } catch { + // Nothing actionable: the read outcome above is already decided. + } + } +} + +function uploadAttachmentBlob(args: { + bytes: Buffer; + mimeType: string; + recordKey: string; +}): Promise { + const rsUrl = process.env.PDPP_RS_URL || process.env.RS_URL; + const ownerToken = process.env.PDPP_OWNER_TOKEN; + if (!(runtimeBlobUploadAvailable(process.env) && rsUrl && ownerToken)) { + return Promise.resolve(null); + } + const uploader = makeReferenceBlobUploader({ + connectorInstanceId: process.env.PDPP_CONNECTOR_INSTANCE_ID || null, + ownerToken, + rsUrl, + }); + return uploader({ + connectorId: "https://registry.pdpp.org/connectors/signal", + content: [args.bytes], + mimeType: args.mimeType, + recordKey: args.recordKey, + stream: "attachments", + }); +} + +function attachmentRecordId(localPath: string): string { + return createHash("sha256").update(localPath).digest("hex"); +} + +interface AttachmentMetadata { + contentType: string | null; + messageId: string | null; + size: number | null; +} + +/** + * Best-effort metadata join: keyed by the exported file's basename against + * `message_attachments.fileName` (schema >= 1360 only — see this file's + * module doc). sigtop does not preserve a stable per-attachment id across + * its export step and Signal's own DB row, so a filename collision (two + * different attachments across different messages sharing an identical + * original filename, e.g. two photos both literally named "IMG_0001.jpg") + * can join to the wrong metadata row. This degrades gracefully: a wrong or + * missing join leaves message_id/content_type null/best-effort rather than + * corrupting the attachment's actual bytes/hash, which are read directly + * from the exported file regardless of whether the join succeeded. + */ +function buildAttachmentMetadataIndex(db: DatabaseSync): Map { + const index = new Map(); + if (!tableExists(db, "message_attachments")) { + return index; + } + const rows = db + .prepare("SELECT messageId, contentType, fileName, size FROM message_attachments") + .iterate() as IterableIterator<{ + contentType: string | null; + fileName: string | null; + messageId: string | null; + size: number | null; + }>; + for (const r of rows) { + if (r.fileName) { + index.set(r.fileName, { contentType: r.contentType, messageId: r.messageId, size: r.size }); + } + } + return index; +} + +/** + * Recursively lists every regular file under `dir`. sigtop's + * `export-attachments` lays files out in a nested directory structure (one + * subdirectory per conversation), so a flat `readdirSync` is not enough. + * Symlinks encountered during the walk are not traversed (Node's + * `withFileTypes` reports them as `isSymbolicLink()`, which this function + * skips) — the eventual read still goes through + * `resolveSafeAttachmentPath` + `readAttachmentFileSync`'s O_NOFOLLOW gate + * regardless, but skipping them here avoids walking into an + * attacker-controlled subtree during enumeration itself. + */ +async function listExportedAttachmentFiles(dir: string): Promise { + const out: string[] = []; + async function walk(current: string): Promise { + let entries: Dirent[]; + try { + entries = await readdir(current, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = join(current, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.isFile()) { + out.push(full); + } + } + } + await walk(dir); + return out; +} + +async function resolveAttachmentHydration( + localPath: string, + contentType: string, + id: string, + maxBytes: number, + exportRoot: string +): Promise { + const safe = resolveSafeAttachmentPath(localPath, exportRoot); + if (!(safe.ok && safe.path)) { + return { + blobRef: null, + bytes: null, + contentSha256: null, + hydrationError: "attachment file is missing, unreadable, or outside the trusted export root.", + hydrationStatus: "missing", + sizeBytes: null, + }; + } + const local = readAttachmentFileSync(safe.path, maxBytes); + if (!(local.hydrationStatus === "deferred" && local.bytes)) { + return local; + } + try { + const blobRef = await uploadAttachmentBlob({ bytes: local.bytes, mimeType: contentType, recordKey: id }); + return { + ...local, + blobRef, + contentSha256: blobRef?.sha256 ?? local.contentSha256, + hydrationStatus: blobRef ? "hydrated" : "deferred", + sizeBytes: blobRef?.size_bytes ?? local.sizeBytes, + }; + } catch (err) { + return { + ...local, + blobRef: null, + hydrationError: err instanceof Error ? err.message : "Attachment blob upload failed.", + hydrationStatus: "failed", + }; + } +} + +async function emitAttachmentRows({ + emitRecord, + exportRoot, + files, + maxBytes, + metadataIndex, + progress, +}: { + emitRecord: (stream: string, data: RecordData) => Promise; + exportRoot: string; + files: readonly string[]; + maxBytes: number; + metadataIndex: Map; + progress: (message: string, extra?: Record) => Promise; +}): Promise { + let emitted = 0; + for (const localPath of files) { + const filename = localPath.split(sep).at(-1) || `attachment-${emitted}`; + const meta = metadataIndex.get(filename) ?? null; + const contentType = meta?.contentType || "application/octet-stream"; + const id = attachmentRecordId(localPath); + const result = await resolveAttachmentHydration(localPath, contentType, id, maxBytes, exportRoot); + + await emitRecord("attachments", { + id, + message_id: meta?.messageId ?? null, + conversation_id: null, + filename, + content_type: contentType, + size_bytes: result.sizeBytes ?? meta?.size ?? null, + content_sha256: result.contentSha256, + hydration_status: result.hydrationStatus, + hydration_error: result.hydrationError, + blob_ref: result.blobRef, + }); + emitted += 1; + + if (emitted % ATTACHMENT_PROGRESS_INTERVAL === 0) { + await progress(`Signal phase=emit pass=emit stream=attachments item=${emitted}`, { stream: "attachments" }); + } + } + return emitted; +} + +async function collectAttachments(ctx: CollectContext): Promise { + const { emit, emitRecord, progress } = ctx; + const maxBytes = resolveMaxAttachmentBytes(process.env); + const exportRoot = + process.env.SIGNAL_ATTACHMENTS_EXPORT_DIR || join(tmpdir(), `pdpp-signal-attachments-${randomUUID()}`); + await mkdir(exportRoot, { recursive: true }); + + await progress("Signal phase=index pass=index stream=attachments exporting via sigtop", { stream: "attachments" }); + const result = await runSigtop(["export-attachments", "-i", exportRoot]); + if (result.code !== 0) { + throw new Error(`sigtop_export_attachments_failed: exit code ${String(result.code)}: ${result.stderr.trim()}`); + } + + const files = await listExportedAttachmentFiles(exportRoot); + const metadataIndex = await withExportedDatabase((db) => Promise.resolve(buildAttachmentMetadataIndex(db))); + + await progress("Signal phase=emit pass=emit stream=attachments hydrating rows", { stream: "attachments" }); + const emitted = await emitAttachmentRows({ emitRecord, exportRoot, files, maxBytes, metadataIndex, progress }); + if (emitted === 0) { + await emit({ + type: "SKIP_RESULT", + stream: "attachments", + reason: "no_attachments_exported", + message: + "sigtop export-attachments produced no files for this account (or all attachments were already exported by a prior incremental run).", + }); + } + await emit({ type: "STATE", stream: "attachments", cursor: { synced_at: new Date().toISOString() } }); +} + +/** + * Emit the `messages` completeness evidence for one run: the in-window + * coverage declaration plus, when the source holds rows the forward-only + * cursor will never revisit, an explicit gap. + * + * Two facts, deliberately kept separate: + * + * 1. DETAIL_COVERAGE carries the WINDOW's own `considered`/`covered` — + * rows this run enumerated vs rows it accounted for. It must NOT carry + * the stream-level source total: `isHealthyBoundedContinuation` + * (reference-implementation/server/continuation-proof.ts) admits a + * bounded window only when `considered === covered`, so substituting a + * stream-level total would pin every incremental run to a permanent + * false `partial`. + * + * 2. The backfill hole is reported as a SET difference: source ids at or + * below the watermark that no prior run ever emitted. A forward-only + * `sent_at > ?` filter can never revisit those rows, so without this + * evidence a re-link backfill carrying older `sent_at` values would be + * permanently invisible. + * + * Deliberately one-directional. Ids this connector holds that are GONE + * from Signal Desktop are NOT reported: PDPP retains records the source + * deletes, so held-but-absent-upstream is preservation working as + * intended, not a gap. Only upstream-present-and-never-emitted counts. + */ +async function emitMessageAnchorEvidence( + emit: CollectContext["emit"], + anchor: MessageAnchorReconciliation, + result: QueriedMessageRows, + since: number, + proveBackfill: boolean +): Promise { + if (!proveBackfill) { + // A legacy cursor (watermark, no emitted-id set) cannot distinguish + // "already emitted" from "newly backfilled". Say so plainly rather than + // reporting either a false gap or an unearned clean bill. + await emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "backfill_check_unproven_legacy_cursor", + message: + "Below-watermark backfill could not be checked this run: the messages cursor predates the emitted-id set. " + + "This run seeds that set; the next run checks properly.", + diagnostics: { source_total: anchor.sourceTotal, below_watermark: anchor.belowWatermark }, + recovery_hint: { action: "retry_by_runtime", retryable: true }, + }); + } else if (anchor.unreachableCount > 0) { + const sample = anchor.unreachableIdSample; + await emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "source_rows_below_watermark_unreachable", + message: + `Signal Desktop holds ${String(anchor.unreachableCount)} message(s) at or below the cursor watermark that this ` + + "connector has never emitted — a backfill carrying older sent_at values. The forward-only sent_at cursor cannot " + + "revisit them; re-run with collection_mode=full_refresh to recover them.", + diagnostics: { + source_total: anchor.sourceTotal, + below_watermark: anchor.belowWatermark, + unreachable_count: anchor.unreachableCount, + unreachable_ids: sample, + truncated: sample.length < anchor.unreachableCount, + watermark_sent_at_ms: since, + }, + recovery_hint: { action: "retry_by_runtime", retryable: true }, + }); + } + await emit( + buildDetailCoverageMessage({ + stream: "messages", + stateStream: "messages", + requiredKeys: [], + hydratedKeys: [], + considered: result.windowConsidered, + covered: result.windowCovered, + }) + ); +} + +/** + * Runs the messages/reactions pass against an already-open exported + * database: derives both streams from the same row scan (reactions live + * inside each message's own json blob, not a standalone table — see + * parsers.ts), emitting only the streams actually requested. Extracted + * from `collect()` to keep that function's cognitive complexity bounded — + * this is one cohesive concern (one query, two derived streams), not + * incidental nesting. + */ +async function collectMessagesAndReactions({ + db, + ctx, + emitMessages, + emitReactions, + priorEmittedIds, + proveBackfill, + since, +}: { + ctx: CollectContext; + db: DatabaseSync; + emitMessages: boolean; + emitReactions: boolean; + priorEmittedIds: ReadonlySet; + /** False on a legacy cursor with no emitted-id set — see the caller. */ + proveBackfill: boolean; + since: number; +}): Promise { + const { emit, emitRecord, progress } = ctx; + await progress("Signal phase=index pass=index stream=messages querying rows", { stream: "messages" }); + + // Measured BEFORE the emit pass, at the source boundary, so the anchor + // cannot be contaminated by anything this run emitted. Throws on a + // malformed or unreadable count — an unmeasurable boundary is not an + // empty one. + const anchor = reconcileMessageAnchor(db, since, priorEmittedIds); + + let result: QueriedMessageRows; + try { + result = await emitMessageRowsAndReactions({ + db, + // A reactions-only request (messages not in scope) must not emit + // `messages` RECORD/SKIP_RESULT traffic for a stream nobody asked + // for — route through a no-op in that case. + emitRecord: emitMessages ? emitRecord : () => Promise.resolve(), + emitReactions, + progress, + since, + }); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`signal_db_query_failed: ${msg}`, { cause: err }); + } + + if (emitMessages) { + if (result.skippedNullDate > 0) { + await emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "message_date_unusable", + message: `Skipped ${result.skippedNullDate} message(s) with a missing or unusable sent_at/received_at_ms; they cannot be placed on the sent_at cursor without fabricating a timestamp.`, + }); + } + await emitMessageAnchorEvidence(emit, anchor, result, since, proveBackfill); + await emit({ + type: "STATE", + stream: "messages", + cursor: { + last_sent_at_ms: result.latestMs, + // Union of what prior runs emitted and what this run emitted. This + // is what makes the backfill check a SET comparison rather than a + // count: without it there is no way to tell a below-watermark row + // we already have from one a re-link just introduced. + emitted_ids: mergeEmittedIds(priorEmittedIds, result.emittedIds), + }, + }); + } + + if (emitReactions) { + await progress("Signal phase=emit pass=emit stream=reactions deriving from message json", { stream: "reactions" }); + await emitReactionRowsFromMessages(result.reactionSourceRows, emitRecord); + await emit({ type: "STATE", stream: "reactions", cursor: { synced_at: new Date().toISOString() } }); + } +} + +async function collectConversations(db: DatabaseSync, ctx: CollectContext): Promise { + const { emit, emitRecord, progress } = ctx; + await progress("Signal phase=index pass=index stream=conversations querying rows", { stream: "conversations" }); + try { + await emitConversationRows(db, emitRecord); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`signal_db_query_failed: ${msg}`, { cause: err }); + } + await emit({ type: "STATE", stream: "conversations", cursor: { synced_at: new Date().toISOString() } }); +} + +async function collectMessagesConversationsReactions(ctx: CollectContext): Promise { + const { state, requested } = ctx; + const emitMessages = requested.has("messages"); + const emitReactions = requested.has("reactions"); + const emitConversationsStream = requested.has("conversations"); + if (!(emitMessages || emitReactions || emitConversationsStream)) { + return; + } + await withExportedDatabase(async (db) => { + if (emitMessages || emitReactions) { + const messagesState = (state.messages ?? {}) as { emitted_ids?: unknown; last_sent_at_ms?: number }; + const since = parseCursorMs(messagesState.last_sent_at_ms ?? 0); + const priorEmittedIds = parseEmittedIds(messagesState.emitted_ids); + // A legacy cursor carries a watermark but no emitted-id set. Every + // below-watermark id would then look "never emitted" — a false gap + // for rows prior runs genuinely did emit. Treat that first + // post-deploy run as unproven (skip the check, seed the set) rather + // than alarming. The run after it has a real set and checks properly. + const hasEmittedIdCursor = Array.isArray(messagesState.emitted_ids); + await collectMessagesAndReactions({ + ctx, + db, + emitMessages, + emitReactions, + priorEmittedIds, + proveBackfill: since === 0 || hasEmittedIdCursor, + since, + }); + } + if (emitConversationsStream) { + await collectConversations(db, ctx); + } + }); +} + +async function runHealthCheck(ctx: CollectContext): Promise { + await ctx.progress("Signal phase=index pass=index sigtop check-database", {}); + const health = await runSigtop(["check-database"]); + if (health.code !== 0) { + throw new Error( + `signal_db_check_failed: sigtop check-database reported a problem: ${health.stderr.trim() || health.stdout.trim()}` + ); + } +} + +// Guarded so importing this module (e.g. from a unit test) never starts the +// stdin-driven Collection Profile protocol loop — that only happens when +// this file is the actual process entry point. See is-main-module.ts. +if (isMainModule(import.meta.url)) { + runConnector({ + name: "signal", + validateRecord, + async collect(ctx) { + await runHealthCheck(ctx); + await collectMessagesConversationsReactions(ctx); + if (ctx.requested.has("attachments")) { + await collectAttachments(ctx); + } + }, + }); +} diff --git a/packages/polyfill-connectors/connectors/signal/integration.test.ts b/packages/polyfill-connectors/connectors/signal/integration.test.ts new file mode 100644 index 000000000..ed28dd3c7 --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/integration.test.ts @@ -0,0 +1,466 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * End-to-end integration tests for the Signal connector, driven through + * the real subprocess entrypoint (index.ts) via + * `runConnectorProtocolSubprocess` — same pattern imessage/integration.test.ts + * uses. `SIGTOP_BIN` is pointed at a mock sigtop script (fixtures.ts's + * `setupMockSigtop`) rather than a real `sigtop` install: no real sigtop + * binary and no real Signal account are available in this environment, so + * this is the honest limit of what can be proven here — see index.ts's + * module doc for what remains unverified (a real sigtop run, a real + * account, non-Linux platforms). + */ + +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import type { EmittedMessage } from "../../src/connector-runtime.ts"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; +import { buildSignalExportFixture, setupMockSigtop } from "./fixtures.ts"; + +const PACKAGE_ROOT = join(import.meta.dirname, "..", ".."); +const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "signal", "index.ts"); + +function records(messages: readonly EmittedMessage[], stream: string): Record[] { + return messages + .filter((m): m is Extract => m.type === "RECORD") + .filter((m) => m.stream === stream) + .map((m) => m.data); +} + +function skips(messages: readonly EmittedMessage[]): Extract[] { + return messages.filter((m): m is Extract => m.type === "SKIP_RESULT"); +} + +function states(messages: readonly EmittedMessage[]): Extract[] { + return messages.filter((m): m is Extract => m.type === "STATE"); +} + +function runSignal( + scriptPath: string, + streams: string[], + env: Record = {}, + state: Record = {} +) { + return runConnectorProtocolSubprocess({ + cwd: PACKAGE_ROOT, + entrypoint: ENTRYPOINT, + env: { + PDPP_OWNER_TOKEN: "", + PDPP_RS_URL: "", + RS_URL: "", + SIGTOP_BIN: scriptPath, + ...env, + }, + start: { + scope: { streams: streams.map((name) => ({ name })) }, + state, + type: "START", + }, + }); +} + +const CONV_A = "11111111-1111-1111-1111-111111111111"; +const CONV_B = "44444444-4444-4444-4444-444444444444"; +const SENDER = "33333333-3333-3333-3333-333333333333"; +// The sender's own resolved conversations.id, distinct from CONV_A (the +// chat thread) and SENDER (the raw ACI/PNI service-id column value) — see +// index.ts's messagesSelect doc: `sender` resolves through +// `LEFT JOIN conversations AS c ON m.sourceServiceId = c.serviceId`, +// selecting c.id, matching sigtop's own sender resolution. +const SENDER_CONV_ID = "66666666-6666-6666-6666-666666666666"; + +test("signal reports a failed DONE when sigtop is not on PATH / SIGTOP_BIN is wrong", async () => { + const result = await runConnectorProtocolSubprocess({ + allowFailedDone: true, + cwd: PACKAGE_ROOT, + entrypoint: ENTRYPOINT, + env: { SIGTOP_BIN: "/nonexistent/definitely-not-sigtop" }, + start: { scope: { streams: [{ name: "messages" }] }, state: {}, type: "START" }, + }); + const done = result.messages.findLast((m): m is Extract => m.type === "DONE"); + assert.equal(done?.status, "failed"); + assert.match(done?.error?.message ?? "", /sigtop_not_found/); + assert.match(done?.error?.message ?? "", /Install sigtop/); +}); + +test("signal reports a failed DONE when check-database fails", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const scriptPath = setupMockSigtop( + dir, + { messages: [] }, + { checkDatabaseExitCode: 1, checkDatabaseStdout: "integrity check failed: foo" } + ); + const result = await runConnectorProtocolSubprocess({ + allowFailedDone: true, + cwd: PACKAGE_ROOT, + entrypoint: ENTRYPOINT, + env: { SIGTOP_BIN: scriptPath }, + start: { scope: { streams: [{ name: "messages" }] }, state: {}, type: "START" }, + }); + const done = result.messages.findLast((m): m is Extract => m.type === "DONE"); + assert.equal(done?.status, "failed"); + assert.match(done?.error?.message ?? "", /signal_db_check_failed/); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal emits messages with a monotonic sent_at cursor", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const t0 = Date.parse("2024-06-05T13:00:00.000Z"); + const scriptPath = setupMockSigtop(dir, { + conversations: [ + { id: CONV_A, name: "Alice", type: "private" }, + { id: SENDER_CONV_ID, name: "Bob", serviceId: SENDER, type: "private" }, + ], + messages: [ + { + body: "hey", + conversationId: CONV_A, + id: "22222222-2222-2222-2222-222222222222", + sentAt: t0, + sourceServiceId: SENDER, + type: "incoming", + }, + { + body: "hi back", + conversationId: CONV_A, + id: "55555555-5555-5555-5555-555555555555", + sentAt: t0 + 60_000, + sourceServiceId: null, + type: "outgoing", + }, + ], + }); + + const result = await runSignal(scriptPath, ["messages"]); + const msgs = records(result.messages, "messages"); + assert.equal(msgs.length, 2); + assert.equal(msgs[0]?.conversation_id, CONV_A); + // sender resolves through the sourceServiceId -> conversations.serviceId + // join to the sender's own conversations.id (SENDER_CONV_ID), not the + // raw sourceServiceId (SENDER) — matching sigtop's own resolution. + assert.equal(msgs[0]?.sender, SENDER_CONV_ID); + assert.equal(msgs[1]?.sender, null); + + const state = states(result.messages).find((s) => s.stream === "messages"); + assert.ok(state, "expected a messages STATE checkpoint"); + const cursor = state.cursor as { last_sent_at_ms: number }; + assert.equal(cursor.last_sent_at_ms, t0 + 60_000); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal cursor carries forward: a second run with prior STATE only emits newer messages", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const t0 = Date.parse("2024-06-05T13:00:00.000Z"); + const scriptPath = setupMockSigtop(dir, { + conversations: [{ id: CONV_A, name: "Alice", type: "private" }], + messages: [ + { + body: "first", + conversationId: CONV_A, + id: "22222222-2222-2222-2222-222222222222", + sentAt: t0, + sourceServiceId: SENDER, + type: "incoming", + }, + { + body: "second", + conversationId: CONV_A, + id: "55555555-5555-5555-5555-555555555555", + sentAt: t0 + 120_000, + sourceServiceId: SENDER, + type: "incoming", + }, + ], + }); + + const first = await runSignal(scriptPath, ["messages"], {}, { messages: { last_sent_at_ms: t0 - 1 } }); + assert.equal(records(first.messages, "messages").length, 2); + + const second = await runSignal(scriptPath, ["messages"], {}, { messages: { last_sent_at_ms: t0 } }); + const secondMsgs = records(second.messages, "messages"); + assert.equal(secondMsgs.length, 1); + assert.equal(secondMsgs[0]?.id, "55555555-5555-5555-5555-555555555555"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal skips a message with no usable sent_at/received_at_ms deterministically instead of stamping the run clock", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const t0 = Date.parse("2024-06-05T13:00:00.000Z"); + const scriptPath = setupMockSigtop(dir, { + messages: [ + { + body: "no date", + conversationId: CONV_A, + id: "22222222-2222-2222-2222-222222222222", + receivedAtMs: null, + sentAt: null, + }, + { body: "has date", conversationId: CONV_A, id: "55555555-5555-5555-5555-555555555555", sentAt: t0 }, + ], + }); + + const result = await runSignal(scriptPath, ["messages"]); + const msgs = records(result.messages, "messages"); + assert.equal(msgs.length, 1); + assert.equal(msgs[0]?.id, "55555555-5555-5555-5555-555555555555"); + + const skip = skips(result.messages).find((s) => s.stream === "messages"); + assert.ok(skip, "expected a messages SKIP_RESULT for the null-date row"); + assert.equal(skip?.reason, "message_date_unusable"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal derives has_attachments/is_edited from the message json blob", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const t0 = Date.parse("2024-06-05T13:00:00.000Z"); + const scriptPath = setupMockSigtop(dir, { + messages: [ + { + body: "photo", + conversationId: CONV_A, + id: "22222222-2222-2222-2222-222222222222", + json: JSON.stringify({ attachments: [{ path: "a" }], editHistory: [{ body: "old" }] }), + sentAt: t0, + }, + { + body: "plain", + conversationId: CONV_A, + id: "55555555-5555-5555-5555-555555555555", + json: JSON.stringify({}), + sentAt: t0 + 1000, + }, + ], + }); + + const result = await runSignal(scriptPath, ["messages"]); + const msgs = records(result.messages, "messages"); + const withAttachments = msgs.find((m) => m.id === "22222222-2222-2222-2222-222222222222"); + const plain = msgs.find((m) => m.id === "55555555-5555-5555-5555-555555555555"); + assert.equal(withAttachments?.has_attachments, true); + assert.equal(withAttachments?.is_edited, true); + assert.equal(plain?.has_attachments, false); + assert.equal(plain?.is_edited, false); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal emits conversations as a full resnapshot with null member_count", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const scriptPath = setupMockSigtop(dir, { + conversations: [ + { id: CONV_A, name: "Alice", type: "private" }, + { groupId: "group-xyz", id: CONV_B, name: "Team Chat", type: "group" }, + ], + messages: [], + }); + + const result = await runSignal(scriptPath, ["conversations"]); + const convs = records(result.messages, "conversations"); + assert.equal(convs.length, 2); + const byId = new Map(convs.map((c) => [c.id, c])); + assert.equal(byId.get(CONV_A)?.type, "private"); + assert.equal(byId.get(CONV_A)?.title, "Alice"); + assert.equal(byId.get(CONV_B)?.type, "group"); + // member_count is never fabricated — see parsers.ts's buildConversationRecord doc. + assert.equal(byId.get(CONV_A)?.member_count, null); + assert.equal(byId.get(CONV_B)?.member_count, null); + + const state = states(result.messages).find((s) => s.stream === "conversations"); + assert.ok(state, "expected a conversations STATE checkpoint"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal derives reactions from message json without a standalone reactions table", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const t0 = Date.parse("2024-06-05T13:00:00.000Z"); + const scriptPath = setupMockSigtop(dir, { + messages: [ + { + body: "look at this", + conversationId: CONV_A, + id: "22222222-2222-2222-2222-222222222222", + json: JSON.stringify({ + reactions: [ + { emoji: "👍", fromId: SENDER, targetTimestamp: t0 }, + { emoji: "❤️", fromId: "other-sender", targetTimestamp: t0 }, + ], + }), + sentAt: t0, + }, + ], + }); + + const result = await runSignal(scriptPath, ["messages", "reactions"]); + const reactions = records(result.messages, "reactions"); + assert.equal(reactions.length, 2); + const byEmoji = new Map(reactions.map((r) => [r.emoji, r])); + assert.equal(byEmoji.get("👍")?.sender, SENDER); + assert.equal(byEmoji.get("👍")?.message_id, "22222222-2222-2222-2222-222222222222"); + assert.equal(byEmoji.get("❤️")?.sender, "other-sender"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal derives reactions when only the reactions stream is requested (messages not in scope)", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const t0 = Date.parse("2024-06-05T13:00:00.000Z"); + const scriptPath = setupMockSigtop(dir, { + messages: [ + { + body: "x", + conversationId: CONV_A, + id: "22222222-2222-2222-2222-222222222222", + json: JSON.stringify({ reactions: [{ emoji: "👍", fromId: SENDER, targetTimestamp: t0 }] }), + sentAt: t0, + }, + ], + }); + + const result = await runSignal(scriptPath, ["reactions"]); + // messages was never requested: no messages RECORD/SKIP_RESULT traffic. + assert.equal(records(result.messages, "messages").length, 0); + assert.equal(skips(result.messages).filter((s) => s.stream === "messages").length, 0); + // reactions is still derived from the underlying message rows. + const reactions = records(result.messages, "reactions"); + assert.equal(reactions.length, 1); + assert.equal(reactions[0]?.emoji, "👍"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal hydrates an exported attachment and joins metadata from message_attachments", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const t0 = Date.parse("2024-06-05T13:00:00.000Z"); + const messageId = "22222222-2222-2222-2222-222222222222"; + const bytes = Buffer.from([1, 2, 3, 4]); + const scriptPath = setupMockSigtop( + dir, + { + messageAttachments: [ + { contentType: "image/jpeg", fileName: "IMG_0001.jpg", messageId, size: bytes.byteLength }, + ], + messages: [{ body: null, conversationId: CONV_A, id: messageId, sentAt: t0 }], + }, + { attachments: [{ bytes, conversationDir: "Alice", filename: "IMG_0001.jpg" }] } + ); + + const result = await runSignal(scriptPath, ["attachments"]); + const attachments = records(result.messages, "attachments"); + assert.equal(attachments.length, 1); + const [a] = attachments; + assert.equal(a?.hydration_status, "deferred"); + assert.equal(a?.filename, "IMG_0001.jpg"); + assert.equal(a?.content_type, "image/jpeg"); + assert.equal(a?.message_id, messageId); + assert.equal(a?.size_bytes, bytes.byteLength); + assert.match(String(a?.content_sha256), /^[0-9a-f]{64}$/); + // id is a sha256 of the local export path, never the raw path itself. + assert.match(String(a?.id), /^[0-9a-f]{64}$/); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal attachment metadata degrades to null on a schema without message_attachments, bytes still hydrate", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const bytes = Buffer.from([9, 9, 9]); + const scriptPath = setupMockSigtop( + dir, + { includeMessageAttachmentsTable: false, messages: [] }, + { attachments: [{ bytes, conversationDir: "Bob", filename: "note.txt" }] } + ); + + const result = await runSignal(scriptPath, ["attachments"]); + const [a] = records(result.messages, "attachments"); + assert.equal(a?.hydration_status, "deferred"); + assert.equal(a?.message_id, null); + assert.equal(a?.content_type, "application/octet-stream"); + assert.match(String(a?.content_sha256), /^[0-9a-f]{64}$/); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal SKIP_RESULTs the attachments stream when sigtop exports nothing", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-")); + try { + const scriptPath = setupMockSigtop(dir, { messages: [] }, { attachments: [] }); + const result = await runSignal(scriptPath, ["attachments"]); + assert.equal(records(result.messages, "attachments").length, 0); + const skip = skips(result.messages).find((s) => s.stream === "attachments"); + assert.ok(skip, "expected an attachments SKIP_RESULT"); + assert.equal(skip?.reason, "no_attachments_exported"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("signal.json declares tier=preview and no consent_time_field claim beyond messages.sent_at", async () => { + const { readFile } = await import("node:fs/promises"); + const manifestPath = join(PACKAGE_ROOT, "manifests", "signal.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as { + capabilities?: { public_listing?: { tier?: string } }; + streams: Array<{ name: string; consent_time_field?: string }>; + }; + assert.equal(manifest.capabilities?.public_listing?.tier, "preview"); + const messages = manifest.streams.find((s) => s.name === "messages"); + assert.equal(messages?.consent_time_field, "sent_at"); +}); + +// Sanity check on the fixture builder itself, independent of the mock +// sigtop wiring: proves buildSignalExportFixture produces a database this +// connector's own SQL (messagesSelect/conversationsSelect column names) +// can actually query without a runtime SQL error, catching a fixture/schema +// drift before it manifests as a confusing subprocess-level failure. +test("buildSignalExportFixture produces a database queryable by the connector's own column names", async () => { + const dir = await mkdtemp(join(tmpdir(), "pdpp-signal-fixture-")); + try { + const { DatabaseSync } = await import("node:sqlite"); + const dbPath = join(dir, "test.sqlite"); + buildSignalExportFixture(dbPath, { + conversations: [{ id: CONV_A, name: "Alice", type: "private" }], + messages: [{ body: "hi", conversationId: CONV_A, id: "22222222-2222-2222-2222-222222222222", sentAt: 1000 }], + }); + const db = new DatabaseSync(dbPath, { readOnly: true }); + try { + const rows = db + .prepare("SELECT id, conversationId, sourceServiceId, sent_at, received_at_ms, body, type, json FROM messages") + .all(); + assert.equal(rows.length, 1); + const convRows = db.prepare("SELECT id, type, name, e164, serviceId, groupId FROM conversations").all(); + assert.equal(convRows.length, 1); + } finally { + db.close(); + } + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); diff --git a/packages/polyfill-connectors/connectors/signal/parsers.test.ts b/packages/polyfill-connectors/connectors/signal/parsers.test.ts new file mode 100644 index 000000000..87001fb40 --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/parsers.test.ts @@ -0,0 +1,208 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + buildConversationRecord, + buildMessageRecord, + buildReactionRecord, + extractReactionsFromMessageJson, + parseMessageJson, + signalEpochMsToIso, +} from "./parsers.ts"; + +// ─── signalEpochMsToIso ─────────────────────────────────────────────────── + +test("signalEpochMsToIso converts a positive epoch-ms value to ISO", () => { + assert.equal(signalEpochMsToIso(1_717_594_922_000), "2024-06-05T13:42:02.000Z"); +}); + +test("signalEpochMsToIso returns null for null/undefined/zero/negative/non-finite", () => { + assert.equal(signalEpochMsToIso(null), null); + assert.equal(signalEpochMsToIso(undefined), null); + assert.equal(signalEpochMsToIso(0), null); + assert.equal(signalEpochMsToIso(-5), null); + assert.equal(signalEpochMsToIso(Number.NaN), null); + assert.equal(signalEpochMsToIso(Number.POSITIVE_INFINITY), null); +}); + +// ─── parseMessageJson ───────────────────────────────────────────────────── + +test("parseMessageJson decodes a well-formed messageJSON blob", () => { + const json = parseMessageJson('{"attachments":[{"path":"a"}],"reactions":[{"emoji":"👍","fromId":"x"}]}'); + assert.equal(json.attachments?.length, 1); + assert.equal(json.reactions?.length, 1); +}); + +test("parseMessageJson returns {} for null/empty/malformed input", () => { + assert.deepEqual(parseMessageJson(null), {}); + assert.deepEqual(parseMessageJson(undefined), {}); + assert.deepEqual(parseMessageJson(""), {}); + assert.deepEqual(parseMessageJson("not json"), {}); + assert.deepEqual(parseMessageJson("42"), {}); + assert.deepEqual(parseMessageJson("null"), {}); +}); + +// ─── buildMessageRecord ─────────────────────────────────────────────────── + +test("buildMessageRecord builds a record from a fully-populated row", () => { + const built = buildMessageRecord({ + body: "hey there", + conversationId: "11111111-1111-1111-1111-111111111111", + id: "22222222-2222-2222-2222-222222222222", + json: '{"attachments":[{"path":"a"}],"editHistory":[{"body":"old"}]}', + receivedAtMs: 1_717_594_930_000, + sentAt: 1_717_594_922_000, + sourceServiceId: "33333333-3333-3333-3333-333333333333", + type: "incoming", + }); + assert.equal(built.record.id, "22222222-2222-2222-2222-222222222222"); + assert.equal(built.record.conversation_id, "11111111-1111-1111-1111-111111111111"); + assert.equal(built.record.sender, "33333333-3333-3333-3333-333333333333"); + assert.equal(built.record.sent_at, "2024-06-05T13:42:02.000Z"); + assert.equal(built.record.body, "hey there"); + assert.equal(built.record.type, "incoming"); + assert.equal(built.record.has_attachments, true); + assert.equal(built.record.is_edited, true); + assert.equal(built.sentAtMs, 1_717_594_922_000); +}); + +test("buildMessageRecord falls back to receivedAtMs when sentAt is unusable", () => { + const built = buildMessageRecord({ + body: null, + conversationId: "11111111-1111-1111-1111-111111111111", + id: "22222222-2222-2222-2222-222222222222", + json: null, + receivedAtMs: 1_717_594_930_000, + sentAt: null, + sourceServiceId: null, + type: null, + }); + assert.equal(built.record.sent_at, "2024-06-05T13:42:10.000Z"); + assert.equal(built.sentAtMs, 1_717_594_930_000); + assert.equal(built.record.sender, null); +}); + +test("buildMessageRecord reports sentAtMs=null and sent_at=null when neither timestamp is usable (skip signal for index.ts)", () => { + const built = buildMessageRecord({ + body: "no usable date", + conversationId: "11111111-1111-1111-1111-111111111111", + id: "22222222-2222-2222-2222-222222222222", + json: null, + receivedAtMs: null, + sentAt: 0, + sourceServiceId: null, + type: null, + }); + assert.equal(built.record.sent_at, null); + assert.equal(built.sentAtMs, null); +}); + +test("buildMessageRecord degrades has_attachments/is_edited to false on malformed json without failing", () => { + const built = buildMessageRecord({ + body: "x", + conversationId: "11111111-1111-1111-1111-111111111111", + id: "22222222-2222-2222-2222-222222222222", + json: "not valid json {", + receivedAtMs: null, + sentAt: 1_717_594_922_000, + sourceServiceId: null, + type: null, + }); + assert.equal(built.record.has_attachments, false); + assert.equal(built.record.is_edited, false); +}); + +test("buildMessageRecord: empty attachments/editHistory arrays report false, not true", () => { + const built = buildMessageRecord({ + body: "x", + conversationId: "11111111-1111-1111-1111-111111111111", + id: "22222222-2222-2222-2222-222222222222", + json: '{"attachments":[],"editHistory":[]}', + receivedAtMs: null, + sentAt: 1_717_594_922_000, + sourceServiceId: null, + type: null, + }); + assert.equal(built.record.has_attachments, false); + assert.equal(built.record.is_edited, false); +}); + +// ─── buildConversationRecord ────────────────────────────────────────────── + +test("buildConversationRecord builds a private conversation record", () => { + const record = buildConversationRecord({ + e164: "+15551234567", + groupId: null, + id: "11111111-1111-1111-1111-111111111111", + name: "Alice", + serviceId: "33333333-3333-3333-3333-333333333333", + type: "private", + }); + assert.equal(record.id, "11111111-1111-1111-1111-111111111111"); + assert.equal(record.type, "private"); + assert.equal(record.title, "Alice"); + assert.equal(record.member_count, null); +}); + +test("buildConversationRecord builds a group conversation record with null title when name is absent", () => { + const record = buildConversationRecord({ + e164: null, + groupId: "group-abc", + id: "44444444-4444-4444-4444-444444444444", + name: null, + serviceId: null, + type: "group", + }); + assert.equal(record.type, "group"); + assert.equal(record.title, null); +}); + +test("buildConversationRecord treats an unrecognized type as null rather than passing it through raw", () => { + const record = buildConversationRecord({ + e164: null, + groupId: null, + id: "44444444-4444-4444-4444-444444444444", + name: "x", + serviceId: null, + type: "some-future-type", + }); + assert.equal(record.type, null); +}); + +// ─── buildReactionRecord / extractReactionsFromMessageJson ─────────────── + +test("buildReactionRecord builds the composite message_id:emoji:sender id", () => { + const record = buildReactionRecord({ + emoji: "👍", + fromId: "33333333-3333-3333-3333-333333333333", + messageId: "22222222-2222-2222-2222-222222222222", + }); + assert.equal(record.id, "22222222-2222-2222-2222-222222222222:👍:33333333-3333-3333-3333-333333333333"); + assert.equal(record.message_id, "22222222-2222-2222-2222-222222222222"); + assert.equal(record.emoji, "👍"); + assert.equal(record.sender, "33333333-3333-3333-3333-333333333333"); +}); + +test("extractReactionsFromMessageJson extracts every well-formed reaction", () => { + const json = parseMessageJson( + '{"reactions":[{"emoji":"👍","fromId":"a","targetTimestamp":1},{"emoji":"❤️","fromId":"b","targetTimestamp":2}]}' + ); + const reactions = extractReactionsFromMessageJson("msg-1", json); + assert.equal(reactions.length, 2); + assert.deepEqual(reactions[0], { emoji: "👍", fromId: "a", messageId: "msg-1" }); + assert.deepEqual(reactions[1], { emoji: "❤️", fromId: "b", messageId: "msg-1" }); +}); + +test("extractReactionsFromMessageJson returns [] when reactions is absent or not an array", () => { + assert.deepEqual(extractReactionsFromMessageJson("msg-1", {}), []); + assert.deepEqual(extractReactionsFromMessageJson("msg-1", parseMessageJson('{"reactions":"not-an-array"}')), []); +}); + +test("extractReactionsFromMessageJson drops a reaction missing emoji or fromId rather than fabricating a placeholder", () => { + const json = parseMessageJson( + '{"reactions":[{"emoji":"","fromId":"a"},{"emoji":"👍","fromId":""},{"fromId":"a"},{"emoji":"👍"}]}' + ); + assert.deepEqual(extractReactionsFromMessageJson("msg-1", json), []); +}); diff --git a/packages/polyfill-connectors/connectors/signal/parsers.ts b/packages/polyfill-connectors/connectors/signal/parsers.ts new file mode 100644 index 000000000..0a523b0db --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/parsers.ts @@ -0,0 +1,224 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Pure parsers for the Signal connector. Kept free of subprocess spawning, + * `node:sqlite`, and any other Node I/O so they can be unit-tested in + * isolation against literal row fixtures (see parsers.test.ts) — mirrors + * the domain-logic/IO split `slack/parsers.ts` established (see that + * file's own module doc). index.ts does the subprocess/SQLite I/O and + * calls into these functions. + * + * Row shapes here are what `index.ts` reads out of the plaintext SQLite + * database `sigtop export-database` produces (a real, regular SQLite file + * — verified directly against sigtop's Go source, + * github.com/tbvdm/sigtop/signal/{message,recipient,reaction}.go — not + * `sigtop query-database`, whose `-o outfile` output is unescaped + * pipe-delimited text unsafe for free-text columns like a message body). + * + * Signal Desktop's `messages` table exposes only a handful of flat SQL + * columns reliably across schema versions (`id`, `conversationId`, `type`, + * `body`, `sent_at`); `hasAttachments`, `isEdited`/edit history, and + * reactions are NOT flat columns — Signal Desktop nests them inside the + * message row's own `json` TEXT column (Signal Desktop's `messageJSON` + * shape: `{ attachments: [...], reactions: [...], editHistory: [...] }`). + * `parseMessageJson` below decodes exactly that shape. This mirrors + * sigtop's own `attachmentsFromJSON`/`parseReactionJSON` (signal/ + * attachment.go, signal/reaction.go), reimplemented here in TypeScript + * rather than shelled out to, since sigtop's CLI has no subcommand that + * emits per-message reaction/attachment-presence data as a flat row. + */ + +import type { RecordData } from "../../src/connector-runtime.ts"; + +/** + * The subset of Signal Desktop's `messageJSON` shape (Signal-Desktop repo: + * ts/model-types.d.ts) this connector needs. Every field is optional/absent + * in older schema-version rows — Signal Desktop has shipped this JSON + * envelope's contents incrementally over many releases, so a message row + * captured by an older Signal Desktop version may simply lack `reactions` + * or `editHistory` altogether. Absence is normal, not corruption. + */ +export interface SignalMessageJson { + attachments?: unknown[]; + editHistory?: unknown[]; + reactions?: Array<{ + emoji?: string; + fromId?: string; + targetTimestamp?: number; + }>; +} + +/** + * Parses a message row's raw `json` column text. Returns an empty object on + * anything that isn't valid JSON (missing column, corrupt row, a schema + * version whose `json` column holds something unexpected) rather than + * throwing — a message with unparseable JSON still has real `id`/`body`/ + * `sent_at` columns worth emitting; only the JSON-derived fields + * (has_attachments, is_edited, reactions) degrade to their empty defaults. + */ +export function parseMessageJson(raw: string | null | undefined): SignalMessageJson { + if (!raw) { + return {}; + } + try { + const parsed = JSON.parse(raw) as unknown; + return parsed && typeof parsed === "object" ? (parsed as SignalMessageJson) : {}; + } catch { + return {}; + } +} + +export interface SignalMessageRow { + body: string | null; + conversationId: string; + id: string; + json: string | null; + receivedAtMs: number | null; + sentAt: number | null; + sourceServiceId: string | null; + type: string | null; +} + +export interface SignalConversationRow { + e164: string | null; + groupId: string | null; + id: string; + name: string | null; + serviceId: string | null; + type: string | null; +} + +/** + * Signal's own timestamps (`sentAt`, `receivedAtMs`) are epoch + * milliseconds — unlike iMessage's Apple-epoch/nanosecond quirks, no + * offset or unit heuristic is needed. A missing/zero/non-finite value is + * absence, not 1970-01-01: callers must skip the row rather than fabricate + * a cursor position (same null-date-skip-not-fabricate rule as imessage's + * appleDateToIso — see index.ts's emitMessageRows for where that happens). + */ +export function signalEpochMsToIso(raw: number | null | undefined): string | null { + if (!raw) { + return null; + } + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + return null; + } + return new Date(n).toISOString(); +} + +export interface BuiltMessage { + record: RecordData; + /** Epoch-ms cursor value this row would advance the `sent_at` cursor to, or null if unusable. */ + sentAtMs: number | null; +} + +/** + * Signal message `id` is the row's own UUID; `conversation_id` is Signal's + * conversation UUID. `sender` is the row's `sourceServiceId` column as + * already resolved by index.ts's SQL: `LEFT JOIN conversations AS c ON + * m.sourceServiceId = c.serviceId`, selecting `c.id` — the sender's own + * canonical `conversations` row id, matching sigtop's own schema-version-88+ + * sender resolution (signal/message.go) rather than the raw ACI/PNI + * `sourceServiceId` UUID, so `sender` foreign-keys against this connector's + * own `conversations.id` (see index.ts's `messagesSelect` doc for why). + * `sent_at` is the record's cursor field: prefer + * `sentAt` (the originating client's send timestamp) and fall back to + * `receivedAtMs` only when `sentAt` is unusable, so a message with a + * genuine send time never cursors off a later receive time. + * `has_attachments`/`is_edited` are derived from the row's `json` blob + * (see `parseMessageJson`) since Signal Desktop does not expose either as + * a flat SQL column. + */ +export function buildMessageRecord(row: SignalMessageRow): BuiltMessage { + const json = parseMessageJson(row.json); + const sentAtMs = row.sentAt && row.sentAt > 0 ? row.sentAt : (row.receivedAtMs ?? null); + const sentAtIso = signalEpochMsToIso(row.sentAt) ?? signalEpochMsToIso(row.receivedAtMs); + return { + record: { + id: row.id, + conversation_id: row.conversationId, + sender: row.sourceServiceId ?? null, + sent_at: sentAtIso, + body: row.body ?? null, + type: row.type ?? null, + has_attachments: Array.isArray(json.attachments) && json.attachments.length > 0, + is_edited: Array.isArray(json.editHistory) && json.editHistory.length > 0, + }, + sentAtMs: sentAtIso === null ? null : sentAtMs, + }; +} + +/** + * Signal conversation `id` is the row's own UUID (private) or group id + * (group). `type` is Signal's own `private`/`group` discriminator. + * `title` prefers the conversation's own `name` (set for groups and + * user-renamed direct chats); falls back to `null` rather than guessing a + * contact display name from profile fields sigtop itself only assembles + * for its own text-export formatting, not something this connector + * reimplements. `member_count` is always null: Signal Desktop's own schema + * (verified against sigtop's recipient/conversation model, which this + * connector's SQL access mirrors) exposes no flat member-count column or + * field for group conversations — sigtop's own CLI never derives one + * either. Standing entity, no natural per-row date bound — same reasoning + * imessage applies to `participants`: full resnapshot every run, no + * incremental cursor. + */ +export function buildConversationRecord(row: SignalConversationRow): RecordData { + const type = row.type === "private" || row.type === "group" ? row.type : null; + return { + id: row.id, + type, + title: row.name ?? null, + member_count: null, + }; +} + +export interface SignalReactionInput { + emoji: string; + fromId: string; + messageId: string; +} + +/** + * Composite id `message_id:emoji:sender`, matching slack.reactions' + * `message_id:emoji:user_id` shape exactly (see slack/parsers.ts's + * buildReactionRecords). `sender` (`fromId`) is whatever recipient + * identifier Signal Desktop's own reaction JSON recorded — a conversation + * id, phone number, or service-id-shaped string depending on schema + * version (see sigtop's `recipientFromReactionID`, which itself branches + * on exactly this ambiguity) — so it is treated as an opaque bounded + * string, not assumed to be a UUID. + */ +export function buildReactionRecord(row: SignalReactionInput): RecordData { + return { + id: `${row.messageId}:${row.emoji}:${row.fromId}`, + message_id: row.messageId, + emoji: row.emoji, + sender: row.fromId, + }; +} + +/** + * Extracts every reaction embedded in a message row's `json` column, + * paired with the owning message's id — this is how index.ts fans a + * single `messages` row out into zero or more `reactions` records (Signal + * Desktop has no standalone `reactions` table; sigtop's own + * `parseReactionJSON` reads the identical `json.reactions` array). + * Reactions with a missing/empty emoji or fromId are dropped rather than + * emitted with a fabricated placeholder — both are required to build the + * composite id. + */ +export function extractReactionsFromMessageJson(messageId: string, json: SignalMessageJson): SignalReactionInput[] { + if (!Array.isArray(json.reactions)) { + return []; + } + const out: SignalReactionInput[] = []; + for (const r of json.reactions) { + if (r && typeof r.emoji === "string" && r.emoji.length > 0 && typeof r.fromId === "string" && r.fromId.length > 0) { + out.push({ emoji: r.emoji, fromId: r.fromId, messageId }); + } + } + return out; +} diff --git a/packages/polyfill-connectors/connectors/signal/schemas.test.ts b/packages/polyfill-connectors/connectors/signal/schemas.test.ts new file mode 100644 index 000000000..680205276 --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/schemas.test.ts @@ -0,0 +1,180 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Schema tests for the Signal connector. Ground truth is `parsers.ts`'s + * record builders (index.ts calls those, never builds a record literal + * itself) — these tests assert the schema against literal records shaped + * exactly as those builders produce them. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { attachmentsSchema, conversationsSchema, messagesSchema, reactionsSchema, validateRecord } from "./schemas.ts"; + +const MESSAGE_ID = "22222222-2222-2222-2222-222222222222"; +const CONVERSATION_ID = "11111111-1111-1111-1111-111111111111"; +const SENDER_ID = "33333333-3333-3333-3333-333333333333"; + +const MESSAGE_RECORD = { + id: MESSAGE_ID, + conversation_id: CONVERSATION_ID, + sender: SENDER_ID, + sent_at: "2024-06-05T13:22:02.000Z", + body: "hey there", + type: "incoming", + has_attachments: false, + is_edited: false, +}; + +test("messages schema accepts a fully-populated record", () => { + const result = messagesSchema.safeParse(MESSAGE_RECORD); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("messages schema accepts null sender/body/type", () => { + const result = messagesSchema.safeParse({ ...MESSAGE_RECORD, sender: null, body: null, type: null }); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("messages schema accepts a non-UUID sender (legacy e164/conversation-id-shaped value)", () => { + const result = messagesSchema.safeParse({ ...MESSAGE_RECORD, sender: "+15551234567" }); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("messages schema rejects a non-UUID id", () => { + assert.equal(messagesSchema.safeParse({ ...MESSAGE_RECORD, id: "not-a-uuid" }).success, false); +}); + +test("messages schema rejects a non-ISO sent_at", () => { + assert.equal(messagesSchema.safeParse({ ...MESSAGE_RECORD, sent_at: "1717594922000" }).success, false); +}); + +test("messages schema rejects a missing sent_at (null must never be emitted for this stream)", () => { + assert.equal(messagesSchema.safeParse({ ...MESSAGE_RECORD, sent_at: null }).success, false); +}); + +test("validateRecord routes messages and passes unknown streams through", () => { + assert.equal(validateRecord("messages", MESSAGE_RECORD).ok, true); + assert.equal(validateRecord("unknown_stream", { x: 1 }).ok, true); +}); + +// ─── conversations ──────────────────────────────────────────────────────── + +const CONVERSATION_RECORD = { + id: CONVERSATION_ID, + type: "private", + title: "Alice", + member_count: null, +}; + +test("conversations schema accepts a private conversation record", () => { + const result = conversationsSchema.safeParse(CONVERSATION_RECORD); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("conversations schema accepts a group conversation record with null title", () => { + const result = conversationsSchema.safeParse({ ...CONVERSATION_RECORD, type: "group", title: null }); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("conversations schema accepts a null type", () => { + const result = conversationsSchema.safeParse({ ...CONVERSATION_RECORD, type: null }); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("conversations schema rejects an unrecognized type literal", () => { + assert.equal(conversationsSchema.safeParse({ ...CONVERSATION_RECORD, type: "channel" }).success, false); +}); + +test("conversations schema rejects a non-UUID id", () => { + assert.equal(conversationsSchema.safeParse({ ...CONVERSATION_RECORD, id: "not-a-uuid" }).success, false); +}); + +// ─── reactions ──────────────────────────────────────────────────────────── + +const REACTION_RECORD = { + id: `${MESSAGE_ID}:👍:${SENDER_ID}`, + message_id: MESSAGE_ID, + emoji: "👍", + sender: SENDER_ID, +}; + +test("reactions schema accepts a well-formed reaction record", () => { + const result = reactionsSchema.safeParse(REACTION_RECORD); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("reactions schema accepts a non-UUID sender (legacy e164/conversation-id-shaped value)", () => { + const result = reactionsSchema.safeParse({ ...REACTION_RECORD, sender: "+15551234567" }); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("reactions schema rejects an empty emoji", () => { + assert.equal(reactionsSchema.safeParse({ ...REACTION_RECORD, emoji: "" }).success, false); +}); + +test("reactions schema rejects a non-UUID message_id", () => { + assert.equal(reactionsSchema.safeParse({ ...REACTION_RECORD, message_id: "not-a-uuid" }).success, false); +}); + +// ─── attachments ────────────────────────────────────────────────────────── + +const ATTACHMENT_HYDRATED = { + id: "a".repeat(64), + message_id: MESSAGE_ID, + conversation_id: CONVERSATION_ID, + filename: "IMG_0001.jpg", + content_type: "image/jpeg", + size_bytes: 4096, + content_sha256: "b".repeat(64), + hydration_status: "hydrated", + hydration_error: null, + blob_ref: { + blob_id: "blob_sha256_abc", + mime_type: "image/jpeg", + sha256: "b".repeat(64), + size_bytes: 4096, + }, +}; + +test("attachments schema accepts a hydrated record", () => { + const result = attachmentsSchema.safeParse(ATTACHMENT_HYDRATED); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("attachments schema accepts a deferred record with null blob_ref and null message_id/conversation_id (unjoined metadata)", () => { + const result = attachmentsSchema.safeParse({ + ...ATTACHMENT_HYDRATED, + hydration_status: "deferred", + blob_ref: null, + message_id: null, + conversation_id: null, + }); + assert.ok(result.success, JSON.stringify(result.error?.issues)); +}); + +test("attachments schema accepts missing/too_large/failed statuses with null size/hash", () => { + for (const status of ["missing", "too_large", "failed"] as const) { + const result = attachmentsSchema.safeParse({ + ...ATTACHMENT_HYDRATED, + hydration_status: status, + hydration_error: "synthetic failure", + blob_ref: null, + content_sha256: null, + size_bytes: null, + }); + assert.ok(result.success, `${status}: ${JSON.stringify(result.error?.issues)}`); + } +}); + +test("attachments schema rejects a non-hex id (would leak a raw local path)", () => { + assert.equal( + attachmentsSchema.safeParse({ ...ATTACHMENT_HYDRATED, id: "/home/tim/.config/Signal/x.jpg" }).success, + false + ); +}); + +test("attachments schema rejects an unknown hydration_status", () => { + assert.equal(attachmentsSchema.safeParse({ ...ATTACHMENT_HYDRATED, hydration_status: "bogus" }).success, false); +}); diff --git a/packages/polyfill-connectors/connectors/signal/schemas.ts b/packages/polyfill-connectors/connectors/signal/schemas.ts new file mode 100644 index 000000000..82d219ec8 --- /dev/null +++ b/packages/polyfill-connectors/connectors/signal/schemas.ts @@ -0,0 +1,135 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Zod schemas for Signal stream records. Shape-check-before-emit per + * docs/reference/connector-authoring-guide.md §3. + * + * Ground truth: `parsers.ts`'s `buildMessageRecord` / `buildConversationRecord` + * / `buildReactionRecord`, and index.ts's attachment emit literal. + * + * messages: { id, conversation_id, sender, sent_at, body, type, + * has_attachments, is_edited } + * conversations: { id, type, title, member_count } + * reactions: { id, message_id, emoji, sender } + * attachments: { id, message_id, conversation_id, filename, content_type, + * size_bytes, content_sha256, hydration_status, + * hydration_error, blob_ref } + * + * Shape notes: + * - `messages.id` / `conversations.id` are Signal's own row UUIDs — + * regex-constrained, unlike iMessage's permissive GUID-or-ROWID string + * (Signal's schema always carries a real UUID primary key, no numeric + * ROWID fallback). + * - `sender` (messages.sender, reactions.sender) is bounded free text, NOT + * assumed UUID-shaped: verified against sigtop's own source + * (signal/reaction.go's `recipientFromReactionID`), Signal Desktop's + * schema has carried a phone-number-prefixed id, a bare legacy + * conversation id, or a service-id UUID depending on schema version and + * record age — treating it as a UUID would silently skip real historical + * data from an older account. + * - `body` is free-form message text → pdppSafeText. + * - `sent_at` is always a real ISO string derived from the row's own + * epoch-ms value; a row with no usable timestamp is never emitted + * (index.ts skips it with a SKIP_RESULT instead of substituting the run + * clock — same discipline as imessage's `date`). + * - `type` is Signal's own message-type string (e.g. "incoming", + * "outgoing", "call-history") — open vocabulary, so pdppSafeText rather + * than a closed enum (Signal Desktop exposes it as free text, and new + * Signal message types have shipped over time). + * - `attachments.id` is a sha256 of the exported attachment's local path + * (never the raw path itself) — same never-leak-a-local-path invariant + * as imessage's attachments.id. + */ + +import { pdppSafeText } from "@pdpp/connector-protocol/pdpp-safe-text"; +import { z } from "zod"; +import { makeValidateRecord } from "../../src/schema-registry.ts"; + +// Module-scoped regexes (Biome useTopLevelRegex). +const ISO_DT_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/; +const UUID_RE = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/; +const ATTACHMENT_ID_RE = /^[0-9a-f]{64}$/; // sha256 hex of local path + +const isoDatetimeSchema = z.string().regex(ISO_DT_RE, "must be an ISO-8601 datetime"); +const uuidSchema = z.string().regex(UUID_RE, "must be a UUID"); + +const blobRefSchema = z + .object({ + blob_id: pdppSafeText.min(1), + mime_type: pdppSafeText.min(1), + sha256: pdppSafeText.min(1), + size_bytes: z.number().int().min(0), + }) + .nullable(); + +/** + * messages stream: one record per Signal message row. + * Cursor: sent_at (epoch-ms high-water mark tracked in STATE). + */ +export const messagesSchema = z.object({ + id: uuidSchema, + conversation_id: uuidSchema, + sender: pdppSafeText.max(320).nullable(), + sent_at: isoDatetimeSchema, + body: pdppSafeText.max(10_000_000).nullable(), + type: pdppSafeText.max(80).nullable(), + has_attachments: z.boolean(), + is_edited: z.boolean(), +}); + +/** + * conversations stream: one record per Signal conversation (direct or + * group). Semantics: mutable_state — full resnapshot each run, no + * incremental cursor (standing entity, no natural per-row date bound). + */ +export const conversationsSchema = z.object({ + id: uuidSchema, + type: z.enum(["private", "group"]).nullable(), + title: pdppSafeText.max(500).nullable(), + member_count: z.number().int().min(0).nullable(), +}); + +/** + * reactions stream: one record per (message, emoji, sender) reaction. + * Semantics: append_only. Composite id, matching slack.reactions' shape. + */ +export const reactionsSchema = z.object({ + id: z.string().min(1).max(600), + message_id: uuidSchema, + emoji: pdppSafeText.min(1).max(40), + sender: pdppSafeText.min(1).max(320), +}); + +/** + * attachments stream: one record per attachment exported by + * `sigtop export-attachments`. Bytes are hydrated via a local read bounded + * to a trusted root (sigtop's own output directory — see index.ts's + * resolveSafeAttachmentPath call, reusing imessage's O_NOFOLLOW primitive + * verbatim) + BlobRef upload; hydration_status/hydration_error report the + * outcome without leaking the local filesystem path. + */ +export const attachmentsSchema = z.object({ + id: z.string().regex(ATTACHMENT_ID_RE, "attachment id must be a sha256 hex digest"), + message_id: uuidSchema.nullable(), + conversation_id: uuidSchema.nullable(), + filename: pdppSafeText.min(1).max(500), + content_type: pdppSafeText.min(1).max(200), + size_bytes: z.number().int().min(0).nullable(), + content_sha256: pdppSafeText.nullable(), + hydration_status: z.enum(["deferred", "hydrated", "failed", "too_large", "missing"]), + hydration_error: pdppSafeText.nullable(), + blob_ref: blobRefSchema, +}); + +/** + * Stream → schema registry. Single source of truth for emitted streams. + */ +export const SCHEMAS: Record = { + messages: messagesSchema, + conversations: conversationsSchema, + reactions: reactionsSchema, + attachments: attachmentsSchema, +}; + +export const validateRecord = makeValidateRecord(SCHEMAS); diff --git a/packages/polyfill-connectors/connectors/slack/archive-enumeration.test.ts b/packages/polyfill-connectors/connectors/slack/archive-enumeration.test.ts new file mode 100644 index 000000000..dda2052df --- /dev/null +++ b/packages/polyfill-connectors/connectors/slack/archive-enumeration.test.ts @@ -0,0 +1,225 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The interrupted-enumeration trap. + * + * slackdump's `archive` enumerates the workspace and walks each channel; + * `resume` re-walks the channels the archive ALREADY HOLDS within a lookback + * window. `resume` never re-enumerates, so a channel that enumeration never + * reached is not in the archive for `resume` to find, and no number of + * resumes will ever request it. + * + * The connector chose between them on `existsSync(archivePath)` alone. That + * makes an interrupted enumeration permanent: the directory exists, so the + * next run resumes, so the unreached channels are still missing, so the + * directory still exists in the same incomplete shape. + * + * Measured on the owner's live archive before this fix: + * - SESSION 1, `MODE = 'archive'`, `FINISHED = 0` — died 16 minutes in. + * - It opened MESSAGES chunks for 5 channels. + * - 1360 `resume` sessions followed over three months. + * - The set of channels holding any MESSAGES chunk is STILL exactly those + * 5. `MIN(SESSION_ID)` over that set is 1 for every one of them. + * - 12 joined, unarchived channels hold zero chunks of any type. + * + * The shapes below are that archive's, reduced to the columns the decision + * reads. Channel ids are synthetic — the real ones are the owner's. + */ + +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { test } from "node:test"; +import { archivePathEnumerationIncomplete, pickResumeTarget } from "./index.ts"; + +// ─── Archive fixtures ──────────────────────────────────────────────────── + +function seedSessionSchema(db: DatabaseSync): void { + db.exec(` + CREATE TABLE SESSION ( + ID INTEGER PRIMARY KEY, CREATED_AT TEXT, UPDATED_AT TEXT, + PAR_SESSION_ID INTEGER, FROM_TS TEXT, TO_TS TEXT, + FINISHED INTEGER NOT NULL, FILES_ENABLED INTEGER, AVATARS_ENABLED INTEGER, + MODE TEXT NOT NULL, ARGS TEXT + ); + `); +} + +function insertSession(db: DatabaseSync, id: number, mode: string, finished: number): void { + db.prepare("INSERT INTO SESSION (ID, FINISHED, MODE, ARGS) VALUES (?, ?, ?, ?)").run( + id, + finished, + mode, + `${mode}|...` + ); +} + +async function withArchive( + seed: (db: DatabaseSync) => void, + body: (sqlitePath: string, archiveDir: string) => void | Promise +): Promise { + const dir = await mkdtemp(join(tmpdir(), "slack-enum-")); + try { + const sqlitePath = join(dir, "slackdump.sqlite"); + const db = new DatabaseSync(sqlitePath); + try { + seed(db); + } finally { + db.close(); + } + await body(sqlitePath, dir); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +// ─── archivePathEnumerationIncomplete ──────────────────────────────────── + +test("an archive whose only 'archive' session never finished still owes an enumeration", async () => { + // The live shape: SESSION 1 archive/FINISHED=0, then a long tail of + // finished resumes. The finished resumes must NOT satisfy the archive. + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "archive", 0); + for (let i = 2; i <= 40; i += 1) { + insertSession(db, i, "resume", 1); + } + }, + (sqlitePath) => { + assert.equal(archivePathEnumerationIncomplete(sqlitePath), true); + } + ); +}); + +test("an archive with a completed 'archive' session owes nothing", async () => { + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "archive", 1); + insertSession(db, 2, "resume", 1); + }, + (sqlitePath) => { + assert.equal(archivePathEnumerationIncomplete(sqlitePath), false); + } + ); +}); + +test("one completed 'archive' session settles the debt even after an earlier one was cut short", async () => { + // MAX(FINISHED) over the archive sessions: an interrupted first attempt + // followed by a completed one is a finished enumeration. + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "archive", 0); + insertSession(db, 2, "archive", 1); + }, + (sqlitePath) => { + assert.equal(archivePathEnumerationIncomplete(sqlitePath), false); + } + ); +}); + +test("an archive with resumes but no 'archive' session at all owes nothing", async () => { + // Absent evidence is not evidence of interruption. Forcing a multi-GB + // re-archive off a missing row is the same defect pointed the other way. + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "resume", 1); + }, + (sqlitePath) => { + assert.equal(archivePathEnumerationIncomplete(sqlitePath), false); + } + ); +}); + +test("an archive with no SESSION table at all owes nothing", async () => { + await withArchive( + (db) => { + db.exec("CREATE TABLE CHANNEL (ID TEXT NOT NULL, CHUNK_ID INTEGER NOT NULL);"); + }, + (sqlitePath) => { + assert.equal(archivePathEnumerationIncomplete(sqlitePath), false); + } + ); +}); + +test("a path with no archive on it owes nothing", () => { + assert.equal( + archivePathEnumerationIncomplete(join(tmpdir(), "slack-enum-does-not-exist", "slackdump.sqlite")), + false + ); +}); + +// ─── pickResumeTarget honors the debt ──────────────────────────────────── + +test("pickResumeTarget resumes a discovered archive when enumeration is complete", async () => { + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "archive", 1); + }, + (_sqlitePath, archiveDir) => { + const { resumeTarget } = pickResumeTarget({}, archiveDir, { forceFullArchive: false }); + assert.equal(resumeTarget, archiveDir, "a complete archive is cheap to resume and must be resumed"); + } + ); +}); + +test("pickResumeTarget refuses to resume when the enumeration is still owed", async () => { + // The defect, stated as a contract: an existing directory is NOT on its own + // a licence to resume. Before this fix `pickResumeTarget` returned + // `archiveDir` here, and that single value is what kept 12 of the owner's + // channels unreachable across 1360 runs. + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "archive", 0); + }, + (_sqlitePath, archiveDir) => { + const { resumeTarget } = pickResumeTarget({}, archiveDir, { forceFullArchive: true }); + assert.equal(resumeTarget, null, "an owed enumeration must run `archive`, not `resume`"); + } + ); +}); + +test("pickResumeTarget still reports priorArchive when it forces a full archive", async () => { + // Callers use `priorArchive` to tell a STATE-named archive from one merely + // found on disk. Choosing a different subcommand does not change that fact. + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "archive", 0); + }, + (_sqlitePath, archiveDir) => { + const state = { messages: { archive_dir: archiveDir } }; + const { resumeTarget, priorArchive } = pickResumeTarget(state, archiveDir, { forceFullArchive: true }); + assert.equal(resumeTarget, null); + assert.equal(priorArchive, archiveDir); + } + ); +}); + +test("pickResumeTarget ignores a STATE-named archive too when the enumeration is owed", async () => { + // The STATE branch was the other road to the same trap: a prior run that + // recorded `archive_dir` made every later run resume, whether or not the + // enumeration behind that directory ever finished. + await withArchive( + (db) => { + seedSessionSchema(db); + insertSession(db, 1, "archive", 0); + }, + (_sqlitePath, archiveDir) => { + const state = { messages: { archive_dir: archiveDir } }; + const { resumeTarget } = pickResumeTarget(state, archiveDir, { + allowStateArchive: true, + forceFullArchive: true, + }); + assert.equal(resumeTarget, null); + } + ); +}); diff --git a/packages/polyfill-connectors/connectors/slack/completeness-anchor.test.ts b/packages/polyfill-connectors/connectors/slack/completeness-anchor.test.ts new file mode 100644 index 000000000..04ad859b0 --- /dev/null +++ b/packages/polyfill-connectors/connectors/slack/completeness-anchor.test.ts @@ -0,0 +1,274 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Completeness-anchor tests for the Slack connector. + * + * Two independent concerns: + * + * 1. `unprovenChannelIds` — the set difference between the archive's own + * CHANNEL inventory and the channels slackdump proved it finished + * walking (`CHUNK.FINAL = 1` on the MESSAGES chunk type). This is the + * only per-channel completeness fact slackdump exposes; there is no + * per-channel message count and no `has_more` flag in its schema. + * + * Deliberately a SET comparison. A count cannot distinguish missing + * from surplus from duplicated, and — because PDPP retains records the + * source later deletes — a channel we hold history for that Slack no + * longer lists must never read as loss. + * + * 2. `emitMessagesPass`'s `covered` — previously the message family + * declared `covered: considered` unconditionally, so the coverage + * number could not fail. It is now counted per-row from the parse + * outcome: a row whose Slack `ts` will not parse gets a fabricated + * `sent_at` and must NOT raise the numerator. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import type { RecordData } from "../../src/connector-runtime.ts"; +import { emitMessagesPass, partitionUnprovenChannels, unprovenChannelIds } from "./index.ts"; +import type { MessageRow } from "./types.ts"; + +// ─── unprovenChannelIds: the set difference ────────────────────────────── + +test("unprovenChannelIds reports inventoried channels with no finalized walk", () => { + const inventory = new Set(["C1", "C2", "C3", "C4"]); + const finalized = new Set(["C1", "C3"]); + assert.deepEqual(unprovenChannelIds(inventory, finalized), ["C2", "C4"]); +}); + +test("unprovenChannelIds reports nothing when every inventoried channel finalized", () => { + const inventory = new Set(["C1", "C2"]); + assert.deepEqual(unprovenChannelIds(inventory, new Set(["C1", "C2"])), []); +}); + +test("unprovenChannelIds reports the whole inventory when nothing finalized", () => { + const inventory = new Set(["C1", "C2", "C3"]); + assert.deepEqual(unprovenChannelIds(inventory, new Set()), ["C1", "C2", "C3"]); +}); + +test("unprovenChannelIds ignores a finalized channel absent from the inventory (preservation, not loss)", () => { + // A channel we archived that Slack no longer lists. PDPP keeps what the + // source deletes, so this must produce NO finding — the comparison is + // one-directional by construction. + const inventory = new Set(["C1"]); + const finalized = new Set(["C1", "C_DELETED_UPSTREAM"]); + assert.deepEqual(unprovenChannelIds(inventory, finalized), []); +}); + +test("unprovenChannelIds returns a stable sorted order", () => { + const inventory = new Set(["C9", "C2", "C5"]); + assert.deepEqual(unprovenChannelIds(inventory, new Set()), ["C2", "C5", "C9"]); +}); + +test("unprovenChannelIds on an empty inventory reports nothing", () => { + assert.deepEqual(unprovenChannelIds(new Set(), new Set(["C1"])), []); +}); + +// ─── emitMessagesPass: covered is counted, not aliased ─────────────────── + +function messageRow(channelId: string, ts: string): MessageRow { + return { + CHANNEL_ID: channelId, + DATA: JSON.stringify({ text: "hi", user: "U1" }), + IS_PARENT: 0, + NUM_FILES: 0, + THREAD_TS: null, + TS: ts, + TXT: "hi", + }; +} + +function passDeps(): { deps: Parameters[0]; emitted: RecordData[] } { + const emitted: RecordData[] = []; + return { + emitted, + deps: { + emitRecord: (_stream: string, data: RecordData) => { + emitted.push(data); + return Promise.resolve(); + }, + emittedAt: "2026-08-20T00:00:00.000Z", + progress: () => Promise.resolve(), + requested: new Map([["messages", { name: "messages" }]]), + } as Parameters[0], + }; +} + +test("emitMessagesPass counts every parseable row as covered", async () => { + const { deps } = passDeps(); + const rows = [messageRow("C1", "1700000000.000100"), messageRow("C1", "1700000001.000200")]; + const result = await emitMessagesPass(deps, rows, null); + assert.equal(result.considered, 2); + assert.equal(result.covered, 2); +}); + +test("emitMessagesPass does NOT count an unparseable-ts row as covered", async () => { + // The row is still emitted (its body is real) but its sent_at is + // fabricated from the run clock, so it is not objectively accounted for. + const { deps } = passDeps(); + const rows = [messageRow("C1", "1700000000.000100"), messageRow("C1", "not-a-timestamp")]; + const result = await emitMessagesPass(deps, rows, null); + assert.equal(result.considered, 2, "both rows were weighed"); + assert.equal(result.covered, 1, "only the parseable row is covered"); + assert.ok(result.covered < result.considered, "a shortfall must read partial, not complete"); +}); + +test("emitMessagesPass does not count a zero ts as covered (Slack's unset, not 1970)", async () => { + const { deps } = passDeps(); + const result = await emitMessagesPass(deps, [messageRow("C1", "0")], null); + assert.equal(result.considered, 1); + assert.equal(result.covered, 0); +}); + +test("emitMessagesPass reports covered === considered only when every row parsed", async () => { + const { deps } = passDeps(); + const rows = ["1700000000.000100", "1700000001.000200", "1700000002.000300"].map((ts) => messageRow("C1", ts)); + const result = await emitMessagesPass(deps, rows, null); + assert.equal(result.covered, result.considered); + assert.equal(result.covered, 3); +}); + +test("emitMessagesPass on an empty row set proves an empty boundary", async () => { + const { deps } = passDeps(); + const result = await emitMessagesPass(deps, [], null); + assert.equal(result.considered, 0); + assert.equal(result.covered, 0); +}); + +test("emitMessagesPass still emits the record for an unparseable-ts row", async () => { + const { deps, emitted } = passDeps(); + await emitMessagesPass(deps, [messageRow("C1", "garbage")], null); + assert.equal(emitted.length, 1, "the row is preserved even though it is not covered"); +}); + +// ─── partitionUnprovenChannels: member scope, and ONLY member scope ────── +// +// `-member-only` filters on `is_member` and nothing else. slackdump v4.4.2, +// `internal/chunk/control/processors.go`: +// +// if c.memberOnly && !structures.IsMember(&ch) { continue } +// +// and `internal/structures/conversation.go`: +// +// if ChannelType(*ch) != CPublic || (ch.ID != "" && ch.ID[0] != 'C') { +// return true +// } +// return ch.IsMember +// +// `is_archived` is never read — not there, not anywhere in slackdump. Slack's +// `conversations.list` includes archived channels by default and slackdump +// never sends `exclude_archived`. So an ARCHIVED channel is collected exactly +// like a live one, and an unwalked archived channel is a REAL gap. +// +// This was previously inverted: archived was treated as out of scope, which +// told this owner his 95 archived channels were absent by design. Live proof +// they are collectable: his Aug-17 archive holds 15 archived channels, all 15 +// of which he is a member of, all 15 finalized, 16,173 messages collected — +// under `-member-only`. + +/** Order-insensitive comparison with an explicit comparator (Biome-clean). */ +function sorted(ids: readonly string[]): string[] { + return [...ids].sort((a, b) => a.localeCompare(b)); +} + +test("partitionUnprovenChannels puts a non-member public channel out of scope", () => { + const { inScope, outOfScope } = partitionUnprovenChannels( + ["C016HTUEMHD"], + new Map([["C016HTUEMHD", { isArchived: false, isMember: false }]]), + true + ); + assert.deepEqual(outOfScope, ["C016HTUEMHD"]); + assert.deepEqual(inScope, []); +}); + +test("partitionUnprovenChannels keeps an ARCHIVED joined channel IN scope", () => { + // The D8 regression guard. `-member-only` does not filter on is_archived, + // so an archived channel the account belongs to was requestable and its + // absence is an unexplained gap, not a configuration choice. + const { inScope, outOfScope } = partitionUnprovenChannels( + ["C016S03HPHU"], + new Map([["C016S03HPHU", { isArchived: true, isMember: true }]]), + true + ); + assert.deepEqual(inScope, ["C016S03HPHU"]); + assert.deepEqual(outOfScope, []); +}); + +test("partitionUnprovenChannels puts an archived NON-member public channel out of scope under member-only", () => { + // 91 of this owner's 94 archived channels are public and not joined: they + // are out of scope because of membership, never because of archiving. + const { inScope, outOfScope } = partitionUnprovenChannels( + ["C0ARCHNOMEM"], + new Map([["C0ARCHNOMEM", { isArchived: true, isMember: false }]]), + true + ); + assert.deepEqual(outOfScope, ["C0ARCHNOMEM"]); + assert.deepEqual(inScope, []); +}); + +test("partitionUnprovenChannels puts NOTHING out of scope when member-only is off", () => { + // With MEMBER_ONLY=false slackdump requests every enumerated channel, so + // there is no configuration excuse left for any unwalked channel. + const { inScope, outOfScope } = partitionUnprovenChannels( + ["C0ARCHNOMEM", "C016HTUEMHD"], + new Map([ + ["C0ARCHNOMEM", { isArchived: true, isMember: false }], + ["C016HTUEMHD", { isArchived: false, isMember: false }], + ]), + false + ); + assert.deepEqual(outOfScope, []); + assert.deepEqual(sorted(inScope), sorted(["C016HTUEMHD", "C0ARCHNOMEM"])); +}); + +test("partitionUnprovenChannels keeps a non-member DM/MPIM/private channel in scope", () => { + // slackdump's IsMember returns true for every non-`C` id regardless of the + // is_member flag, so member-only never explains a missing DM. + const { inScope, outOfScope } = partitionUnprovenChannels( + ["D01DIRECT01", "G01PRIVATE1"], + new Map([ + ["D01DIRECT01", { isArchived: false, isMember: false }], + ["G01PRIVATE1", { isArchived: true, isMember: false }], + ]), + true + ); + assert.deepEqual(sorted(inScope), sorted(["D01DIRECT01", "G01PRIVATE1"])); + assert.deepEqual(outOfScope, []); +}); + +test("partitionUnprovenChannels keeps a joined, unarchived channel in scope", () => { + // The genuinely unexplained bucket — a channel member-only archiving DID + // request and slackdump still did not finish. + const { inScope, outOfScope } = partitionUnprovenChannels( + ["C021ZPKLP7G"], + new Map([["C021ZPKLP7G", { isArchived: false, isMember: true }]]), + true + ); + assert.deepEqual(inScope, ["C021ZPKLP7G"]); + assert.deepEqual(outOfScope, []); +}); + +test("partitionUnprovenChannels treats a channel with NO reachability evidence as in scope", () => { + // Absent evidence must never downgrade a gap into "explained" — the same + // rule the finalized-set read follows when CHUNK is missing. + const { inScope, outOfScope } = partitionUnprovenChannels(["C0UNKNOWN01"], new Map(), true); + assert.deepEqual(inScope, ["C0UNKNOWN01"]); + assert.deepEqual(outOfScope, []); +}); + +test("partitionUnprovenChannels accounts for every unproven channel exactly once", () => { + const unproven = ["C0AAA", "C0BBB", "C0CCC", "C0DDD"]; + const { inScope, outOfScope } = partitionUnprovenChannels( + unproven, + new Map([ + ["C0AAA", { isArchived: false, isMember: false }], + ["C0BBB", { isArchived: true, isMember: true }], + ["C0CCC", { isArchived: false, isMember: true }], + ]), + true + ); + assert.equal(inScope.length + outOfScope.length, unproven.length); + assert.deepEqual([...inScope, ...outOfScope].sort(), [...unproven].sort()); +}); diff --git a/packages/polyfill-connectors/connectors/slack/emitted-watermark.test.ts b/packages/polyfill-connectors/connectors/slack/emitted-watermark.test.ts new file mode 100644 index 000000000..055bccbd7 --- /dev/null +++ b/packages/polyfill-connectors/connectors/slack/emitted-watermark.test.ts @@ -0,0 +1,296 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The Slack cursor invariant: a durable watermark records what was EMITTED, + * never what was merely ITERATED. + * + * Why this is a data-loss invariant and not a reporting nicety: the next + * run asks the archive for rows with `TS > cursor` (buildMessageRowsQuery). + * A row that raises the cursor without being emitted is therefore never + * fetched again and never stored — silent, permanent loss, with no gap or + * diagnostic to show for it. + * + * The two ways a walked row can fail to be emitted, both covered here: + * + * 1. The run is channel-scoped, so `emitMessageRecordScopedByChannel` + * drops rows for channels outside the scope. A scoped run reads the + * whole BASE archive, so it walks rows for every other channel in the + * workspace on its way past. + * 2. The `messages` stream is not requested at all (a reactions-only or + * attachments-only run still co-traverses the same MESSAGE rows). + * + * Plus the query-side half of the same rule: a channel with no committed + * cursor must start from zero, not inherit an unrelated global floor. + */ + +import assert from "node:assert/strict"; +import { DatabaseSync } from "node:sqlite"; +import { test } from "node:test"; +import type { RecordData } from "../../src/connector-runtime.ts"; +import { buildMessageRowsQuery, emitMessagesPass } from "./index.ts"; +import type { MessageRow } from "./types.ts"; + +function messageRow(channelId: string, ts: string): MessageRow { + return { + CHANNEL_ID: channelId, + DATA: JSON.stringify({ text: "hi", user: "U1" }), + IS_PARENT: 0, + NUM_FILES: 0, + THREAD_TS: null, + TS: ts, + TXT: "hi", + }; +} + +/** + * Mirrors the production wiring in `collect()`: `messages` records go + * through the channel-scope guard, which resolves `false` for a record + * outside the scope. Everything else is emitted unconditionally. + */ +function scopedDeps(inScope: ReadonlySet): { + deps: Parameters[0]; + emitted: RecordData[]; +} { + const emitted: RecordData[] = []; + return { + emitted, + deps: { + emitRecord: (stream: string, data: RecordData) => { + if (stream === "messages" && !(typeof data.channel_id === "string" && inScope.has(data.channel_id))) { + return Promise.resolve(false); + } + emitted.push(data); + return Promise.resolve(true); + }, + emittedAt: "2026-08-21T00:00:00.000Z", + progress: () => Promise.resolve(), + requested: new Map([["messages", { name: "messages" }]]), + } as Parameters[0], + }; +} + +// ─── 1. Dropped by the channel-scope guard ─────────────────────────────── + +test("a row dropped by the channel-scope guard does not advance that channel's watermark", async () => { + const { deps, emitted } = scopedDeps(new Set(["C_IN"])); + const result = await emitMessagesPass( + deps, + [messageRow("C_IN", "1700000000.000100"), messageRow("C_OUT", "1700000009.000900")], + null + ); + + assert.equal(emitted.length, 1, "only the in-scope row reached the runtime"); + assert.equal( + result.channelMaxTs.C_OUT, + undefined, + "the dropped channel must have NO durable watermark: a cursor here would make its history unreachable" + ); + assert.equal(result.channelMaxTs.C_IN, "1700000000.000100", "the emitted channel still advances"); +}); + +test("the global last_ts does not advance past a row the scope guard dropped", async () => { + // The dropped row carries the HIGHEST ts in the pass, so a watermark + // taken over iteration would commit the global floor to a message that + // was never stored. + const { deps } = scopedDeps(new Set(["C_IN"])); + const result = await emitMessagesPass( + deps, + [messageRow("C_IN", "1700000000.000100"), messageRow("C_OUT", "1700009999.000900")], + null + ); + + assert.equal(result.maxMessageTs, "1700000000.000100", "last_ts reflects the emitted max, not the iterated max"); +}); + +test("iterated-max stays observable, and separate from the durable cursor", async () => { + const { deps } = scopedDeps(new Set(["C_IN"])); + const result = await emitMessagesPass( + deps, + [messageRow("C_IN", "1700000000.000100"), messageRow("C_OUT", "1700009999.000900")], + null + ); + + assert.equal( + result.iteratedChannelMaxTs.C_OUT, + "1700009999.000900", + "progress reporting can still see the walked row" + ); + assert.equal(result.channelMaxTs.C_OUT, undefined, "but it must not leak into the durable cursor"); + assert.notDeepEqual(result.channelMaxTs, result.iteratedChannelMaxTs, "the two are genuinely distinct"); +}); + +test("a fully out-of-scope pass commits no cursor at all", async () => { + const { deps, emitted } = scopedDeps(new Set(["C_ELSEWHERE"])); + const result = await emitMessagesPass(deps, [messageRow("C_A", "1700000001.000000")], null); + + assert.equal(emitted.length, 0); + assert.deepEqual(result.channelMaxTs, {}, "nothing emitted means nothing committed"); + assert.equal(result.maxMessageTs, null); +}); + +test("considered/covered still count every walked row, emitted or not", async () => { + // Coverage accounting is deliberately NOT changed by the cursor fix: the + // rows really were weighed. Only the cursor is emission-gated. + const { deps } = scopedDeps(new Set(["C_IN"])); + const result = await emitMessagesPass( + deps, + [messageRow("C_IN", "1700000000.000100"), messageRow("C_OUT", "1700000009.000900")], + null + ); + + assert.equal(result.considered, 2); + assert.equal(result.covered, 2); +}); + +// ─── 2. `messages` not requested ───────────────────────────────────────── + +test("a reactions-only pass does not advance the messages cursor", async () => { + const emitted: RecordData[] = []; + const deps = { + emitRecord: (_stream: string, data: RecordData) => { + emitted.push(data); + return Promise.resolve(); + }, + emittedAt: "2026-08-21T00:00:00.000Z", + progress: () => Promise.resolve(), + requested: new Map([["reactions", { name: "reactions" }]]), + } as Parameters[0]; + + const result = await emitMessagesPass(deps, [messageRow("C1", "1700000000.000100")], null); + + assert.deepEqual( + result.channelMaxTs, + {}, + "no messages record was emitted, so the messages cursor must not move past this row" + ); + assert.equal(result.maxMessageTs, null); +}); + +// ─── 3. A void-returning emitRecord still counts as accepted ───────────── + +test("an emitRecord that resolves void is treated as accepted", async () => { + // Every non-scoping caller passes ctx.emitRecord straight through, which + // resolves void. Those runs must keep advancing their cursor normally. + const deps = { + emitRecord: () => Promise.resolve(), + emittedAt: "2026-08-21T00:00:00.000Z", + progress: () => Promise.resolve(), + requested: new Map([["messages", { name: "messages" }]]), + } as Parameters[0]; + + const result = await emitMessagesPass(deps, [messageRow("C1", "1700000000.000100")], null); + + assert.equal(result.channelMaxTs.C1, "1700000000.000100"); + assert.equal(result.maxMessageTs, "1700000000.000100"); +}); + +// ─── 4. The query-side half: no global floor for an unwalked channel ───── + +interface Seed { + channelId: string; + ts: string; +} + +function makeArchive(rows: readonly Seed[]): DatabaseSync { + const db = new DatabaseSync(":memory:"); + db.exec(` + CREATE TABLE MESSAGE ( + CHANNEL_ID TEXT NOT NULL, + TS TEXT NOT NULL, + THREAD_TS TEXT, + IS_PARENT INTEGER, + TXT TEXT, + NUM_FILES INTEGER, + DATA BLOB, + CHUNK_ID INTEGER NOT NULL + ); + `); + const stmt = db.prepare( + "INSERT INTO MESSAGE (CHANNEL_ID, TS, THREAD_TS, IS_PARENT, TXT, NUM_FILES, DATA, CHUNK_ID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ); + for (const r of rows) { + stmt.run(r.channelId, r.ts, null, 0, "hi", 0, "hi", 1); + } + return db; +} + +function selectRows(db: DatabaseSync, thresholds: Parameters[0]): string[] { + const { sql, params } = buildMessageRowsQuery(thresholds); + return db + .prepare(sql) + .all(...params) + .map((row) => `${String(row.CHANNEL_ID)}:${String(row.TS)}`) + .sort(); +} + +test("a channel absent from channel_last_ts does not inherit the global floor", () => { + // C_KNOWN has walked up to ...500. C_UNSEEN has never been walked, and + // all of its history predates that floor. Under the old + // `COALESCE(t.last_ts, legacy)` shape every C_UNSEEN row was suppressed + // forever — the query never returns it, so no cursor is ever written for + // it, so the same floor applies again on the next run. + const db = makeArchive([ + { channelId: "C_KNOWN", ts: "1700000600.000000" }, + { channelId: "C_UNSEEN", ts: "1700000100.000000" }, + { channelId: "C_UNSEEN", ts: "1700000200.000000" }, + ]); + try { + const got = selectRows(db, { + channelLastTs: { C_KNOWN: "1700000500.000000" }, + legacyLastTs: "1700000500.000000", + sinceTs: null, + }); + + assert.deepEqual( + got, + ["C_KNOWN:1700000600.000000", "C_UNSEEN:1700000100.000000", "C_UNSEEN:1700000200.000000"], + "the unwalked channel's full history is reachable; the known channel stays incremental" + ); + } finally { + db.close(); + } +}); + +test("a legacy cursor with NO per-channel map still floors every channel", () => { + // The pre-migration shape: one workspace-wide cursor, no per-channel + // rows. That floor was genuinely derived from a walk of everything, so it + // legitimately applies to every channel. Dropping it here would re-emit + // the entire archive on every run. + const db = makeArchive([ + { channelId: "C_A", ts: "1700000100.000000" }, + { channelId: "C_B", ts: "1700000900.000000" }, + ]); + try { + const got = selectRows(db, { + channelLastTs: {}, + legacyLastTs: "1700000500.000000", + sinceTs: null, + }); + + assert.deepEqual(got, ["C_B:1700000900.000000"], "the legacy floor still applies when there is no channel map"); + } finally { + db.close(); + } +}); + +test("collection_scope.since still composes with the per-channel predicate", () => { + // A declared `since` boundary is a different kind of claim from a cursor + // and must keep bounding an unwalked channel, which now has no cursor + // floor of its own. + const db = makeArchive([ + { channelId: "C_UNSEEN", ts: "1700000100.000000" }, + { channelId: "C_UNSEEN", ts: "1700000800.000000" }, + ]); + try { + const got = selectRows(db, { + channelLastTs: { C_KNOWN: "1700000500.000000" }, + legacyLastTs: null, + sinceTs: "1700000700.000000", + }); + + assert.deepEqual(got, ["C_UNSEEN:1700000800.000000"], "since bounds the unwalked channel even with no cursor"); + } finally { + db.close(); + } +}); diff --git a/packages/polyfill-connectors/connectors/slack/gap-streams.test.ts b/packages/polyfill-connectors/connectors/slack/gap-streams.test.ts index 4a445f780..a840e10f7 100644 --- a/packages/polyfill-connectors/connectors/slack/gap-streams.test.ts +++ b/packages/polyfill-connectors/connectors/slack/gap-streams.test.ts @@ -330,7 +330,7 @@ test("contrast: a REQUIRED stream's failure is NOT caught by runOptionalStream a const deps: StreamDeps = { db, emit: () => Promise.resolve(), - emitRecord: () => Promise.reject(new Error("emitRecord_boom")), + emitRecord: (): Promise => Promise.reject(new Error("emitRecord_boom")), emittedAt: "2026-07-10T00:00:00.000Z", fingerprintCursors: new Map(), progress: () => Promise.resolve(), diff --git a/packages/polyfill-connectors/connectors/slack/index.ts b/packages/polyfill-connectors/connectors/slack/index.ts index b308aa30e..d2d48a095 100755 --- a/packages/polyfill-connectors/connectors/slack/index.ts +++ b/packages/polyfill-connectors/connectors/slack/index.ts @@ -43,6 +43,11 @@ * SLACK_CHANNEL_ALLOWLIST (csv of channel IDs — maps to slackdump positional args) * SLACK_CHANNEL_TYPES (csv: public,private,im,mpim — default all four) * SLACK_MEMBER_ONLY (bool, default true — -member-only flag) + * Set false to archive EVERY channel the workspace + * lists, including public channels this account has + * left and channels Slack has archived. The flag + * filters on `is_member` alone; archived is a + * separate axis slackdump never filters on. * SLACK_SKIP_FILES (bool, default true) * * PDPP scope mapping: @@ -95,6 +100,7 @@ import { parseMessageRow, selectCommittedMaxTs, toSlackTime, + tsToIso, WORKSPACE_LIST_ARROW, } from "./parsers.ts"; import { validateRecord } from "./schemas.ts"; @@ -206,6 +212,182 @@ function missingPreviouslyObservedChannelIds( return priorObservedChannelIds.filter((id) => !current.has(id)).sort(); } +/** + * Channel ids slackdump proved it finished paginating. + * + * `CHUNK.FINAL` is slackdump's OWN end-of-pagination marker for a chunk + * (see its schema: "FINAL SMALLINT NOT NULL DEFAULT FALSE" alongside + * `NUM_REC`), and `TYPE_ID = 0` is the MESSAGES chunk type per the + * archive's `TYPES` table. A channel with a final messages chunk is one + * slackdump walked to the end; a channel with only non-final message + * chunks was cut short mid-walk. + * + * This is the provider-side completeness anchor for Slack messages. It is + * measured from the archive tool's own bookkeeping, not from anything this + * connector emitted, and it is the only per-channel completeness fact + * slackdump exposes — there is no per-channel message count and no + * `has_more` flag anywhere in the archive schema. + * + * Returns an empty set on an archive too old to carry `CHUNK` (the + * `safeAll` fallback). The caller treats an empty result as "cannot + * prove", never as "nothing is complete". + */ +function archiveFinalizedChannelIds(db: DatabaseSync): Set { + const rows = safeAll<{ id: string }>( + db, + ` + SELECT DISTINCT CHANNEL_ID AS id + FROM CHUNK + WHERE TYPE_ID = 0 AND FINAL = 1 AND CHANNEL_ID IS NOT NULL AND CHANNEL_ID != '' + ` + ); + return new Set(rows.map((r) => r.id)); +} + +/** + * Every channel id in the archive's own CHANNEL inventory — what Slack told + * slackdump this account can see, independent of how much of each channel + * was actually archived. + * + * This is the denominator side of the message-coverage set comparison: + * inventory minus finalized is the set of channels whose history is NOT + * proven complete. On this owner's workspace that difference is large (973 + * channels in PDPP's inventory against 552 with a finalized message chunk + * across all archives), and before this evidence existed it was entirely + * invisible. + */ +function archiveInventoryChannelIds(db: DatabaseSync): Set { + const rows = safeAll<{ id: string }>( + db, + ` + SELECT DISTINCT ID AS id + FROM CHANNEL + WHERE ID IS NOT NULL AND ID != '' + ` + ); + return new Set(rows.map((r) => r.id)); +} + +/** + * Channels present in the archive's inventory whose message history + * slackdump never proved it finished walking. + * + * Deliberately a SET difference over channel ids, not a count comparison. + * A count cannot distinguish "we are short N channels" from "we hold N + * extra" from "N are duplicated", and — critically for a preservation + * product — a channel we hold history for that Slack has since archived or + * deleted must NOT read as loss. This asks only the one-directional + * question: which channels does the SOURCE list that we cannot prove we + * finished? Channels we hold but Slack no longer lists never appear here. + */ +export function unprovenChannelIds(inventory: ReadonlySet, finalized: ReadonlySet): string[] { + return [...inventory].filter((id) => !finalized.has(id)).sort((a, b) => a.localeCompare(b)); +} + +/** Per-channel membership facts needed to classify an unproven channel. */ +export interface ChannelReachability { + isArchived: boolean; + isMember: boolean; +} + +/** + * Does `-member-only` cause slackdump to skip this channel? + * + * Mirrors slackdump's own filter exactly. In v4.4.2 + * (`internal/chunk/control/processors.go`): + * + * if c.memberOnly && !structures.IsMember(&ch) { continue } + * + * and `structures.IsMember` (`internal/structures/conversation.go`): + * + * if ChannelType(*ch) != CPublic || (ch.ID != "" && ch.ID[0] != 'C') { + * return true // member of any non-public channel by assumption + * } + * return ch.IsMember + * + * Two facts follow, and both were previously stated backwards in this file: + * + * 1. The filter reads ONLY `is_member`. `is_archived` is never consulted, + * here or anywhere else in slackdump. Archiving a channel does not + * remove the account's membership, so an archived channel the account + * belongs to IS walked under `-member-only` — verified on this owner's + * own Aug-17 archive, where all 15 archived channels were members and + * all 15 finished with 16,173 messages collected. + * 2. Only `C`-prefixed public channels can ever be skipped. DMs, MPIMs and + * private channels are unconditionally in scope. + * + * So `-member-only` excludes exactly: public channels the account has left or + * never joined. That is the real axis, and it is orthogonal to archived. + */ +export function memberOnlySkipsChannel(id: string, facts: ChannelReachability): boolean { + return id.startsWith("C") && !facts.isMember; +} + +/** + * Split unproven channels by whether this run could ever have walked them. + * + * `outOfScope` — the run's own configuration guaranteed slackdump would never + * request this channel's history, so a re-run changes nothing. Under + * `-member-only` that is precisely the public channels the account is not a + * member of (see `memberOnlySkipsChannel`). With member-only OFF, nothing is + * out of scope: slackdump requests every channel Slack enumerates, archived + * included. + * + * `inScope` — slackdump was allowed to walk it and still did not finish. This + * is the only genuinely unexplained bucket, and the only one a re-archive can + * close. + * + * Being ARCHIVED is deliberately NOT a reason to call a channel out of scope. + * Slack's `conversations.list` includes archived channels by default + * (`exclude_archived` defaults to false and slackdump never sets it), so an + * archived channel that went unwalked is a real gap that must be reported as + * one. Treating archived as "explained" is what buried this owner's 95 + * archived channels behind a message telling him they were absent by design. + * + * A channel missing from `reachability` is treated as IN scope: absent + * evidence must never silently downgrade a gap into "explained". + */ +export function partitionUnprovenChannels( + unproven: readonly string[], + reachability: ReadonlyMap, + memberOnly: boolean +): { inScope: string[]; outOfScope: string[] } { + const inScope: string[] = []; + const outOfScope: string[] = []; + for (const id of unproven) { + const facts = reachability.get(id); + if (memberOnly && facts && memberOnlySkipsChannel(id, facts)) { + outOfScope.push(id); + } else { + inScope.push(id); + } + } + return { inScope, outOfScope }; +} + +/** + * Membership/archived facts for every channel in the archive inventory, read + * from the newest CHUNK per channel (same latest-row join + * `currentDmMpimChannelIds` uses, so a stale early chunk cannot win). + */ +function archiveChannelReachability(db: DatabaseSync): Map { + const rows = safeAll( + db, + ` + SELECT c.ID AS id, c.DATA AS data + FROM CHANNEL c + JOIN (SELECT ID, MAX(CHUNK_ID) AS mx FROM CHANNEL GROUP BY ID) m + ON m.ID = c.ID AND m.mx = c.CHUNK_ID + ` + ); + const out = new Map(); + for (const r of rows) { + const d = parseBlob(r.data); + out.set(r.id, { isArchived: d.is_archived === true, isMember: d.is_member === true }); + } + return out; +} + async function emitMissingChannelDiagnostic( emit: CollectContext["emit"], missingChannelIds: readonly string[] @@ -234,6 +416,124 @@ async function emitMissingChannelDiagnostic( }); } +/** + * Emit the per-channel message-completeness evidence for this archive: the + * set of inventoried channels slackdump never proved it finished walking. + * + * Before this existed, a channel that the archive simply never visited was + * indistinguishable from a channel with no messages — an invisible hole + * (403 of this owner's 973 channels hold zero messages, 278 of them public). + * This converts that into durable, operator-visible evidence. + * + * Emits nothing when the archive carries no `CHUNK` bookkeeping at all: an + * archive that cannot report finality cannot prove anything is missing + * either, and inventing a gap from absent evidence is the same defect as + * inventing coverage from it. + */ +async function emitUnprovenChannelDiagnostic( + emit: CollectContext["emit"], + db: DatabaseSync, + messageFamilyRequested: boolean, + memberOnly: boolean +): Promise { + if (!messageFamilyRequested) { + return; + } + const inventory = archiveInventoryChannelIds(db); + const finalized = archiveFinalizedChannelIds(db); + if (finalized.size === 0) { + return; + } + const unproven = unprovenChannelIds(inventory, finalized); + if (unproven.length === 0) { + return; + } + const { inScope, outOfScope } = partitionUnprovenChannels(unproven, archiveChannelReachability(db), memberOnly); + // Split the in-scope gap by whether slackdump ever opened a messages chunk + // for the channel. Both are gaps, but they have different causes and + // different fixes, and reporting them as one bucket is what made this + // owner's real defect unreadable for months: "unproven history, a + // re-archive is required" was emitted every run while every run was + // choosing `resume`, which by construction could never close it. + const untouched = untouchedChannelIds(db); + const neverRequested = inScope.filter((id) => untouched.has(id)); + const startedUnfinished = inScope.filter((id) => !untouched.has(id)); + if (neverRequested.length > 0) { + const visibleIds = neverRequested.slice(0, MAX_MISSING_CHANNEL_IDS_IN_DIAGNOSTIC); + await emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "channel_history_never_requested", + message: + `${String(neverRequested.length)} in-scope Slack channel(s) hold no message data at all: the ` + + "archive's channel enumeration was cut short before it reached them, so their history has never been " + + "requested even once. This is recoverable — a full archive pass (not a resume) collects them.", + diagnostics: { + inventory_count: inventory.size, + finalized_count: finalized.size, + never_requested_count: neverRequested.length, + never_requested_channel_ids: visibleIds, + truncated: visibleIds.length < neverRequested.length, + }, + recovery_hint: { + action: "retry_by_runtime", + retryable: true, + }, + }); + } + if (startedUnfinished.length > 0) { + const visibleIds = startedUnfinished.slice(0, MAX_MISSING_CHANNEL_IDS_IN_DIAGNOSTIC); + await emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "channel_history_not_finalized", + message: + `Slack archive lists ${String(inventory.size)} channels but slackdump proved a finished message walk for only ` + + `${String(finalized.size)}; ${String(startedUnfinished.length)} in-scope channel(s) were walked ` + + "part-way and never finished. Their messages are partial. A further archive pass is required to close this.", + diagnostics: { + inventory_count: inventory.size, + finalized_count: finalized.size, + unproven_count: startedUnfinished.length, + unproven_channel_ids: visibleIds, + truncated: visibleIds.length < startedUnfinished.length, + }, + recovery_hint: { + action: "retry_by_runtime", + retryable: true, + }, + }); + } + if (outOfScope.length > 0) { + const visibleIds = outOfScope.slice(0, MAX_MISSING_CHANNEL_IDS_IN_DIAGNOSTIC); + await emit({ + type: "SKIP_RESULT", + stream: "messages", + reason: "channel_history_out_of_member_scope", + message: + `${String(outOfScope.length)} public Slack channel(s) were never walked because this account is not a member ` + + "of them. This is a setting you control: set the connector's Slack option MEMBER_ONLY to false and run a " + + "full archive to collect every channel the workspace lists for you, including channels you have left and " + + "channels that have been archived. Their messages are not lost — they have simply never been requested.", + diagnostics: { + inventory_count: inventory.size, + finalized_count: finalized.size, + out_of_scope_count: outOfScope.length, + out_of_scope_channel_ids: visibleIds, + truncated: visibleIds.length < outOfScope.length, + collect_by_setting: "SLACK_MEMBER_ONLY=false", + }, + recovery_hint: { + // `not_retriable` (not a free-form token) — RECOVERY_ACTIONS is a closed + // set in the runtime's gap normalizer; an unrecognized action is + // silently replaced by a regex guess over the reason/message text. + action: "not_retriable", + retryable: false, + }, + }); + } +} + function selectCommittedChannelLastTs( priorChannelLastTs: Record, runChannelMaxTs: Record @@ -247,20 +547,33 @@ function selectCommittedChannelLastTs( return out; } +/** + * Emits a `messages` record only when its channel is inside the run's + * resource scope, and REPORTS whether it did. + * + * The boolean is the whole point. This guard drops rows, and the durable + * per-channel cursor must never advance past a row this guard dropped: the + * next run asks the archive for `TS > cursor`, so a dropped row that moved + * the cursor is unreachable forever. Returning `void` here made that + * silent — the caller had no way to distinguish "emitted" from "swallowed" + * and so recorded a watermark for both. See `emitMessagesPass`, which + * threads this outcome into `channelMaxTs`. + */ async function emitMessageRecordScopedByChannel(deps: { channelIds: ReadonlySet; emitRecord: CollectContext["emitRecord"]; record: RecordData; -}): Promise { +}): Promise { if ( // biome-ignore lint/suspicious/noEqualsToNull: check for both null and undefined deps.record.id == null || typeof deps.record.channel_id !== "string" || !deps.channelIds.has(deps.record.channel_id) ) { - return; + return false; } await deps.emitRecord("messages", deps.record, { skipResourceFilter: true }); + return true; } interface SlackdumpProgressSnapshot { @@ -848,8 +1161,10 @@ function unionStrings(...values: ReadonlyArray): string[] { function mergeMessagesPassResults(left: MessagesPassResult, right: MessagesPassResult): MessagesPassResult { return { channelMaxTs: selectCommittedChannelLastTs(left.channelMaxTs, right.channelMaxTs), + iteratedChannelMaxTs: selectCommittedChannelLastTs(left.iteratedChannelMaxTs, right.iteratedChannelMaxTs), maxMessageTs: selectMaxSlackTs(left.maxMessageTs, right.maxMessageTs), considered: left.considered + right.considered, + covered: left.covered + right.covered, }; } @@ -1387,18 +1702,146 @@ async function mergeScopedMessageArchivePasses(deps: { return merged; } +/** + * Channel ids the archive's own inventory lists that slackdump never opened + * a single messages chunk for. + * + * Distinct from `unprovenChannelIds`, and the distinction is the whole point. + * "Unproven" spans two very different states: a channel slackdump STARTED and + * did not finish (non-final chunks exist — `resume` closes it), and a channel + * slackdump NEVER TOUCHED (no chunk of any type — `resume` will never reach + * it). `resume` walks the channels already recorded in the archive within a + * lookback window; it does not re-enumerate the workspace, so a channel absent + * from the archive stays absent no matter how many times it runs. + * + * On this owner's workspace the `archive` pass (SESSION 1) died 16 minutes in + * having opened messages chunks for 5 channels. The 1360 `resume` sessions + * that followed over the next three months never grew that set past 5 — the + * set of channels with any messages chunk is still exactly the 5 SESSION 1 + * reached. Twelve joined, unarchived channels have never been requested even + * once. + */ +function untouchedChannelIds(db: DatabaseSync): Set { + const rows = safeAll<{ id: string }>( + db, + ` + SELECT DISTINCT c.ID AS id + FROM CHANNEL c + WHERE c.ID IS NOT NULL AND c.ID != '' + AND NOT EXISTS ( + SELECT 1 FROM CHUNK k + WHERE k.TYPE_ID = 0 AND k.CHANNEL_ID = c.ID + ) + ` + ); + return new Set(rows.map((r) => r.id)); +} + +/** + * Whether this archive still owes a full `archive` enumeration. + * + * `resume` is the right tool for an archive whose enumeration finished: it + * carries every channel forward cheaply within its lookback. It is the WRONG + * tool for an archive whose enumeration never finished, because the channels + * that enumeration never reached are not in the archive for `resume` to walk. + * Choosing resume purely on `existsSync(archivePath)` — which is what this + * connector did — makes that state permanent: the directory exists, so every + * subsequent run resumes, so the missing channels are never requested, so the + * directory keeps existing in exactly the same incomplete shape. + * + * The archive records the fact needed to tell the two apart. slackdump writes + * a SESSION row per invocation with its own `FINISHED` flag and `MODE`. An + * archive whose `MODE = 'archive'` session never set `FINISHED = 1` is one + * whose enumeration was cut short, and it stays owed until an `archive` pass + * actually completes. + * + * Reads as "not owed" when the archive carries no SESSION bookkeeping at all. + * An archive that cannot report its own session state cannot prove it was + * interrupted either, and forcing a multi-GB re-archive off absent evidence is + * the same defect in the other direction. + */ +/** + * `archiveEnumerationIncomplete` for an archive on disk, by path. + * + * Opens read-only and always closes. A path that does not exist, or a file + * too damaged to open, reports `false` — same absent-evidence rule as the + * in-DB check: never force a multi-GB re-archive off a failure to read. + */ +export function archivePathEnumerationIncomplete(sqlitePath: string): boolean { + if (!existsSync(sqlitePath)) { + return false; + } + let db: DatabaseSync; + try { + db = new DatabaseSync(sqlitePath, { readOnly: true }); + } catch { + return false; + } + try { + return archiveEnumerationIncomplete(db); + } finally { + db.close(); + } +} + +function archiveEnumerationIncomplete(db: DatabaseSync): boolean { + const rows = safeAll<{ finished: number }>( + db, + ` + SELECT MAX(FINISHED) AS finished + FROM SESSION + WHERE MODE = 'archive' + ` + ); + const finished = rows[0]?.finished; + if (finished === undefined || finished === null) { + return false; + } + return Number(finished) !== 1; +} + +/** + * Whether the archive at `sqlitePath` still owes a full enumeration, saying so + * in the run log when it does. + * + * The disclosure matters as much as the decision. This run is about to spend a + * full `archive` pass instead of a cheap `resume`, and the owner's run log is + * the only place that choice — and the reason for it — is visible. + */ +function reportOwedEnumeration(sqlitePath: string, archivePath: string, progress: CollectContext["progress"]): boolean { + if (!archivePathEnumerationIncomplete(sqlitePath)) { + return false; + } + progress( + `Slack: the archive at ${archivePath} has no completed 'archive' session — its channel enumeration was cut ` + + "short, so channels it never reached hold no data and 'resume' would never request them. Running a full " + + "'archive' against the existing directory to finish the enumeration.", + { stream: "messages" } + ); + return true; +} + /** * Incremental via slackdump resume, full via archive. * Resume path: (a) explicit state.archive_dir from a prior successful run, * or (b) an archive directory already exists on disk from a timed-out or * crashed prior run. Resuming salvages partial progress — slackdump picks - * up from the last recorded chunk for each channel, so a previously-timed- - * out 1.1 GB archive turns into "finish the rest" rather than "restart". + * up from the last recorded chunk for each channel it already holds. + * + * `forceFullArchive` overrides both: an archive whose enumeration never + * completed (see `archiveEnumerationIncomplete`) must re-run `archive`, not + * resume, or the channels enumeration never reached stay unreachable forever. + * slackdump's `archive` is itself resumable against the same directory, so + * this finishes the interrupted enumeration rather than discarding the 4.8 GB + * already on disk. */ -function pickResumeTarget( +export function pickResumeTarget( state: CollectContext["state"], archivePath: string, - { allowStateArchive = true }: { allowStateArchive?: boolean } = {} + { + allowStateArchive = true, + forceFullArchive = false, + }: { allowStateArchive?: boolean; forceFullArchive?: boolean } = {} ): { resumeTarget: string | null; priorArchive: string | undefined } { // STATE is stream-keyed per Collection Profile: state is returned as // { : , ... }. We write `archive_dir` into the messages @@ -1406,6 +1849,12 @@ function pickResumeTarget( const messagesState = state.messages as MessagesState | undefined; const legacyArchiveDir = (state as Record).archive_dir as string | undefined; const priorArchive = messagesState?.archive_dir || legacyArchiveDir; // fallback for pre-fix state + if (forceFullArchive) { + // `priorArchive` is still reported: callers use it to distinguish an + // archive named by STATE from one merely discovered on disk, and that + // fact is unchanged by which subcommand we choose to run. + return { resumeTarget: null, priorArchive }; + } const discoveredArchive = existsSync(archivePath) ? archivePath : null; const resumeTarget = allowStateArchive && priorArchive && existsSync(priorArchive) ? priorArchive : discoveredArchive; return { resumeTarget, priorArchive }; @@ -1505,6 +1954,26 @@ async function runArchiveOrResume(deps: RunArchiveDeps): Promise { // ─── Cross-stream messages pass (sqlite-free, testable) ─────────────── +/** + * Emits one record, optionally reporting whether it was accepted. + * + * Resolving `false` means the record was deliberately dropped downstream + * and never reached the runtime. Resolving anything else — including no + * value at all, which is what every non-filtering caller does — means it + * landed. + * + * The reported outcome exists so that dropping a record is VISIBLE to the + * cursor logic. While the drop was silent, the messages pass advanced the + * durable watermark past rows it had not emitted, and the next run — which + * queries `TS > cursor` — could never fetch them again. + * + * Written as a union of two Promise types rather than `Promise`: it keeps `void` in return position (where it is not the + * confusing-union that Biome's noConfusingVoidType rejects) while letting + * the many existing `Promise` callbacks satisfy it unchanged. + */ +type EmitRecordFn = (stream: string, data: RecordData) => Promise | Promise; + /** * Subset of the per-stream dependency bag that the unified messages pass * actually needs. The sqlite-bound helpers in this file extend this with a @@ -1512,15 +1981,50 @@ async function runArchiveOrResume(deps: RunArchiveDeps): Promise { * without opening a DB. Mirrors the gmail/chase/usaa EmitDeps shape. */ export interface MessagesPassDeps { - emitRecord: (stream: string, data: RecordData) => Promise; + /** + * Only a record this reports as accepted may advance the emitting + * channel's durable cursor. See `EmitRecordFn`. + */ + emitRecord: EmitRecordFn; emittedAt: string; progress: CollectContext["progress"]; requested: CollectContext["requested"]; } export interface MessagesPassResult { + /** + * The DURABLE per-channel cursor contribution: the max Slack ts among + * rows this pass actually EMITTED and had accepted, per channel. A row + * that was iterated but dropped (out of channel scope, or `messages` not + * requested) contributes nothing here, because the next run refetches + * strictly above this value — advancing it past an unemitted row makes + * that row permanently unreachable. + * + * Distinct from `iteratedChannelMaxTs`, which is progress reporting only. + */ channelMaxTs: Record; considered: number; + /** + * Rows this pass actually accounted for: enumerated AND successfully + * shaped into a record. Measured per-row from the parse outcome, never + * aliased to `considered` — a row whose timestamp could not be parsed is + * counted in `considered` but not here, so it reads an honest `partial` + * instead of the tautological `complete` the prior `covered: considered` + * produced. + */ + covered: number; + /** + * The max Slack ts per channel among rows this pass WALKED, emitted or + * not. Observational: safe for progress/diagnostics, never durable. + * Kept separate from `channelMaxTs` so neither can be mistaken for the + * other at a call site. + */ + iteratedChannelMaxTs: Record; + /** + * Durable global cursor contribution: max Slack ts among EMITTED rows. + * Same rule as `channelMaxTs` — it is written to `messages.last_ts`, which + * the next run uses as a floor, so an unemitted row must not raise it. + */ maxMessageTs: string | null; } @@ -1547,7 +2051,18 @@ function recordChannelMaxTs(channelMaxTs: Record, channelId: str /** * Single-pass co-traversal of pre-loaded MESSAGE rows, emitting into * messages, reactions, and message_attachments streams as requested. - * Tracks maxMessageTs across every row for the post-loop STATE checkpoint. + * + * Cursor rule (the load-bearing invariant): the DURABLE watermarks + * (`maxMessageTs`, `channelMaxTs`) advance only for rows this pass actually + * emitted AND had accepted. Rows that were merely walked feed + * `iteratedChannelMaxTs`, which is observational only. + * + * Why the split exists: the archive query the next run issues is + * `TS > cursor`. A row that raises the cursor without being emitted is + * therefore never fetched again — silent, permanent loss. This pass walks + * rows for channels outside the run's scope (a scoped run reads the whole + * base archive), so "walked" and "emitted" genuinely differ, and conflating + * them lost data rather than merely mis-reporting it. * * Contract pinned by integration.test.ts: * - Per row, the `messages` record emits BEFORE its reactions and @@ -1555,11 +2070,11 @@ function recordChannelMaxTs(channelMaxTs: Record, channelId: str * - Scope gating is per-stream: disabling one of the three does not * suppress the other two — they share the pass but not the guard. * - When all three are disabled, the loop still runs (rows are iterated) - * but emits nothing; maxMessageTs still advances so the STATE - * checkpoint is accurate. This is the current pre-decomposition - * behavior: the caller guards entry to this function on + * but emits nothing, and the DURABLE watermarks stay put — an + * unemitted row must not be checkpointed as collected. Only + * `iteratedChannelMaxTs` moves. The caller guards entry on * `requested.has("messages" | "reactions" | "message_attachments")`, - * so in practice an all-disabled call is a harmless no-op. + * so an all-disabled call is a no-op in practice either way. * - A message with no reactions / no attachments still emits its * messages record; enrichment is additive, not gating. * - This function does not dedupe — dedup happens in `iterateMessageRows` @@ -1593,19 +2108,35 @@ export async function emitMessagesPass( const wantMsgAttachments = deps.requested.has("message_attachments"); const channelMaxTs: Record = {}; + const iteratedChannelMaxTs: Record = {}; let maxMessageTs: string | null = null; let considered = 0; + let covered = 0; for (const r of rows) { considered += 1; + // A row whose Slack `ts` will not parse gets a fabricated `sent_at` + // (parseMessageRow's `?? sentAtFallback`). It is still emitted — the + // body is real — but it is NOT objectively accounted for, so it must + // not raise the coverage numerator. Measured here, at the enumeration + // site, from the row's own parse outcome. + if (tsToIso(r.TS) !== null) { + covered += 1; + } const parsed = parseMessageRow(r, nowIso()); const { ts } = parsed; - // Track the max ts seen in this run for the post-loop STATE emit. - // Slack ts is a fixed-shape "seconds.micros" string; string compare - // matches numeric order because both halves are zero-padded by Slack. - maxMessageTs = selectMaxSlackTs(maxMessageTs, ts); - recordChannelMaxTs(channelMaxTs, r.CHANNEL_ID, ts); + // Observational max: every row we walked, emitted or not. Slack ts is a + // fixed-shape "seconds.micros" string; string compare matches numeric + // order because both halves are zero-padded by Slack. + recordChannelMaxTs(iteratedChannelMaxTs, r.CHANNEL_ID, ts); if (wantMessages) { - await deps.emitRecord("messages", buildMessageRecord(parsed)); + // The durable cursor advances HERE and only here — after the emit + // resolved and reported acceptance. A `void`-returning emitRecord + // (every non-scoping caller) counts as accepted. + const accepted = (await deps.emitRecord("messages", buildMessageRecord(parsed))) !== false; + if (accepted) { + maxMessageTs = selectMaxSlackTs(maxMessageTs, ts); + recordChannelMaxTs(channelMaxTs, r.CHANNEL_ID, ts); + } } if (wantReactions) { for (const rec of buildReactionRecords(parsed)) { @@ -1618,7 +2149,7 @@ export async function emitMessagesPass( } } } - return { channelMaxTs, maxMessageTs, considered }; + return { channelMaxTs, covered, iteratedChannelMaxTs, maxMessageTs, considered }; } // ─── Per-stream helpers ──────────────────────────────────────────────── @@ -1645,7 +2176,13 @@ export interface StreamDeps { * accidentally route here instead of `emitRecord`. */ emit: (msg: Extract) => Promise; - emitRecord: (stream: string, data: RecordData) => Promise; + /** + * Reports whether the record landed — today a `messages` record outside + * the run's channel scope is dropped and reports `false`. + * `emitMessagesPass` needs that to keep the durable cursor off unemitted + * rows. See `EmitRecordFn`. + */ + emitRecord: EmitRecordFn; emittedAt: string; fingerprintCursors: Map; progress: CollectContext["progress"]; @@ -1704,21 +2241,46 @@ async function declareListConsidered( ); } -async function declareMessageFamilyCoverage(deps: StreamDeps, considered: number): Promise { - for (const stream of ["reactions", "message_attachments"] as const) { - if (deps.requested.has(stream)) { - await deps.emit( - buildDetailCoverageMessage({ - stream, - stateStream: "messages", - requiredKeys: [], - hydratedKeys: [], - considered, - covered: considered, - }) - ); - } +/** + * Declares the messages self-coverage ONCE, using the fully-merged + * `considered` total (the base archive plus every scoped archive + * `mergeScopedMessageArchivePasses` folded in). A no-op when the message + * family wasn't requested this run. Called unconditionally, once per run, + * from `collect()` — never from inside `runRequestedStreams`, which runs once + * per scoped archive during a fold, and the runtime rejects a repeated + * (state_stream, stream) DETAIL_COVERAGE pair. + * + * `reactions` and `message_attachments` deliberately emit NO DETAIL_COVERAGE. + * The manifest declares both `state_stream: messages`, i.e. static + * single-parent detail streams, whose checkpoint status is projected from the + * parent's own commit outcome — so `validateDetailCoverageAgainstManifest` + * fails the ENTIRE run if either emits coverage of its own. + * + * Withholding is also the honest outcome on the numbers alone. The only counts + * in scope here are the PARENT message pass's: how many messages were walked, + * not how many reactions or attachments were derived against a per-key + * denominator. Reporting them under a child stream's name asserts + * `covered == considered` for children that were never accounted for — the + * fabricated-denominator anti-pattern this codebase has worked to remove. The + * children are left honestly unproven rather than falsely complete, exactly as + * `apple_contacts` withholds a contacts claim it cannot establish. + */ +async function declareMergedMessageCoverage(deps: StreamDeps, considered: number, covered: number): Promise { + if ( + !(deps.requested.has("messages") || deps.requested.has("reactions") || deps.requested.has("message_attachments")) + ) { + return; } + await deps.emit( + buildDetailCoverageMessage({ + stream: "messages", + stateStream: "messages", + requiredKeys: [], + hydratedKeys: [], + considered, + covered, + }) + ); } /** @@ -2020,11 +2582,37 @@ export function buildMessageRowsQuery(thresholds: MessageCursorThresholds): { pa // (the 6-digit suffix). This is exact and handles all epoch widths correctly. const dedupJoin = channelThresholds.length > 0 ? "LEFT JOIN thresholds t ON t.channel_id = m.CHANNEL_ID" : ""; let dedupWhere = ""; - if (channelThresholds.length > 0 && thresholds.legacyLastTs) { - dedupWhere = "WHERE m.TS > COALESCE(t.last_ts, ?)"; - params.push(thresholds.legacyLastTs); - } else if (channelThresholds.length > 0) { - dedupWhere = "WHERE t.last_ts IS NULL OR m.TS > t.last_ts"; + if (channelThresholds.length > 0) { + // A channel with NO row in `thresholds` has never had a cursor + // committed for it, so nothing about it has been walked. It therefore + // starts from zero (fetch its full history), NOT from the global + // `legacyLastTs` floor — which is why `legacyLastTs` is deliberately + // NOT consulted on this branch even when it is set. + // + // The prior shape was `m.TS > COALESCE(t.last_ts, ?)`, which handed an + // unwalked channel an unrelated global floor derived from OTHER + // channels' progress. Every message in that channel older than the + // floor was then permanently unreachable: the query never returns it, + // so it is never emitted, so no cursor is ever written for it, so the + // next run applies the same floor again. The cursor committed past data + // it had never processed — the exact failure this connector's cursor + // rule forbids. It suppressed no rows on this owner's archive only + // because every channel present there already had a cursor row; that + // made it latent, not safe. + // + // Cost of starting at zero is bounded and one-time: the channel is + // walked in full once, after which it has its own row here and rejoins + // the incremental path. Correctness is not traded for that. + // + // `legacyLastTs` still applies on the branch below, where there is no + // per-channel map at all: that is a pre-migration cursor covering the + // whole workspace uniformly, so it floors every channel legitimately. + // + // Parenthesized as one clause. SQL binds AND tighter than OR, so a bare + // `a IS NULL OR ts > a` composed with `AND ` would parse as + // `a IS NULL OR (ts > a AND )` — letting an unwalked channel + // escape the since boundary entirely. + dedupWhere = "WHERE (t.last_ts IS NULL OR m.TS > t.last_ts)"; } else if (thresholds.legacyLastTs) { dedupWhere = "WHERE m.TS > ?"; params.push(thresholds.legacyLastTs); @@ -2592,7 +3180,13 @@ export async function runRequestedStreams( await runUsersStream(deps); } // Messages, reactions, message_attachments share one pass for efficiency. - let result: MessagesPassResult = { channelMaxTs: {}, maxMessageTs: null, considered: 0 }; + let result: MessagesPassResult = { + channelMaxTs: {}, + covered: 0, + iteratedChannelMaxTs: {}, + maxMessageTs: null, + considered: 0, + }; if (deps.requested.has("messages") || deps.requested.has("reactions") || deps.requested.has("message_attachments")) { const messagesState = state.messages as MessagesState | undefined; const priorTs = options.allowLegacyMessageCursorFallback === false ? null : (messagesState?.last_ts ?? null); @@ -2605,24 +3199,12 @@ export async function runRequestedStreams( legacyLastTs: priorTs, sinceTs: options.sinceTs ?? null, }); - // One archive traversal supplies the parent denominator. Reactions and - // attachments ride this checkpoint window via manifest state_stream and - // must not receive a fabricated child-row denominator. - await deps.emit( - buildDetailCoverageMessage({ - stream: "messages", - stateStream: "messages", - requiredKeys: [], - hydratedKeys: [], - considered: result.considered, - covered: result.considered, - }) - ); - // Reactions and message attachments are derived from the same retained - // MESSAGE rows. Declare that archive enumeration as their measured - // checkpoint boundary too; never use the child-record counts, which are - // not the boundary this pass enumerates. - await declareMessageFamilyCoverage(deps, result.considered); + // The messages/reactions/message_attachments DETAIL_COVERAGE is NOT + // emitted here: a scoped-archive fold calls this function once per + // archive (mergeScopedMessageArchivePasses), and the runtime rejects a + // repeated (state_stream, stream) DETAIL_COVERAGE pair. The caller emits + // coverage once, after every archive this run touches has been folded + // into a single merged `considered` total. } if (deps.requested.has("files")) { deps.progress("Slack: emitting files", { stream: "files" }); @@ -2757,8 +3339,13 @@ if (isMainModule(import.meta.url)) { const { archivePath, sqlitePath } = resolveScopedArchivePaths(baseArchivePaths, positionalChannels); await mkdir(dumpDir, { recursive: true }); + // An archive whose `archive` enumeration never finished still owes one. + // Resuming it can only ever re-walk the channels enumeration already + // reached, so the ones it never reached would stay missing forever. + const enumerationIncomplete = reportOwedEnumeration(sqlitePath, archivePath, progress); const { resumeTarget, priorArchive } = pickResumeTarget(state, archivePath, { allowStateArchive: isUnscopedMessageBoundary, + forceFullArchive: enumerationIncomplete, }); const useResume = Boolean(resumeTarget); const messagesState = state.messages as MessagesState | undefined; @@ -2889,6 +3476,14 @@ if (isMainModule(import.meta.url)) { await emitMissingChannelDiagnostic(emit, reconciledSourceCache.missingChannelIds); } + // The existing diagnostic above compares the archive against this + // connector's OWN prior state, so a channel never archived in the + // first place is invisible to it forever. This one compares the + // archive's inventory against slackdump's own per-channel + // end-of-pagination marker — a source-side fact — and so surfaces + // exactly that never-visited hole. + await emitUnprovenChannelDiagnostic(emit, db, messageFamilyRequested, opts.MEMBER_ONLY); + // Register the opt-in __uploads reclaim once every archive this run // actually read is known: the base/scoped archive, every scoped archive // reconcileMessageSourceCache refreshed or repaired AND folded into the @@ -2910,6 +3505,20 @@ if (isMainModule(import.meta.url)) { ] : null; + // Everything from here through mergeScopedMessageArchivePasses below + // reads only the already-downloaded local sqlite archive(s) and posts + // to this run's own ingest endpoint — no further slackdump subprocess, + // no further Slack API call, no provider rate limit. `maxRunWallClockMs` + // (run-executor.ts) is sized for the external walk that already + // finished above; this marker tells the scheduler watchdog to stop + // applying it for the remainder of the attempt, so a large local + // archive being read into the store is not truncated as if it were + // still rate-limited by Slack. See run_1787407222861: slackdump had + // archived 1,066,135 messages to disk and only this local read-and-emit + // pass was in flight when the external-walk ceiling killed the run. + progress("Slack: external archive walk complete; beginning local archive read", { + phase_boundary: "local_only_phase_started", + }); let messageResult = await timedPhase(progress, "read-and-emit", () => runRequestedStreams(deps, state, { workspace, token, cookie }, emit, { allowLegacyMessageCursorFallback: isUnscopedMessageBoundary, @@ -2929,6 +3538,8 @@ if (isMainModule(import.meta.url)) { }); } + await declareMergedMessageCoverage(deps, messageResult.considered, messageResult.covered); + // Drop fingerprint entries for IDs that disappeared from the source // since the prior run on streams we actually requested. Streams the // caller did not exercise keep their full carry-forward — an diff --git a/packages/polyfill-connectors/connectors/slack/integration.test.ts b/packages/polyfill-connectors/connectors/slack/integration.test.ts index c1ee4525c..97ac87a0d 100644 --- a/packages/polyfill-connectors/connectors/slack/integration.test.ts +++ b/packages/polyfill-connectors/connectors/slack/integration.test.ts @@ -261,29 +261,29 @@ test("runRequestedStreams: archive message enumeration bounds both derived strea requested, }; + let result: Awaited>; try { - await runRequestedStreams(deps, {}, {} as Parameters[2], harness.emit); + result = await runRequestedStreams(deps, {}, {} as Parameters[2], harness.emit); } finally { db.close(); } - const coverage = harness.protocolMessages - .filter((message) => message.type === "DETAIL_COVERAGE") - .map((message) => ({ - stream: message.stream, - stateStream: message.state_stream, - considered: message.considered, - covered: message.covered, - })); - assert.deepEqual( - coverage, - [ - { stream: "messages", stateStream: "messages", considered: 2, covered: 2 }, - { stream: "reactions", stateStream: "messages", considered: 2, covered: 2 }, - { stream: "message_attachments", stateStream: "messages", considered: 2, covered: 2 }, - ], - "each derived stream uses the two retained MESSAGE rows as its measured boundary, not child counts" + // runRequestedStreams itself no longer emits the messages/reactions/ + // message_attachments DETAIL_COVERAGE (see + // openspec/changes/fix-slack-scoped-archive-coverage-duplication): a + // scoped-archive fold calls this function once per archive, and emitting + // here made every call after the first a duplicate (state_stream, stream) + // pair the runtime rejects. The caller now emits once, after every archive + // is folded into one merged total. What's still true here, and what this + // test asserts: the two retained MESSAGE rows are the boundary BOTH + // derived streams' eventual coverage is measured against, carried on the + // returned `considered`, not the emitted child-record counts (3 reactions, + // 2 attachments). + assert.equal( + harness.protocolMessages.some((message) => message.type === "DETAIL_COVERAGE"), + false ); + assert.equal(result.considered, 2, "considered is the two retained MESSAGE rows, not the child counts"); }); // ─── Invariant 7a: parent-before-child within a single row ─────────────── @@ -354,8 +354,15 @@ test("emitMessagesPass: messages disabled — reactions + attachments still flow test("emitMessagesPass: all three streams disabled — no records emit, rows still iterate", async () => { // Production caller guards entry on `requested.has("messages" | ...)`, // so this is the defense-in-depth contract: if called with none of the - // three requested, the loop runs silently. maxMessageTs still advances - // so a STATE checkpoint written by the caller stays correct. + // three requested, the loop runs silently. + // + // The DURABLE cursor must NOT advance here. This assertion previously + // required the opposite ("ts tracking still advances") on the reasoning + // that it kept the caller's STATE checkpoint accurate — but a checkpoint + // over rows that were never emitted is precisely what makes them + // unreachable on the next run, whose query is `TS > cursor`. Accurate + // meant "matches what we collected", and we collected nothing. + // The walked ts stays visible via `iteratedChannelMaxTs`. const { deps, emitted } = makeHarness({ requested: ["channels"] }); const row = makeRow( {}, @@ -366,7 +373,12 @@ test("emitMessagesPass: all three streams disabled — no records emit, rows sti ); const result = await emitMessagesPass(deps, [row], null); assert.equal(emitted.length, 0, "no records emit when no relevant stream requested"); - assert.equal(result.maxMessageTs, "1700000000.000100", "ts tracking still advances"); + assert.equal(result.maxMessageTs, null, "an unemitted row must not raise the durable cursor"); + assert.equal( + result.iteratedChannelMaxTs[row.CHANNEL_ID], + "1700000000.000100", + "the walked ts stays observable for progress reporting" + ); }); // ─── Invariant 4: null/missing enrichment fallback ─────────────────────── diff --git a/packages/polyfill-connectors/connectors/slack/message-query-incremental.test.ts b/packages/polyfill-connectors/connectors/slack/message-query-incremental.test.ts index 1d71f5c36..951c7ac75 100644 --- a/packages/polyfill-connectors/connectors/slack/message-query-incremental.test.ts +++ b/packages/polyfill-connectors/connectors/slack/message-query-incremental.test.ts @@ -70,11 +70,18 @@ function referenceQuery(thresholds: Thresholds): { params: string[]; sql: string : ""; const join = channelThresholds.length > 0 ? "LEFT JOIN thresholds t ON t.channel_id = m.CHANNEL_ID" : ""; let where = ""; - if (channelThresholds.length > 0 && thresholds.legacyLastTs) { - where = "WHERE m.TS > COALESCE(t.last_ts, ?)"; - params.push(thresholds.legacyLastTs); - } else if (channelThresholds.length > 0) { - where = "WHERE t.last_ts IS NULL OR m.TS > t.last_ts"; + if (channelThresholds.length > 0) { + // A channel with no cursor row starts from zero. It must NOT inherit + // `legacyLastTs`, which is a floor derived from OTHER channels' walks: + // the old `COALESCE(t.last_ts, ?)` made every message below that floor + // permanently unreachable in an unwalked channel (no row returned → no + // cursor written → same floor next run). + // + // This reference exists to pin the dedup-CTE rewrite as emit-identical + // to the pre-rewrite QUERY SHAPE, independently of the code under test. + // It is not a second opinion on cursor POLICY, so it tracks the policy + // rather than freezing the defect. + where = "WHERE (t.last_ts IS NULL OR m.TS > t.last_ts)"; } else if (thresholds.legacyLastTs) { where = "WHERE m.TS > ?"; params.push(thresholds.legacyLastTs); @@ -129,7 +136,7 @@ const SHAPES: Array<{ name: string; thresholds: Thresholds }> = [ thresholds: { channelLastTs: { C1: "100.000001", C2: "250.000001" }, legacyLastTs: null, sinceTs: null }, }, { - name: "per-channel + legacy fallback", + name: "per-channel cursors alongside a legacy global cursor", thresholds: { channelLastTs: { C1: "100.000001" }, legacyLastTs: "180.000000", sinceTs: null }, }, ]; @@ -144,6 +151,28 @@ for (const shape of SHAPES) { }); } +test("a channel with no cursor row is walked in full, not floored by the legacy cursor", () => { + // Asserted against literal expected rows rather than the reference, so + // this policy cannot silently follow the reference if that is edited. + // C2 has no cursor row; both of its timestamps sit BELOW legacyLastTs. + const db = makeArchive(FIXTURE); + const rows = runRows( + db, + buildMessageRowsQuery({ channelLastTs: { C1: "100.000001" }, legacyLastTs: "180.000000", sinceTs: null }) + ); + assert.deepEqual( + rows, + [ + "C1|200.000001|c1-200-v2-latest", + "C1|300.000001|c1-300-only", + "C2|150.000001|c2-150-only", + "C2|250.000001|c2-250-v2-latest", + ], + "C2:150 is below the legacy floor and would be unreachable forever if the floor applied to it" + ); + db.close(); +}); + test("no cursor emits every unique (channel, ts) with the latest chunk's DATA", () => { const db = makeArchive(FIXTURE); const rows = runRows(db, buildMessageRowsQuery({ channelLastTs: {}, legacyLastTs: null, sinceTs: null })); diff --git a/packages/polyfill-connectors/connectors/slack/scoped-archive-coverage.test.ts b/packages/polyfill-connectors/connectors/slack/scoped-archive-coverage.test.ts new file mode 100644 index 000000000..d763d36d7 --- /dev/null +++ b/packages/polyfill-connectors/connectors/slack/scoped-archive-coverage.test.ts @@ -0,0 +1,228 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Proves the scoped-archive message-family DETAIL_COVERAGE duplication bug + * (production: 8 Slack runs, 0 successes — see + * openspec/changes/fix-slack-scoped-archive-coverage-duplication). + * + * `reconcileMessageSourceCache` heals a base archive missing channels a prior + * run observed by pulling each missing channel's existing scoped archive + * (archive-scoped//) — see the single-scoped-archive case in + * archive-reclaim.test.ts. When 2+ DISTINCT scoped archives are needed (each + * covering a different missing channel — production sees 3), + * `mergeScopedMessageArchivePasses`'s `for (const archive of + * deps.scopedArchives)` loop calls `runRequestedStreams` once per archive. + * + * Before the fix, `runRequestedStreams` itself emitted the messages + * self-coverage DETAIL_COVERAGE (state_stream=messages, stream=messages) on + * EVERY call — so N archives meant N emissions of the pair through the same + * emit side-channel. + * reference-implementation/runtime/index.ts's `trackDetailCoverage` rejects + * any repeated (state_stream, stream) pair ("Connector emitted duplicate + * DETAIL_COVERAGE for state_stream=messages stream=messages"), so a + * production run touching 2+ scoped archives never completed. + * + * This test seeds a base archive missing TWO channels, each recoverable from + * its own separate pre-existing scoped archive, drives the real connector + * (PDPP_SLACK_SKIP_SLACKDUMP=1 — no real slackdump subprocess) via + * `runConnectorProtocolSubprocess`, and asserts on the RAW emitted message + * sequence: each (state_stream, stream) DETAIL_COVERAGE pair must appear + * EXACTLY ONCE, with `considered` equal to the summed row count across the + * base archive + both scoped archives (mergeMessagesPassResults sums + * `considered`). + */ + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { resolveConnectorArtifactDir } from "../../src/connector-artifact-root.ts"; +import type { EmittedMessage } from "../../src/connector-runtime.ts"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PACKAGE_ROOT = resolve(__dirname, "../.."); +const SLACK_ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "slack", "index.ts"); + +function seedArchiveRoot(artifactRoot: string, workspace: string): string { + return resolveConnectorArtifactDir("slack", [workspace], { + PDPP_CONNECTOR_ARTIFACT_ROOT: artifactRoot, + }).root; +} + +/** Mirrors archive-reclaim.test.ts's digest helper: the connector selects a + * scoped archive by hashing its sorted covered-channel-id set. */ +async function scopedArchiveDigest(channels: readonly string[]): Promise { + const { createHash } = await import("node:crypto"); + return createHash("sha256") + .update(JSON.stringify([...new Set(channels)].sort())) + .digest("hex") + .slice(0, 12); +} + +function seedArchiveSchema(db: DatabaseSync): void { + db.exec(` + CREATE TABLE CHANNEL (ID TEXT NOT NULL, NAME TEXT, DATA TEXT, CHUNK_ID INTEGER NOT NULL); + CREATE TABLE MESSAGE ( + CHANNEL_ID TEXT NOT NULL, TS TEXT NOT NULL, THREAD_TS TEXT, IS_PARENT INTEGER, + TXT TEXT, NUM_FILES INTEGER, DATA BLOB, CHUNK_ID INTEGER NOT NULL + ); + `); +} + +function insertChannel(db: DatabaseSync, id: string, name: string): void { + db.prepare("INSERT INTO CHANNEL (ID, NAME, DATA, CHUNK_ID) VALUES (?, ?, ?, ?)").run( + id, + name, + JSON.stringify({ is_channel: true, is_member: true, name }), + 1 + ); +} + +function insertMessages(db: DatabaseSync, channelId: string, count: number, tsPrefix: string): void { + const stmt = db.prepare( + "INSERT INTO MESSAGE (CHANNEL_ID, TS, THREAD_TS, IS_PARENT, TXT, NUM_FILES, DATA, CHUNK_ID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ); + for (let i = 0; i < count; i += 1) { + const ts = `${tsPrefix}${String(i).padStart(6, "0")}`; + const text = `message ${String(i)}`; + stmt.run(channelId, ts, null, 1, text, null, new TextEncoder().encode(JSON.stringify({ text, user: "U1" })), 1); + } +} + +function detailCoverageMessages(result: { messages: EmittedMessage[] }) { + return result.messages.filter( + (m): m is Extract => m.type === "DETAIL_COVERAGE" + ); +} + +test("scoped-archive fold across 2+ archives: each (state_stream, stream) DETAIL_COVERAGE pair emits exactly once, with the summed considered", async () => { + const artifactRoot = await mkdtemp(join(tmpdir(), "pdpp-slack-scoped-coverage-")); + try { + const workspace = "scoped-coverage-ws"; + const baseArchiveDir = join(seedArchiveRoot(artifactRoot, workspace), "archive"); + const scopedDigestA = await scopedArchiveDigest(["C0MISSING_A"]); + const scopedDigestB = await scopedArchiveDigest(["C0MISSING_B"]); + const scopedArchiveDirA = join(seedArchiveRoot(artifactRoot, workspace), "archive-scoped", scopedDigestA); + const scopedArchiveDirB = join(seedArchiveRoot(artifactRoot, workspace), "archive-scoped", scopedDigestB); + await mkdir(baseArchiveDir, { recursive: true }); + await mkdir(scopedArchiveDirA, { recursive: true }); + await mkdir(scopedArchiveDirB, { recursive: true }); + + // Base archive: only the still-present channel. C0MISSING_A/B disappeared + // from it (e.g. slackdump export scope drifted) but a prior run's state + // still lists them as observed, forcing reconcileMessageSourceCache to + // heal both from their own separate scoped archives. + const baseDb = new DatabaseSync(join(baseArchiveDir, "slackdump.sqlite")); + try { + seedArchiveSchema(baseDb); + insertChannel(baseDb, "C0PRESENT", "present"); + insertMessages(baseDb, "C0PRESENT", 2, "1714032800."); + } finally { + baseDb.close(); + } + const scopedDbA = new DatabaseSync(join(scopedArchiveDirA, "slackdump.sqlite")); + try { + seedArchiveSchema(scopedDbA); + insertChannel(scopedDbA, "C0MISSING_A", "missing-a"); + insertMessages(scopedDbA, "C0MISSING_A", 3, "1714032810."); + } finally { + scopedDbA.close(); + } + const scopedDbB = new DatabaseSync(join(scopedArchiveDirB, "slackdump.sqlite")); + try { + seedArchiveSchema(scopedDbB); + insertChannel(scopedDbB, "C0MISSING_B", "missing-b"); + insertMessages(scopedDbB, "C0MISSING_B", 5, "1714032820."); + } finally { + scopedDbB.close(); + } + + const result = await runConnectorProtocolSubprocess({ + cwd: PACKAGE_ROOT, + entrypoint: SLACK_ENTRYPOINT, + env: { + PDPP_CONNECTOR_ARTIFACT_ROOT: artifactRoot, + PDPP_SLACK_SKIP_SLACKDUMP: "1", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + SLACK_WORKSPACE: workspace, + }, + start: { + type: "START", + scope: { streams: [{ name: "messages" }, { name: "reactions" }, { name: "message_attachments" }] }, + state: { + messages: { + channel_last_ts: { + C0MISSING_A: "1714032700.000000", + C0MISSING_B: "1714032700.000000", + C0PRESENT: "1714032700.000000", + }, + last_ts: "1714032700.000000", + observed_channel_ids: ["C0MISSING_A", "C0MISSING_B", "C0PRESENT"], + }, + }, + }, + }); + + const done = result.messages.findLast((m): m is Extract => m.type === "DONE"); + assert.ok(done, "connector reached a terminal DONE"); + assert.equal( + done.status, + "succeeded", + `run must succeed after healing both missing channels: ${JSON.stringify(done)}` + ); + + const coverage = detailCoverageMessages(result); + const messagesCoverage = coverage.filter((m) => m.state_stream === "messages" && m.stream === "messages"); + const reactionsCoverage = coverage.filter((m) => m.state_stream === "messages" && m.stream === "reactions"); + const attachmentsCoverage = coverage.filter( + (m) => m.state_stream === "messages" && m.stream === "message_attachments" + ); + + // THE BUG: with 2 scoped archives folded (base + A + B = 3 total + // runRequestedStreams calls), each (state_stream, stream) pair emitted 3 + // times pre-fix. The RI runtime rejects any repeat, so production Slack + // never completed a run with 2+ scoped archives. Fixed behavior: exactly + // one emission per pair, regardless of archive count. + assert.equal( + messagesCoverage.length, + 1, + "messages self-coverage must emit exactly once across the base + 2 scoped archives, not once per archive " + + `(got ${messagesCoverage.length} — this is the duplicate the runtime rejects)` + ); + // `reactions` and `message_attachments` are declared `state_stream: + // messages` in the manifest, so they are static single-parent detail + // streams and MUST emit no DETAIL_COVERAGE at all — + // `validateDetailCoverageAgainstManifest` fails the whole run if they do. + // This assertion previously required exactly one emission each, which + // encoded the defect as the contract: it passed only because the guard was + // not yet deployed, and every Slack run failed with `runtime_error` the + // moment drain29 shipped it. The counts were fabricated besides — they + // mirrored the PARENT messages denominator rather than any per-key tally + // of reactions or attachments actually accounted for. + assert.deepEqual( + reactionsCoverage, + [], + `reactions is state_stream-parented in the manifest and must emit NO DETAIL_COVERAGE (got ${reactionsCoverage.length})` + ); + assert.deepEqual( + attachmentsCoverage, + [], + "message_attachments is state_stream-parented in the manifest and must emit NO DETAIL_COVERAGE " + + `(got ${attachmentsCoverage.length})` + ); + + // The merged denominator: mergeMessagesPassResults sums `considered` + // across archives. Base (2 rows) + scoped A (3 rows) + scoped B (5 rows) + // = 10. A regression that keeps emission single but reverts to only the + // LAST archive's total (5) or the FIRST call's total (2) must fail here. + assert.equal(messagesCoverage[0]?.considered, 10, "considered is the SUMMED total across every archive folded"); + } finally { + await rm(artifactRoot, { recursive: true, force: true }); + } +}); diff --git a/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts b/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts index 7c62c0137..e23bdaeb0 100644 --- a/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts +++ b/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts @@ -866,7 +866,7 @@ test("slack connector does not emit a missing-partition diagnostic when prior ch } }); -test("slack connector uses per-channel message cursors with legacy global fallback", async () => { +test("slack connector walks a channel with no cursor in full instead of applying the global floor", async () => { const artifactRoot = await mkdtemp(join(tmpdir(), "pdpp-slack-channel-cursor-")); try { const workspace = "channel-cursor-test"; @@ -879,8 +879,10 @@ test("slack connector uses per-channel message cursors with legacy global fallba insertChannel(db, "C2", "two"); insertMessage(db, "C1", "1714031500.000000", "new for C1 but older than global"); insertMessage(db, "C1", "1714030900.000000", "old for C1"); - insertMessage(db, "C2", "1714031600.000000", "older than global fallback"); - insertMessage(db, "C2", "1714032500.000000", "new by global fallback"); + // C2 has NO cursor row. Both of its messages must be collected, + // including this one below the global `last_ts` floor. + insertMessage(db, "C2", "1714031600.000000", "below the global floor, in an unwalked channel"); + insertMessage(db, "C2", "1714032500.000000", "above the global floor"); } finally { db.close(); } @@ -920,7 +922,13 @@ test("slack connector uses per-channel message cursors with legacy global fallba } return a > b ? 1 : 0; }), - ["C1:1714031500.000000", "C2:1714032500.000000"] + // C1 stays incremental against its own cursor (…0900 is below it and + // is correctly skipped). C2, having no cursor, is walked in full. + // The prior contract expected only C2:…2500 here — the global + // `last_ts` floor silently swallowed C2:…1600, and because that row + // was never emitted no C2 cursor was ever written, so the same floor + // would swallow it again on every subsequent run. + ["C1:1714031500.000000", "C2:1714031600.000000", "C2:1714032500.000000"] ); const cursor = messagesState(result); diff --git a/packages/polyfill-connectors/connectors/steam/index.ts b/packages/polyfill-connectors/connectors/steam/index.ts index ecc347f90..c10d6b36a 100644 --- a/packages/polyfill-connectors/connectors/steam/index.ts +++ b/packages/polyfill-connectors/connectors/steam/index.ts @@ -154,6 +154,36 @@ function requireSteamResponse(value: unknown): Record { return requireSteamObject(envelope.response, "response"); } +/** + * Bind a Steam-declared inventory total to the coverage denominator. + * + * `GetOwnedGames` reports `game_count` and `GetRecentlyPlayedGames` reports + * `total_count` alongside the array they serve. Both calls are unpaginated, so + * without this check the denominator would be the length of whatever array + * arrived — a silently truncated response would read as fully covered. Using + * the source's own count makes a short array a visible coverage shortfall + * instead of an invisible one (the same posture Jellyfin takes with + * `TotalRecordCount`). + * + * Absent is tolerated: the field is optional in the wire contract and a missing + * total simply falls back to the served length. A malformed or impossible total + * fails closed, because a nonsense denominator is worse than no denominator. + */ +function steamDeclaredTotal(value: unknown, field: string, servedLength: number): number { + if (value === undefined || value === null) { + return servedLength; + } + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { + throw new Error(`steam_response_malformed: ${field} must be a nonnegative integer`); + } + if (value < servedLength) { + throw new Error( + `steam_response_malformed: ${field} (${value}) is less than the served item count (${servedLength})` + ); + } + return value; +} + // ─── HTTP helpers ───────────────────────────────────────────────────────── type ProgressFn = ( @@ -483,6 +513,9 @@ async function collectOwnedGames( ); const response = requireSteamResponse(gamesRes); const games = requireSteamArray(response.games, "response.games"); + // Steam's own inventory size for this account. Load-bearing as the coverage + // denominator below so a truncated `games` array cannot read as complete. + const declaredTotal = steamDeclaredTotal(response.game_count, "response.game_count", games.length); await deps.progress("Fetched owned games", { stream: "owned_games", count: games.length }); const gamesCursor = openFingerprintCursor((newState.owned_games as unknown) ?? {}); @@ -503,7 +536,9 @@ async function collectOwnedGames( stateStream: "owned_games", requiredKeys: [], hydratedKeys: [], - considered: coverage.considered, + // Source-declared total, not the served array length: if Steam says it + // owns N games and served fewer, this reads partial rather than complete. + considered: declaredTotal, covered: coverage.covered, } ); @@ -524,7 +559,19 @@ async function collectRecentlyPlayed( { stream: "recently_played_games" } ); const response = requireSteamResponse(recentRes); - const recentGames = requireSteamArray(response.games, "response.games"); + // GetRecentlyPlayedGames omits `games` entirely when the account has played + // nothing in the trailing two-week window -- the documented shape is + // `{"response":{"total_count":0}}`. That is a well-formed empty answer, not a + // malformed one, so requiring an array here failed the whole run for an + // account that simply had not played recently (observed 2026-08-17: + // `steam_response_malformed: response.games must be an array`). Absent is + // empty; a present non-array is still a real protocol violation and still + // throws. + const recentGames = + response.games === undefined ? [] : requireSteamArray(response.games, "response.games"); + // Steam reports `total_count` even for the empty case, so it is the honest + // denominator for this window's inventory. + const declaredTotal = steamDeclaredTotal(response.total_count, "response.total_count", recentGames.length); await deps.progress("Fetched recently played games", { stream: "recently_played_games", count: recentGames.length, @@ -548,7 +595,8 @@ async function collectRecentlyPlayed( stateStream: "recently_played_games", requiredKeys: [], hydratedKeys: [], - considered: coverage.considered, + // Source-declared total, not the served array length. + considered: declaredTotal, covered: coverage.covered, } ); diff --git a/packages/polyfill-connectors/connectors/steam/mutation.test.ts b/packages/polyfill-connectors/connectors/steam/mutation.test.ts index 34eb76912..a0eac0ebe 100644 --- a/packages/polyfill-connectors/connectors/steam/mutation.test.ts +++ b/packages/polyfill-connectors/connectors/steam/mutation.test.ts @@ -42,7 +42,6 @@ function makeContext(streams: readonly string[]): { const missingArrayCases = [ { body: { response: { game_count: 3 } }, stream: "owned_games" }, - { body: { response: { total_count: 3 } }, stream: "recently_played_games" }, { body: { friendslist: {} }, stream: "friends" }, ] as const; @@ -65,6 +64,48 @@ for (const { body, stream } of missingArrayCases) { }); } +test("steam: recently_played_games with games entirely absent is a well-formed empty answer, not malformed", async () => { + // GetRecentlyPlayedGames documented shape when the account played nothing + // in the trailing two-week window: {"response":{"total_count":0}}, no + // `games` key at all. This must succeed with zero records, not throw + // steam_response_malformed (regression for 3ccca8000). + globalThis.fetch = async () => jsonResponse({ response: { total_count: 0 } }); + const { ctx, messages } = makeContext(["recently_played_games"]); + + await steamCollect(ctx); + assert.equal( + messages.filter((message) => message.type === "STATE" && message.stream === "recently_played_games").length, + 1, + "an absent list must still advance its cursor as a real empty snapshot" + ); + const coverage = messages.find( + (message): message is Extract => + message.type === "DETAIL_COVERAGE" && message.stream === "recently_played_games" + ); + assert.ok(coverage); + assert.equal(coverage.considered, 0); + assert.equal(coverage.covered, 0); +}); + +test("steam: recently_played_games with games present but not an array is still malformed", async () => { + // A present-but-wrong-shaped `games` field is a genuine protocol violation + // (unlike an absent field), and must still fail before state or coverage. + globalThis.fetch = async () => jsonResponse({ response: { total_count: 3, games: "not-an-array" } }); + const { ctx, messages } = makeContext(["recently_played_games"]); + + await assert.rejects(() => steamCollect(ctx), /steam_response_malformed/); + assert.equal( + messages.some((message) => message.type === "STATE" && message.stream === "recently_played_games"), + false, + "a malformed list must not advance its cursor" + ); + assert.equal( + messages.some((message) => message.type === "DETAIL_COVERAGE" && message.stream === "recently_played_games"), + false, + "a malformed list must not prove an empty boundary" + ); +}); + test("steam: an explicit empty games array remains valid zero proof", async () => { globalThis.fetch = async () => jsonResponse({ response: { games: [] } }); const { ctx, messages } = makeContext(["owned_games"]); diff --git a/packages/polyfill-connectors/connectors/steam/provider-total-coverage.test.ts b/packages/polyfill-connectors/connectors/steam/provider-total-coverage.test.ts new file mode 100644 index 000000000..5796730d9 --- /dev/null +++ b/packages/polyfill-connectors/connectors/steam/provider-total-coverage.test.ts @@ -0,0 +1,167 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Steam declares its own inventory size (`game_count` on GetOwnedGames, + * `total_count` on GetRecentlyPlayedGames) alongside the array it serves. Both + * calls are unpaginated, so before this contract the coverage denominator was + * the length of whatever array arrived — a silently truncated response read as + * fully covered. + * + * These tests drive the real `steamCollect` path with a stubbed transport and + * assert on the emitted DETAIL_COVERAGE, so they fail if the denominator ever + * reverts to the served length. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { steamCollect } from "./index.ts"; + +const STEAM_ID = "76561198000000000"; + +interface CoverageMessage { + considered: number | undefined; + covered: number | undefined; + stream: string; +} + +function ownedGame(appid: number): Record { + return { appid, name: `Game ${appid}`, playtime_forever: 0 }; +} + +function recentGame(appid: number): Record { + return { appid, name: `Game ${appid}`, playtime_forever: 0, playtime_2weeks: 0 }; +} + +/** + * Run one Steam stream against a canned wire payload and return the coverage + * message the connector emitted for it. + */ +async function collectStream( + stream: "owned_games" | "recently_played_games", + payload: unknown +): Promise<{ coverage: CoverageMessage | undefined; recordCount: number }> { + const originalFetch = globalThis.fetch; + const originalUserId = process.env.STEAM_USER_ID; + const coverages: CoverageMessage[] = []; + let recordCount = 0; + + globalThis.fetch = (input) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/GetOwnedGames/v0001") || url.pathname.endsWith("/GetRecentlyPlayedGames/v0001")) { + return Promise.resolve(new Response(JSON.stringify(payload), { status: 200 })); + } + throw new Error(`unexpected Steam request: ${url.pathname}`); + }; + process.env.STEAM_USER_ID = STEAM_ID; + + try { + await steamCollect({ + state: {}, + requested: new Map([[stream, { name: stream }]]), + credentials: { STEAM_API_KEY: "synthetic-api-key" }, + emit: (msg) => { + const candidate = msg as unknown as { type?: string } & CoverageMessage; + if (candidate.type === "DETAIL_COVERAGE" && candidate.stream === stream) { + coverages.push({ + stream: candidate.stream, + considered: candidate.considered, + covered: candidate.covered, + }); + } + return Promise.resolve(); + }, + emitRecord: () => { + recordCount += 1; + return Promise.resolve(); + }, + progress: () => Promise.resolve(), + }); + } finally { + globalThis.fetch = originalFetch; + if (originalUserId === undefined) { + delete process.env.STEAM_USER_ID; + } else { + process.env.STEAM_USER_ID = originalUserId; + } + } + + return { coverage: coverages.at(-1), recordCount }; +} + +test("steam owned_games - a truncated games array reads partial against the declared game_count", async () => { + // Steam says the account owns 10 games but serves only 3. Before the provider + // total was bound, considered would have been 3 and the run read complete. + const { coverage, recordCount } = await collectStream("owned_games", { + response: { game_count: 10, games: [ownedGame(10), ownedGame(20), ownedGame(30)] }, + }); + + assert.equal(recordCount, 3, "only the served games can be emitted"); + assert.equal(coverage?.considered, 10, "denominator must be Steam's declared game_count, not the served length"); + assert.equal(coverage?.covered, 3, "covered counts only what was actually served and validated"); + assert.ok( + (coverage?.covered ?? 0) < (coverage?.considered ?? 0), + "a truncated response must read as a coverage shortfall" + ); +}); + +test("steam owned_games - a complete response reads fully covered", async () => { + const { coverage } = await collectStream("owned_games", { + response: { game_count: 2, games: [ownedGame(10), ownedGame(20)] }, + }); + + assert.equal(coverage?.considered, 2); + assert.equal(coverage?.covered, 2, "an untruncated snapshot still proves full coverage"); +}); + +test("steam owned_games - a game_count below the served length is a protocol violation", async () => { + // An impossible total is worse than no total: it would understate the + // denominator and could make a partial run read as over-covered. + await assert.rejects( + collectStream("owned_games", { + response: { game_count: 1, games: [ownedGame(10), ownedGame(20)] }, + }), + /steam_response_malformed: response\.game_count \(1\) is less than the served item count \(2\)/ + ); +}); + +test("steam owned_games - a non-integer game_count fails closed", async () => { + await assert.rejects( + collectStream("owned_games", { + response: { game_count: "many", games: [ownedGame(10)] }, + }), + /steam_response_malformed: response\.game_count must be a nonnegative integer/ + ); +}); + +test("steam owned_games - an absent game_count falls back to the served length", async () => { + // The field is optional in the wire contract; absence must not fail the run. + const { coverage } = await collectStream("owned_games", { + response: { games: [ownedGame(10), ownedGame(20)] }, + }); + + assert.equal(coverage?.considered, 2); + assert.equal(coverage?.covered, 2); +}); + +test("steam recently_played_games - a truncated array reads partial against total_count", async () => { + const { coverage } = await collectStream("recently_played_games", { + response: { total_count: 5, games: [recentGame(10)] }, + }); + + assert.equal(coverage?.considered, 5, "denominator must be Steam's declared total_count"); + assert.equal(coverage?.covered, 1); +}); + +test("steam recently_played_games - the documented empty shape proves an empty window", async () => { + // `{"response":{"total_count":0}}` with no `games` key is Steam's well-formed + // answer for an account that played nothing recently. That is proven-empty, + // not a failure. + const { coverage, recordCount } = await collectStream("recently_played_games", { + response: { total_count: 0 }, + }); + + assert.equal(recordCount, 0); + assert.equal(coverage?.considered, 0); + assert.equal(coverage?.covered, 0); +}); diff --git a/packages/polyfill-connectors/connectors/usaa/index.ts b/packages/polyfill-connectors/connectors/usaa/index.ts index d712172dc..0c1462337 100644 --- a/packages/polyfill-connectors/connectors/usaa/index.ts +++ b/packages/polyfill-connectors/connectors/usaa/index.ts @@ -82,6 +82,7 @@ import { import { validateRecord as validateRecordRaw } from "./schemas.ts"; import { computeStatementCoverage, type StatementCoverageRow } from "./statement-coverage.ts"; import { fileUrlForPath, hydrateStatementPdfs, parsePdfStatement } from "./statement-pdfs.ts"; +import { buildReconciliationDiagnostics } from "./statement-reconciliation.ts"; import type { BillingKv, DashboardAccount, @@ -152,7 +153,25 @@ const EXPORT_DIALOG_MESSAGE_SELECTOR = const EXPORT_NO_DATA_RE = /no transactions|nothing to export/iu; const USAA_ACCOUNT_DETAIL_ROUTE_RE = /^\/my\/(?:checking|savings|credit-card)(?:\/|$)/u; const USAA_INTERSTITIAL_ROUTE_RE = - /\/(?:my\/logon|access-management\/oauth2\/member\/authorize|security(?:\/|$)|challenge(?:\/|$))/iu; + /\/(?:my\/logon|access-management\/oauth2\/member\/authorize|security(?:\/|$)|challenge(?:\/|$)|my\/banking-offer(?:\/|$))/iu; +/** + * Marketing interstitials USAA injects in FRONT of an account page. Distinct + * from the auth/challenge interstitials above: the session is perfectly alive, + * the member is simply being shown an offer (e.g. `/my/banking-offer/atm-deposit`, + * "Depositing cash just got more convenient!") before the page they asked for. + * + * Observed live on the owner's mailbox 2026-07-14: BOTH checking accounts + * landed here instead of `/my/checking`, so `findExportAffordance` found no + * Export button and the run reported `source_structure_changed` — "USAA changed + * their UI" — for two accounts that were fine. Credit-card accounts, which are + * not offered ATM deposits, exported normally. That is the whole of the + * long-standing USAA `transactions` 2-of-4 coverage gap. + * + * These pages carry the originally-requested URL in their own `goto` query + * param, which is the honest way back: it is USAA's own statement of where the + * member was headed, not a URL this connector guessed. + */ +const USAA_OFFER_INTERSTITIAL_ROUTE_RE = /^\/my\/banking-offer(?:\/|$)/iu; const USAA_ACCOUNT_DETAIL_MARKER_SELECTOR = ".ent-as-utility-bar, .as_credit__utility-bar"; const USAA_TRANSACTION_MARKER_SELECTOR = 'table[aria-label*="transaction" i], [data-testid*="transaction" i], [id*="transaction" i]'; @@ -214,6 +233,16 @@ export interface EmitDeps { * A reached account emits recovery for its supplied gap id; this prevents a * successful later export from leaving the durable gap pending forever. */ servedAccountTransactionGaps?: ReadonlyMap; + /** Pending USAA credit-card billing/stats gaps served by the runtime this + * run, one map per stream (see `buildServedCreditCardGapLookups`). A + * successfully-navigated-and-scraped card emits recovery for its supplied + * gap id on whichever stream(s) had one — see `recoverServedCreditCardGaps`. */ + servedCreditCardGaps?: { billing: ReadonlyMap; billingStats: ReadonlyMap }; + /** Pending USAA `statements` PDF gaps served by the runtime this run. A + * statement whose PDF is present again emits recovery for its supplied gap + * id; without this a downloaded statement leaves its gap pending forever + * (see `recoverServedStatementGaps`). */ + servedStatementGaps?: ReadonlyMap; } /** Aggregate shape from the PDF hydration pass. Exposed so the emit- @@ -837,9 +866,55 @@ export async function emitStatementCoverage( for (const gap of result.gaps) { await deps.emit(gap); } + await recoverServedStatementGaps(deps, result.coverage.hydratedKeys); await emitDetailCoverage(deps, result.coverage); } +/** + * Close the served `statements` gaps whose PDF is present again this run. + * + * A statement gap is opened whenever a run does not hold that statement's PDF, + * and it is genuinely retryable — the next run re-attempts every row. But + * nothing ever CLOSED one: a pending detail gap only leaves `pending` on an + * explicit `DETAIL_GAP_RECOVERED` (see `connector-detail-gap-store.ts`), and + * this connector emitted that message for `transactions` and the credit-card + * streams while `statements` was left out. + * + * The result on the owner's instance: 4 `statements` gaps sat `pending` from a + * 2026-08-04 run, three of them never re-attempted (`attempt_count = 0`), while + * every one of those four statements had in fact been downloaded — each has a + * durable `pdf_sha256` on its record, and the same stream reported + * `covered: 10 / considered: 10, checkpoint: committed`. The stream was fully + * collected and simultaneously showed a retryable gap, purely as stale + * bookkeeping. + * + * `hydratedKeys` is the honest input: it is exactly the set + * `computeStatementCoverage` proved artifact-present this run (`isHydrated` on + * the resolved body), the same predicate that put those keys in the coverage + * numerator. Only ids the runtime actually served as pending gaps are closed — + * the lookup enforces that, so a recovery can never close unrelated work. + */ +async function recoverServedStatementGaps(deps: EmitDeps, hydratedKeys: readonly (string | number)[]): Promise { + const served = deps.servedStatementGaps; + if (!served || served.size === 0) { + return; + } + for (const key of hydratedKeys) { + const statementId = String(key); + const gapId = served.get(statementId); + if (!gapId) { + continue; + } + await deps.emit({ + type: "DETAIL_GAP_RECOVERED", + reference_only: true, + gap_id: gapId, + stream: "statements", + record_key: statementId, + }); + } +} + /** * Parse the prior `statements` STATE cursor's `fingerprints` map. Keyed * by statement `id`. Legacy cursors (only `{ fetched_at }`) decode to an @@ -1139,6 +1214,84 @@ function capturePageDiagnostics(page: Page): Promise { .catch((): PageDiagnostics | null => null); } +/** + * If `finalUrl` is a USAA marketing interstitial, return the account URL to + * retry; otherwise `null` (we are already where we asked to be, or somewhere + * this function has no honest opinion about). + * + * Pure and exported so the escape decision is testable without Playwright. + * + * The retry target is the interstitial's own `goto` param when it names a USAA + * account-detail route, else the `accountUrl` originally requested. Both are + * URLs the caller already had or USAA itself supplied; a `goto` pointing + * off-host or at some other section is refused rather than followed, so a + * redirect chain can never steer the connector somewhere it did not intend to + * go. Returns `null` when the escape target matches the URL we just loaded, so + * a self-referential offer page cannot produce an infinite retry. + */ +export function resolveUsaaOfferInterstitialEscape(finalUrl: string, accountUrl: string): string | null { + let parsed: URL; + try { + parsed = new URL(finalUrl); + } catch { + return null; + } + if (parsed.hostname !== "www.usaa.com" || !USAA_OFFER_INTERSTITIAL_ROUTE_RE.test(parsed.pathname)) { + return null; + } + let target = accountUrl; + const goto = parsed.searchParams.get("goto"); + if (goto) { + try { + const gotoUrl = new URL(goto, "https://www.usaa.com"); + if (gotoUrl.hostname === "www.usaa.com" && USAA_ACCOUNT_DETAIL_ROUTE_RE.test(gotoUrl.pathname)) { + target = gotoUrl.toString(); + } + } catch { + // Unparseable `goto` — fall back to the requested account URL. + } + } + return target === finalUrl ? null : target; +} + +/** + * Ensure the just-loaded page is the account page we asked for, navigating + * through a USAA marketing offer if one was served in front of it. + * + * Returns `false` when this candidate URL should be abandoned (the escape + * navigation itself failed); `true` when the caller should look for the export + * affordance on the current page. + * + * Throws {@link SessionDeadRedirectError} if the session died — before OR + * after the escape hop, since an offer page can itself redirect to logon. + * + * Exactly one escape hop is attempted. If the offer re-serves itself, the + * caller's ordinary no-affordance path reports it rather than looping. + */ +async function settleOnAccountPage(page: Page, accountUrl: string, settleDelayMs: number): Promise { + const assertSessionAlive = async (): Promise => { + if (LOGON_REDIRECT_RE.test(page.url())) { + throw new SessionDeadRedirectError(await captureNoExportAffordanceObservation(page)); + } + }; + await assertSessionAlive(); + const escapeUrl = resolveUsaaOfferInterstitialEscape(page.url(), accountUrl); + if (!escapeUrl) { + return true; + } + try { + await page.goto(escapeUrl, { + waitUntil: "domcontentloaded", + timeout: ACCOUNT_NAV_TIMEOUT_MS, + }); + } catch { + return false; + } + await politeDelay(settleDelayMs); + await assertSessionAlive(); + return true; +} + async function locateExportPage( page: Page, accountUrl: string, @@ -1161,9 +1314,8 @@ async function locateExportPage( continue; } await politeDelay(settleDelayMs); - const finalUrl = page.url(); - if (LOGON_REDIRECT_RE.test(finalUrl)) { - throw new SessionDeadRedirectError(await captureNoExportAffordanceObservation(page)); + if (!(await settleOnAccountPage(page, url, settleDelayMs))) { + continue; } const btn = await findExportAffordance(page); if (btn) { @@ -1219,7 +1371,21 @@ async function emitDialogUnexpectedShapeDiagnostic( }); } -/** Click Export, then confirm the date-range selector rendered. */ +/** Click Export, then confirm the date-range selector rendered. + * + * The readiness check is a bounded WAIT (`waitFor`), not a fixed sleep + * followed by a single immediate `.count()`. A one-shot count taken after a + * fixed delay cannot tell "the dialog will never render this select" apart + * from "the dialog just hasn't finished rendering yet" — on a page that + * renders a moment slower than `EXPORT_DIALOG_DELAY_MS` (network jitter, + * client-side render variance), the one-shot check reads a false + * `export_dialog_unexpected_shape`, which `tryExportLadder` treats as fatal + * and reports as `export_affordance_missing` (a "source changed, needs a + * code fix" outcome) instead of the transient rendering delay it actually + * was. `waitFor` polls up to the same overall budget and resolves the + * instant the select appears, so a slow-but-real render still succeeds; + * only a select that never appears within the budget reaches the existing + * unexpected-shape/Escape path, unchanged. */ async function openExportDialog(page: Page, located: LocatedExportPage, options: DriveExportOptions): Promise { const { onDiagnostics } = options; try { @@ -1228,12 +1394,14 @@ async function openExportDialog(page: Page, located: LocatedExportPage, options: await emitExportClickFailedDiagnostic(page, onDiagnostics, err); return false; } - await politeDelay(EXPORT_DIALOG_DELAY_MS); - const selectCount = await page - .locator('[role="dialog"] select[name="selectionType"], select[name="selectionType"]') - .count() - .catch((): number => 0); + const selectLocator = page.locator('[role="dialog"] select[name="selectionType"], select[name="selectionType"]'); + const rendered = await selectLocator + .first() + .waitFor({ state: "visible", timeout: EXPORT_DIALOG_DELAY_MS }) + .then((): boolean => true) + .catch((): boolean => false); + const selectCount = rendered ? await selectLocator.count().catch((): number => 0) : 0; if (!selectCount) { if (onDiagnostics) { await emitDialogUnexpectedShapeDiagnostic(page, onDiagnostics); @@ -2033,42 +2201,93 @@ export function buildAccountTransactionDetailGap(outcome: { } /** - * Keep only USAA account-level transaction gaps the runtime actually served - * this run. The connector may recover only these supplied ids: synthesizing - * one, or accepting a foreign/malformed locator, could close unrelated work. + * Keep only the pending USAA detail gaps on `stream` whose `detail_locator` + * has the expected `kind` and whose `locatorField` matches `record_key` — the + * same closed-world shape check every served-gap lookup in this connector + * needs. The connector may recover only gaps the runtime actually served this + * run: synthesizing an id, or accepting a foreign/malformed locator, could + * close unrelated work. Shared by `buildServedAccountTransactionGapLookup` + * (transactions, locator field `account_id`) and the credit-card billing + * streams (locator field `card_id`) so the same closed-world proof isn't + * hand-rolled per stream. */ -export function buildServedAccountTransactionGapLookup( - detailGaps: readonly BrowserCollectContext["detailGaps"][number][] +function buildServedGapLookup( + detailGaps: readonly BrowserCollectContext["detailGaps"][number][], + stream: string, + locatorKind: string, + locatorField: string ): Map { const lookup = new Map(); for (const gap of detailGaps) { - if (gap.stream !== "transactions" || gap.status !== "pending") { + if (gap.stream !== stream || gap.status !== "pending") { continue; } const locator = gap.detail_locator; - if (locator?.kind !== "usaa.account") { + if (locator?.kind !== locatorKind) { continue; } - const accountId = locator.account_id; + const key = locator[locatorField]; const recordKey = gap.record_key; if ( - typeof accountId !== "string" || - accountId.length === 0 || + typeof key !== "string" || + key.length === 0 || typeof recordKey !== "string" || recordKey.length === 0 || - recordKey !== accountId || + recordKey !== key || typeof gap.gap_id !== "string" || !gap.gap_id ) { continue; } - if (!lookup.has(accountId)) { - lookup.set(accountId, gap.gap_id); + if (!lookup.has(key)) { + lookup.set(key, gap.gap_id); } } return lookup; } +/** + * Keep only USAA account-level transaction gaps the runtime actually served + * this run. The connector may recover only these supplied ids: synthesizing + * one, or accepting a foreign/malformed locator, could close unrelated work. + */ +export function buildServedAccountTransactionGapLookup( + detailGaps: readonly BrowserCollectContext["detailGaps"][number][] +): Map { + return buildServedGapLookup(detailGaps, "transactions", "usaa.account", "account_id"); +} + +/** + * Keep only `statements` PDF gaps the runtime actually served this run. Locator + * shape is `buildStatementDetailGap`'s (`usaa.statement` / `statement_id`). + */ +export function buildServedStatementGapLookup( + detailGaps: readonly BrowserCollectContext["detailGaps"][number][] +): Map { + return buildServedGapLookup(detailGaps, "statements", "usaa.statement", "statement_id"); +} + +/** + * Keep only USAA credit-card billing/stats gaps the runtime actually served + * this run, one lookup per stream (a card's `credit_card_billing` gap and its + * `credit_card_billing_stats` gap are independent DETAIL_GAP rows, served and + * recovered independently — see `emitCreditCardNavFailureGaps`). + */ +export function buildServedCreditCardGapLookups(detailGaps: readonly BrowserCollectContext["detailGaps"][number][]): { + billing: Map; + billingStats: Map; +} { + return { + billing: buildServedGapLookup(detailGaps, "credit_card_billing", "usaa.credit_card_billing", "card_id"), + billingStats: buildServedGapLookup( + detailGaps, + "credit_card_billing_stats", + "usaa.credit_card_billing_stats", + "card_id" + ), + }; +} + /** * A served account gap is recovered only after this run reaches that same * account. `hydrated` and source-limited `no_activity` are both complete @@ -2402,7 +2621,11 @@ function scrapeStatementsIndex(page: Page): Promise { }); } -async function hydratePdfsForIndex(deps: StatementsSubDeps, indexRows: readonly IndexRow[]): Promise { +async function hydratePdfsForIndex( + deps: StatementsSubDeps, + indexRows: readonly IndexRow[], + context?: BrowserContext +): Promise { const results = new Map(); let attempts = 0; let successes = 0; @@ -2411,6 +2634,8 @@ async function hydratePdfsForIndex(deps: StatementsSubDeps, indexRows: readonly const hydrated = await hydrateStatementPdfs({ page: deps.page, statements: indexRows as IndexRow[], + capture: deps.capture ?? null, + context, onProgress: ({ index, total }) => { attempts = index + 1; // Fire-and-forget: hydrateStatementPdfs signature is sync callback. @@ -2462,6 +2687,12 @@ interface PdfParseCounters { parsedStatements: number; pdfTxnCount: number; unknownTemplates: number; + /** Periods whose transactions failed to sum to USAA's own printed + * beginning/ending balance delta. Counted separately from + * `unknownTemplates` because a template we DID match but could not + * reconcile is a stronger, more specific finding than one we never + * matched at all. */ + unreconciledStatements: number; } async function processPdfStatementRow( @@ -2480,12 +2711,29 @@ async function processPdfStatementRow( const acct = row.account_id ? accountById.get(row.account_id) : null; const accountName = acct?.name ?? row.account_reference ?? null; try { - const { txns, parseMeta } = await parsePdfStatement({ + const { txns, parseMeta, reconciliation } = await parsePdfStatement({ buffer: ok.buffer, accountId: row.account_id || row.account_reference || "unknown", accountName, period, }); + // The completeness anchor: USAA's own printed period totals say what + // this period's transactions must sum to. Report a failed reconciliation + // BEFORE the empty-parse early return below, because "the balance moved + // but we parsed nothing" is exactly the case that must not slip out as a + // bare template-unknown notice. `unavailable` is silent by design — a + // credit-card statement prints no such summary and has no anchor to + // fail. + if (reconciliation.status === "mismatched") { + counters.unreconciledStatements += 1; + await deps.emit({ + type: "SKIP_RESULT", + stream: "transactions", + reason: "statement_unreconciled", + message: `Statement period at row ${row.rowIndex + 1} does not reconcile against its own printed balances`, + diagnostics: buildReconciliationDiagnostics(row.id, reconciliation), + }); + } if (!txns.length) { counters.unknownTemplates += 1; await deps.emit({ @@ -2533,7 +2781,12 @@ async function emitPdfStatementTransactions( .filter((a): a is DashboardAccount & { account_id_raw: string } => Boolean(a.account_id_raw)) .map((a) => [a.account_id_raw, a]) ); - const counters: PdfParseCounters = { pdfTxnCount: 0, parsedStatements: 0, unknownTemplates: 0 }; + const counters: PdfParseCounters = { + pdfTxnCount: 0, + parsedStatements: 0, + unknownTemplates: 0, + unreconciledStatements: 0, + }; for (const row of indexRows) { const ok = hydrationSuccess(hydrationResults.get(row.rowIndex)); if (!ok) { @@ -2544,7 +2797,7 @@ async function emitPdfStatementTransactions( await deps.emit({ type: "PROGRESS", stream: "transactions", - message: `PDF parse complete: ${counters.pdfTxnCount} transaction(s) across ${counters.parsedStatements} statement(s) (${counters.unknownTemplates} unknown templates)`, + message: `PDF parse complete: ${counters.pdfTxnCount} transaction(s) across ${counters.parsedStatements} statement(s) (${counters.unknownTemplates} unknown templates, ${counters.unreconciledStatements} unreconciled)`, }); } @@ -2606,7 +2859,7 @@ export async function runStatementsStream( stream: "statements", message: `Found ${indexRows.length} statement index row(s)`, }); - const summary = await hydratePdfsForIndex(deps, indexRows); + const summary = await hydratePdfsForIndex(deps, indexRows, context); if (requested.has("statements")) { await emitStatementRecords( @@ -2707,6 +2960,12 @@ export async function runInboxStream( return false; } await politeDelay(DOCUMENTS_SETTLE_DELAY_MS); + // Diagnostic-only DOM/ARIA/screenshot snapshot of the inbox table before + // the fixed-position [c0,c1,c2] scrape below. Every buildInboxMessageRecord + // failure (empty date_short) traces back to this scrape's column mapping, + // and there was previously no captured artifact showing the real table + // shape to confirm or correct it against. No-op unless PDPP_CAPTURE_*. + await deps.capture?.captureDom(page, "inbox-listing").catch((): undefined => undefined); const msgs = await scrapeInboxRows(page); await deps.emit({ type: "PROGRESS", @@ -2762,6 +3021,30 @@ export async function runInboxStream( considered: inboxCoverage.considered, covered: inboxCoverage.covered, }); + // Every listed row failing to resolve (covered === 0 with a nonzero + // considered) is a structural-drift signal, not ordinary per-row noise: + // `buildInboxMessageRecord` only drops a row for a missing/unparseable + // `date_short`, and it is very unlikely every row on a real inbox page + // shares that defect at once — far more likely the table's column + // layout shifted (an inserted leading cell, or status/date/preview + // reordered) and `date_short` is silently reading the wrong cell for + // every row. Statements (pdf_download_timeout) and transactions + // (export_affordance_missing) already surface this class of failure as + // a diagnostic SKIP_RESULT; inbox previously reported only a bare + // partial DETAIL_COVERAGE with no signal telling anyone why. This is + // purely diagnostic — retryable, reference-only, no PII (row count only, + // no dates/preview text) — never a hard error, and a partial (some but + // not all rows unresolved) intentionally stays silent to avoid noise on + // the ordinary case. + if (inboxCoverage.considered > 0 && inboxCoverage.covered === 0) { + await deps.emit({ + type: "SKIP_RESULT", + stream: "inbox_messages", + reason: "inbox_rows_unresolved", + message: `Inbox scrape found ${inboxCoverage.considered} row(s) but none resolved into a record (likely a table structure change); retry by runtime`, + diagnostics: { considered: inboxCoverage.considered }, + }); + } return true; } catch (err) { await deps.emit({ @@ -2867,6 +3150,51 @@ async function emitCreditCardNavFailureGaps( } } +/** + * Emit `DETAIL_GAP_RECOVERED` for a successfully-navigated-and-scraped card + * on whichever of the two credit-card streams the runtime is holding a + * served, pending gap for. Mirrors `recoverServedAccountTransactionGaps`: + * before this, a card gapped by a crashed/interrupted run (e.g. the + * mid-run `runtime_error` that produced this connection's stuck + * `credit_card_billing`/`credit_card_billing_stats` gaps) stayed `pending` + * forever on every later successful run, because `emitCreditCardNavFailureGaps` + * had a DETAIL_GAP emit path but no matching recovery path — the connector + * never told the runtime "this card is fine now." Only called for cards that + * actually reached `emitCreditCardBillingForCard` (outcome `"ok"`); a + * navigation failure keeps the gap pending via `emitCreditCardNavFailureGaps` + * instead. + */ +async function recoverServedCreditCardGaps( + deps: EmitDeps, + cardId: string, + served: EmitDeps["servedCreditCardGaps"], + { emitEntity, emitStats }: Pick +): Promise { + if (!served) { + return; + } + const billingGapId = emitEntity ? served.billing.get(cardId) : undefined; + if (billingGapId) { + await deps.emit({ + type: "DETAIL_GAP_RECOVERED", + reference_only: true, + gap_id: billingGapId, + stream: "credit_card_billing", + record_key: cardId, + }); + } + const statsGapId = emitStats ? served.billingStats.get(cardId) : undefined; + if (statsGapId) { + await deps.emit({ + type: "DETAIL_GAP_RECOVERED", + reference_only: true, + gap_id: statsGapId, + stream: "credit_card_billing_stats", + record_key: cardId, + }); + } +} + /** Outcome of navigating to one card's page: * - `ok`: navigation succeeded (first try, or after a logon-bounce repair); * scraping may proceed. @@ -3004,6 +3332,7 @@ export async function runCreditCardBillingStream( }); if (outcome === "ok") { await emitCreditCardBillingForCard(deps, page, a, options); + await recoverServedCreditCardGaps(deps, cardId, deps.servedCreditCardGaps, { emitEntity, emitStats }); continue; } navFailedIds.add(cardId); @@ -3205,6 +3534,8 @@ export async function collectUsaa(ctx: BrowserCollectContext): Promise { emit, emitRecord, servedAccountTransactionGaps: buildServedAccountTransactionGapLookup(ctx.detailGaps), + servedCreditCardGaps: buildServedCreditCardGapLookups(ctx.detailGaps), + servedStatementGaps: buildServedStatementGapLookup(ctx.detailGaps), }; // Run-scoped state shared across every stream below, constructed diff --git a/packages/polyfill-connectors/connectors/usaa/integration.test.ts b/packages/polyfill-connectors/connectors/usaa/integration.test.ts index b06d6ac07..0f1ac6431 100644 --- a/packages/polyfill-connectors/connectors/usaa/integration.test.ts +++ b/packages/polyfill-connectors/connectors/usaa/integration.test.ts @@ -52,6 +52,7 @@ import { type EmittedRecord, makeRecordingEmit } from "../../src/test-harness.ts import { buildIndexRows, buildPdfTemplateUnknownDiagnostics, + buildServedStatementGapLookup, classifyUsaaNoExportRoute, DEFERRED_STREAMS, driveExport, @@ -59,10 +60,12 @@ import { emitAccountsStream, emitDeferredStreams, emitExportFailure, + emitStatementCoverage, emitStatementRecords, type HydrationSummary, hydrationSuccess, isNoDataExportMessage, + resolveUsaaOfferInterstitialEscape, runSingleLadderAttempt, shouldParseStatementTitle, USAA_RETRYABLE_PATTERN, @@ -574,6 +577,328 @@ test("classifyUsaaNoExportRoute requires both a closed account route and structu assert.equal(classifyUsaaNoExportRoute("https://private.example/my/checking", true), "unknown"); }); +const STATEMENT_HYDRATED = { + document_url: "file:///home/user/.pdpp/usaa-statements/chk/2026-04-aaaaaaaa.pdf", + pdf_path: "/home/user/.pdpp/usaa-statements/chk/2026-04-aaaaaaaa.pdf", + pdf_sha256: "aa".repeat(32), +}; +const STATEMENT_NOT_HYDRATED = { document_url: null, pdf_path: null, pdf_sha256: null }; + +/** A served pending `statements` gap in the shape the runtime supplies. */ +function servedStatementGap(statementId: string, gapId: string) { + return { + gap_id: gapId, + record_key: statementId, + status: "pending", + stream: "statements", + detail_locator: { kind: "usaa.statement", statement_id: statementId }, + }; +} + +/** + * A pending detail gap only leaves `pending` on an explicit + * `DETAIL_GAP_RECOVERED`. This connector emitted that for `transactions` and + * the credit-card streams but never for `statements`, so a statement PDF that + * failed once and downloaded fine afterwards kept its gap forever. + * + * Owner-visible symptom: 4 `statements` gaps pending from a 2026-08-04 run + * (three never even re-attempted, `attempt_count = 0`) while the same stream + * reported `covered: 10 / considered: 10, checkpoint: committed` — and all four + * of those statements had a durable `pdf_sha256` on their record. Fully + * collected and showing a retryable gap at the same time. + */ +test("emitStatementCoverage: recovers a served statement gap once the PDF is present again", async () => { + const { deps, messages } = makeHarness(); + await emitStatementCoverage( + { + ...deps, + servedStatementGaps: new Map([["stmt-recovered", "gap-recovered"]]), + }, + [{ id: "stmt-recovered", isCandidate: true, pointers: STATEMENT_HYDRATED }] + ); + + assert.deepEqual( + messages.filter((m) => m.type === "DETAIL_GAP_RECOVERED"), + [ + { + type: "DETAIL_GAP_RECOVERED", + reference_only: true, + gap_id: "gap-recovered", + stream: "statements", + record_key: "stmt-recovered", + }, + ], + "a statement whose PDF is present again must close its served gap, or the gap stays pending forever" + ); +}); + +test("emitStatementCoverage: never closes a gap for a statement still missing its PDF", async () => { + const { deps, messages } = makeHarness(); + await emitStatementCoverage( + { + ...deps, + // Both ids are served, but only one statement actually has its PDF. + servedStatementGaps: new Map([ + ["stmt-have", "gap-have"], + ["stmt-missing", "gap-missing"], + ]), + }, + [ + { id: "stmt-have", isCandidate: true, pointers: STATEMENT_HYDRATED }, + { id: "stmt-missing", isCandidate: true, pointers: STATEMENT_NOT_HYDRATED }, + ] + ); + + assert.deepEqual( + messages.filter((m) => m.type === "DETAIL_GAP_RECOVERED").map((m) => m.gap_id), + ["gap-have"], + "recovery must follow the hydration proof, never merely the fact that a gap was served" + ); + assert.ok( + messages.some((m) => m.type === "DETAIL_GAP" && m.record_key === "stmt-missing"), + "the still-missing statement must keep a pending gap" + ); +}); + +test("emitStatementCoverage: only closes gaps the runtime actually served", async () => { + const { deps, messages } = makeHarness(); + await emitStatementCoverage({ ...deps, servedStatementGaps: new Map() }, [ + { id: "stmt-unserved", isCandidate: true, pointers: STATEMENT_HYDRATED }, + ]); + + assert.deepEqual( + messages.filter((m) => m.type === "DETAIL_GAP_RECOVERED"), + [], + "a gap id the runtime did not serve must never be synthesized — that could close unrelated work" + ); +}); + +test("buildServedStatementGapLookup: accepts only well-formed pending usaa.statement gaps", () => { + const lookup = buildServedStatementGapLookup([ + servedStatementGap("stmt-ok", "gap-ok"), + // Wrong stream, wrong locator kind, non-pending, and a locator/record_key + // disagreement must all be refused. + { ...servedStatementGap("stmt-other-stream", "gap-x"), stream: "transactions" }, + { ...servedStatementGap("stmt-bad-kind", "gap-y"), detail_locator: { kind: "usaa.account", account_id: "a" } }, + { ...servedStatementGap("stmt-recovered", "gap-z"), status: "recovered" }, + { ...servedStatementGap("stmt-mismatch", "gap-w"), record_key: "different-id" }, + ] as never); + + assert.deepEqual( + [...lookup.entries()], + [["stmt-ok", "gap-ok"]], + "only a pending usaa.statement gap whose locator agrees with its record_key may be recoverable" + ); +}); + +/** + * USAA serves a marketing offer in front of an account page. The session is + * alive; the member is simply being shown "Depositing cash just got more + * convenient!" before the page they asked for. + * + * Regression (owner-reported, USAA `transactions` stuck at 2 of 4 accounts): + * BOTH checking accounts landed on + * `/my/banking-offer/atm-deposit?...&accountType=checking&goto=...` instead of + * `/my/checking`, so no Export button was found and the run reported + * `source_structure_changed` — "USAA changed their UI" — for two accounts that + * were perfectly fine. Credit cards, which are not offered ATM deposits, + * exported normally. Captured live 2026-07-14 (page title "Find an ATM | USAA"). + */ +test("resolveUsaaOfferInterstitialEscape: prefers the offer page's own goto target", () => { + assert.equal( + resolveUsaaOfferInterstitialEscape( + "https://www.usaa.com/my/banking-offer/atm-deposit?accountId=private&accountType=checking&goto=https://www.usaa.com/my/checking%3FaccountId%3Dprivate", + "https://www.usaa.com/my/checking?accountId=fallback" + ), + "https://www.usaa.com/my/checking?accountId=private", + "the offer page states where the member was headed; that is the honest way back" + ); +}); + +test("resolveUsaaOfferInterstitialEscape: falls back to the requested account URL", () => { + // A `goto` that is off-host or points somewhere other than an account detail + // route is REFUSED, so a redirect chain cannot steer the connector. + for (const goto of [ + "https://www.usaa.com/inet/ent_home/CpHome", + "https://evil.example/my/checking", + "%%not-a-url%%", + ]) { + assert.equal( + resolveUsaaOfferInterstitialEscape( + `https://www.usaa.com/my/banking-offer/atm-deposit?goto=${encodeURIComponent(goto)}`, + "https://www.usaa.com/my/checking?accountId=private" + ), + "https://www.usaa.com/my/checking?accountId=private", + `goto=${goto} must not be followed` + ); + } +}); + +test("resolveUsaaOfferInterstitialEscape: only offer routes escape, and never into a loop", () => { + // Pages that are already the destination, or that this function has no + // honest opinion about, must NOT trigger a re-navigation. + for (const finalUrl of [ + "https://www.usaa.com/my/checking?accountId=private", + "https://www.usaa.com/challenge/step", + "https://www.usaa.com/my/dashboard", + "https://evil.example/my/banking-offer/atm-deposit", + "not-a-url", + ]) { + assert.equal( + resolveUsaaOfferInterstitialEscape(finalUrl, "https://www.usaa.com/my/checking?accountId=private"), + null, + `${finalUrl} must not be treated as an escapable offer interstitial` + ); + } + // A self-referential offer page cannot produce an infinite retry. + assert.equal( + resolveUsaaOfferInterstitialEscape( + "https://www.usaa.com/my/banking-offer/atm-deposit", + "https://www.usaa.com/my/banking-offer/atm-deposit" + ), + null, + "an escape target identical to the page just loaded must be refused" + ); +}); + +/** A page that serves the ATM-deposit offer first and the real account page + * (with a working Export button) only after the connector navigates through + * it — exactly the live sequence. Records every `goto` so the test proves the + * connector actually navigated rather than merely tolerating the offer. */ +function makeOfferInterstitialPage(visited: string[]): Page { + const offerUrl = + "https://www.usaa.com/my/banking-offer/atm-deposit?accountId=private&accountType=checking" + + "&goto=https%3A%2F%2Fwww.usaa.com%2Fmy%2Fchecking%3FaccountId%3Dprivate"; + return Object.assign({} as Page, { + evaluate() { + return Promise.resolve({ + account_detail_marker_count: 0, + navigation_marker_count: 0, + target_count: 0, + transaction_marker_count: 0, + }); + }, + goto(target: string) { + visited.push(target); + return Promise.resolve(null); + }, + keyboard: { + press: () => Promise.resolve(), + }, + locator(selector: string) { + // The Export button exists ONLY once we are off the offer page. + const onAccountPage = visited.length > 1; + if (selector === "button.ent-as-utility-bar__item.export" && onAccountPage) { + return { + click: () => Promise.resolve(), + count: () => Promise.resolve(1), + first() { + return this; + }, + }; + } + return { + count: () => Promise.resolve(0), + filter() { + return this; + }, + first() { + return this; + }, + innerHTML: () => Promise.reject(new Error("no dialog")), + waitFor: () => Promise.reject(new Error("timeout waiting for select")), + }; + }, + url() { + return visited.length > 1 ? "https://www.usaa.com/my/checking?accountId=private" : offerUrl; + }, + }); +} + +test("driveExport navigates through the ATM-deposit offer instead of reporting a missing export affordance", async () => { + const visited: string[] = []; + const diagnostics: DiagnosticInfo[] = []; + await driveExport(makeOfferInterstitialPage(visited), "https://www.usaa.com/my/checking?accountId=private", { + onDiagnostics: (info) => diagnostics.push(info), + settleDelayMs: 0, + sinceDate: "2026-01-01", + untilDate: "2026-07-16", + }); + + assert.deepEqual( + visited, + ["https://www.usaa.com/my/checking?accountId=private", "https://www.usaa.com/my/checking?accountId=private"], + "the connector must re-navigate to the account page after landing on the offer" + ); + // The point of the fix: this account is NOT reported as a structural change. + assert.equal( + diagnostics.find((info) => info.phase === "no_export_affordance"), + undefined, + "an offer interstitial must never be reported as a missing export affordance — that is the false " + + "`source_structure_changed` that held USAA transactions at 2 of 4 accounts" + ); +}); + +/** + * The offer page can itself bounce to logon — the session may die DURING the + * escape hop, not only before it. That must surface as the existing + * session-dead outcome (which triggers re-auth and excludes the account from + * the coverage denominator), never as a missing export affordance, which would + * blame the source's UI for an expired session. + */ +test("driveExport reports a session death that happens during the offer escape, not a missing affordance", async () => { + const visited: string[] = []; + const diagnostics: DiagnosticInfo[] = []; + const offerUrl = + "https://www.usaa.com/my/banking-offer/atm-deposit?goto=https%3A%2F%2Fwww.usaa.com%2Fmy%2Fchecking%3FaccountId%3Dprivate"; + const page = Object.assign({} as Page, { + evaluate: () => + Promise.resolve({ + account_detail_marker_count: 0, + navigation_marker_count: 0, + target_count: 0, + transaction_marker_count: 0, + }), + goto(target: string) { + visited.push(target); + return Promise.resolve(null); + }, + keyboard: { press: () => Promise.resolve() }, + locator: () => ({ + count: () => Promise.resolve(0), + filter() { + return this; + }, + first() { + return this; + }, + }), + // First load lands on the offer; the escape hop lands on logon. + url: () => (visited.length > 1 ? "https://www.usaa.com/my/logon" : offerUrl), + }); + + const outcome = await driveExport(page, "https://www.usaa.com/my/checking?accountId=private", { + onDiagnostics: (info) => diagnostics.push(info), + settleDelayMs: 0, + sinceDate: "2026-01-01", + untilDate: "2026-07-16", + }).then( + () => "resolved", + (err: unknown) => (err instanceof Error ? err.message : String(err)) + ); + + assert.equal( + outcome, + "session_dead_redirect_to_logon", + "a logon bounce during the escape hop must raise the session-dead signal" + ); + assert.equal( + diagnostics.find((info) => info.phase === "no_export_affordance"), + undefined, + "an expired session must never be reported as USAA changing their export UI" + ); +}); + test("driveExport records account, challenge, and unrelated routes through the actual no-export path", async () => { for (const { counts, finalUrl, route } of [ { @@ -668,7 +993,8 @@ function makeDialogNotOpenPage(callOrder: string[]): Page { }; } // Every other locator (the dialog select, the unexpected-shape - // dialog probe) reports not found. + // dialog probe) reports not found — the select never renders, so the + // bounded wait times out exactly like a real never-appearing select. return { count() { return Promise.resolve(0); @@ -682,6 +1008,9 @@ function makeDialogNotOpenPage(callOrder: string[]): Page { innerHTML() { return Promise.reject(new Error("no dialog")); }, + waitFor() { + return Promise.reject(new Error("timeout waiting for select")); + }, }; }, url() { @@ -723,6 +1052,115 @@ test("driveExport captures the dialog-not-open checkpoint before pressing Escape ); }); +/** A page whose date-range select renders slightly after the Export click — + * simulating ordinary client-side render variance rather than a genuinely + * missing/changed export affordance. The select locator's `count()` would + * read 0 if sampled immediately (as the old fixed-sleep-then-one-shot-count + * check did), but `waitFor({ state: "visible" })` resolves once it renders. + * Exercises the fix for the false `export_dialog_unexpected_shape` this + * timing gap produced live (run_1787007889181): a slow-but-real render must + * not be reported as `export_affordance_missing`. */ +function makeSlowRenderingSelectPage(): Page { + return Object.assign({} as Page, { + evaluate() { + return Promise.resolve({ + dialog_html_preview: null, + dialogs_open: 1, + export_candidates: [], + has_utility_bar: false, + nav_candidates: [], + title: "", + url: "https://www.usaa.com/my/checking?accountId=private", + }); + }, + goto() { + return Promise.resolve(null); + }, + keyboard: { + press() { + return Promise.resolve(); + }, + }, + locator(selector: string) { + if (selector === "button.ent-as-utility-bar__item.export") { + return { + click() { + return Promise.resolve(); + }, + count() { + return Promise.resolve(1); + }, + first() { + return this; + }, + }; + } + if (selector.includes("selectionType")) { + // Simulates a select that has not yet rendered the instant it is + // checked, but appears shortly after (ordinary client-side render + // variance): `.count()` reads 0 until `waitFor` has resolved once, + // matching how a real Playwright locator's count reflects the live + // DOM only once the element has actually mounted. The old + // fixed-sleep-then-one-shot-count code called `.count()` directly + // without ever calling `waitFor`, so it always saw 0 here and + // misclassified the dialog as unexpected-shape; the fix awaits + // `waitFor` first and only then re-checks `count()`, observing 1. + let renderedAfterWait = false; + return { + count() { + return Promise.resolve(renderedAfterWait ? 1 : 0); + }, + first() { + return this; + }, + waitFor() { + renderedAfterWait = true; + return Promise.resolve(); + }, + }; + } + return { + count() { + return Promise.resolve(0); + }, + filter() { + return this; + }, + first() { + return this; + }, + innerHTML() { + return Promise.reject(new Error("no dialog")); + }, + waitFor() { + return Promise.reject(new Error("timeout")); + }, + }; + }, + url() { + return "https://www.usaa.com/my/checking?accountId=private"; + }, + }); +} + +test("driveExport treats a slow-but-real date-range select render as ready, not a structure change", async () => { + const diagnostics: DiagnosticInfo[] = []; + await driveExport(makeSlowRenderingSelectPage(), "https://www.usaa.com/my/checking", { + onDiagnostics: (info) => diagnostics.push(info), + captureLabel: "usaa-export", + settleDelayMs: 0, + sinceDate: "2026-01-01", + untilDate: "2026-07-16", + }).catch((): undefined => undefined); // downstream submit machinery is unmocked past this point; only the + // dialog-open decision under test needs to run cleanly. + + const fatalPhases = diagnostics.map((d) => d.phase); + assert.ok( + !fatalPhases.includes("export_dialog_unexpected_shape"), + `a select that renders within the wait budget must not be reported as a structure change; saw phases: ${fatalPhases.join(", ")}` + ); +}); + test("runSingleLadderAttempt retains a logon interstitial on the existing re-auth failure outcome", async () => { const { deps, messages } = makeHarness(); deps.browserSurface = "managed"; diff --git a/packages/polyfill-connectors/connectors/usaa/parsers.ts b/packages/polyfill-connectors/connectors/usaa/parsers.ts index 2a8f9f6bf..d2307cfb2 100644 --- a/packages/polyfill-connectors/connectors/usaa/parsers.ts +++ b/packages/polyfill-connectors/connectors/usaa/parsers.ts @@ -7,6 +7,7 @@ import { createHash } from "node:crypto"; import { pathToFileURL } from "node:url"; +import { isStatementSummaryDescription } from "./statement-reconciliation.ts"; import type { AccountRecord, AccountStatsRecord, @@ -415,6 +416,16 @@ function parseModernTxnLine( return null; } const description = descRaw.replace(WS_RUN_2PLUS_RE, " ").trim(); + // USAA's checking-era table prints the period summary as a row shaped + // exactly like a transaction — "02/04 Ending Balance -- -- $33,821.48" — + // so this regex matches it and would store the closing BALANCE as a + // transaction AMOUNT. Live evidence: 14 such rows reached this owner's + // `transactions` stream, with amounts up to $52,334.41 that never + // happened. The summary is not a transaction; drop it here, at the point + // of parse, so no downstream consumer has to know about it. + if (isStatementSummaryDescription(description)) { + return null; + } const amount = currencyToCentsFromStatement(amountRaw); const balance = balanceRaw ? currencyToCentsFromStatement(balanceRaw) : null; if (amount === null) { diff --git a/packages/polyfill-connectors/connectors/usaa/singleton-checkpoint-coverage-wiring.test.ts b/packages/polyfill-connectors/connectors/usaa/singleton-checkpoint-coverage-wiring.test.ts index 4008e61f2..a79bb2632 100644 --- a/packages/polyfill-connectors/connectors/usaa/singleton-checkpoint-coverage-wiring.test.ts +++ b/packages/polyfill-connectors/connectors/usaa/singleton-checkpoint-coverage-wiring.test.ts @@ -40,11 +40,18 @@ import type { BrowserCollectContext, DetailCoverageMessage, DetailGapMessage, + DetailGapStartEntry, EmittedMessage, } from "../../src/connector-runtime.ts"; import { openFingerprintCursor } from "../../src/fingerprint-cursor.ts"; import { makeRecordingEmit } from "../../src/test-harness.ts"; -import { type EmitDeps, runCreditCardBillingStream, runInboxStream, type UsaaRunState } from "./index.ts"; +import { + buildServedCreditCardGapLookups, + type EmitDeps, + runCreditCardBillingStream, + runInboxStream, + type UsaaRunState, +} from "./index.ts"; import { validateRecord } from "./schemas.ts"; import type { DashboardAccount, InboxRow } from "./types.ts"; @@ -86,6 +93,27 @@ function gapsFor(messages: EmittedMessage[], stream: string): DetailGapMessage[] return messages.filter((m): m is DetailGapMessage => m.type === "DETAIL_GAP" && m.stream === stream); } +function recoveriesFor( + messages: EmittedMessage[], + stream: string +): Extract[] { + return messages.filter( + (m): m is Extract => + m.type === "DETAIL_GAP_RECOVERED" && m.stream === stream + ); +} + +function servedCreditCardGap(stream: string, locatorKind: string, cardId: string, gapId: string): DetailGapStartEntry { + return { + gap_id: gapId, + stream, + status: "pending", + reference_only: true, + record_key: cardId, + detail_locator: { kind: locatorKind, card_id: cardId }, + }; +} + /** Runs `fn` with `node:test`'s fake setTimeout enabled and auto-ticking, so * any `politeDelay(ms)` inside resolves immediately instead of waiting for * real wall-clock time. Ticks a large fixed amount after every macrotask @@ -197,6 +225,63 @@ test("wiring: runInboxStream on a genuinely empty inbox proves verified-empty vi }); }); +test("wiring: runInboxStream emits a diagnostic SKIP_RESULT when every listed row fails to resolve a record (live regression: 0/13 covered, no diagnostic ever emitted)", async () => { + await withFastTimers(async () => { + const run = makeHarness(); + // Every row is missing date_short — buildInboxMessageRecord returns null + // for all of them (parsers.ts:579-581), the same shape a column-index + // drift on the inbox table (an extra leading cell, or a re-ordered + // status/date/preview layout) would produce: rows are found (considered + // > 0) but none resolve into a record (covered === 0). Before this fix, + // the coverage math correctly read partial (0 < 13) but the run emitted + // NO SKIP_RESULT and NO diagnostic — the only other USAA streams that can + // silently degrade this way (statements' PDF download, transactions' + // export ladder) always emit a structural diagnostic on failure; inbox + // did not. + const rows: InboxRow[] = Array.from({ length: 13 }, (_unused, i) => ({ + status: "Unread", + date_short: "", + preview: `message ${i}`, + })); + const page = makeInboxPage(rows); + await runInboxStream(run.deps, FAKE_CONTEXT, page, NEVER_CALLED_SEND_INTERACTION, {}, freshRunState()); + + assert.equal(run.emitted.filter((e) => e.stream === "inbox_messages").length, 0, "no row resolved into a record"); + const cov = coverageFor(run.messages, "inbox_messages"); + assert.ok(cov, "coverage is still declared"); + assert.equal(cov?.considered, 13); + assert.equal(cov?.covered, 0, "an honest partial, not a false complete"); + const skips = skipsFor(run.messages, "inbox_messages"); + assert.equal( + skips.length, + 1, + "a total resolution failure (0 covered out of a nonzero considered) must emit a diagnostic SKIP_RESULT, mirroring statements/transactions' structural-drift diagnostics" + ); + assert.equal(skips[0]?.reason, "inbox_rows_unresolved"); + }); +}); + +test("wiring: runInboxStream does NOT emit a diagnostic SKIP_RESULT when only some rows fail to resolve", async () => { + await withFastTimers(async () => { + const run = makeHarness(); + const rows: InboxRow[] = [ + { status: "Unread", date_short: "6/1", preview: "resolves fine" }, + { status: "Read", date_short: "", preview: "missing date" }, + ]; + const page = makeInboxPage(rows); + await runInboxStream(run.deps, FAKE_CONTEXT, page, NEVER_CALLED_SEND_INTERACTION, {}, freshRunState()); + + const cov = coverageFor(run.messages, "inbox_messages"); + assert.equal(cov?.considered, 2); + assert.equal(cov?.covered, 1); + assert.equal( + skipsFor(run.messages, "inbox_messages").length, + 0, + "a partial (not total) resolution gap is not a structural-drift signal — no diagnostic noise on ordinary per-row drops" + ); + }); +}); + // ─── credit_card_billing / credit_card_billing_stats wiring ──────────── /** Per-card-aware fake Page: `.goto` records which card URL was navigated @@ -280,6 +365,81 @@ test("wiring: runCreditCardBillingStream emits DETAIL_COVERAGE for both streams }); }); +test("wiring: runCreditCardBillingStream emits DETAIL_GAP_RECOVERED for a card the runtime served a pending gap for, once it scrapes successfully (live regression: gaps from a crashed run stayed pending forever)", async () => { + await withFastTimers(async () => { + const cc1 = makeCard({ account_id_raw: "CC1", account_url: "/my/credit-card?accountId=CC1", last_four: "0001" }); + const cc1Url = `https://www.usaa.com${cc1.account_url}`; + const { page, billingByUrl } = makeCreditCardPage(); + billingByUrl[cc1Url] = { "Current Balance": "$75.00" }; + + const cardId = "CC1"; // creditCardId() falls back to account_id_raw + const detailGaps: DetailGapStartEntry[] = [ + servedCreditCardGap("credit_card_billing", "usaa.credit_card_billing", cardId, "gap_billing_1"), + servedCreditCardGap("credit_card_billing_stats", "usaa.credit_card_billing_stats", cardId, "gap_stats_1"), + ]; + + const run = makeHarness(); + run.deps.servedCreditCardGaps = buildServedCreditCardGapLookups(detailGaps); + const fingerprintCursor = openFingerprintCursor(undefined, { excludeFromFingerprint: ["fetched_at"] }); + await runCreditCardBillingStream( + run.deps, + FAKE_CONTEXT, + page, + NEVER_CALLED_SEND_INTERACTION, + [cc1], + freshRunState(), + { + emitEntity: true, + emitStats: true, + fingerprintCursor, + observedOn: "2026-06-01", + } + ); + + const billingRecoveries = recoveriesFor(run.messages, "credit_card_billing"); + const statsRecoveries = recoveriesFor(run.messages, "credit_card_billing_stats"); + assert.equal(billingRecoveries.length, 1, "the successfully-scraped card recovers its credit_card_billing gap"); + assert.equal(billingRecoveries[0]?.gap_id, "gap_billing_1"); + assert.equal(billingRecoveries[0]?.record_key, cardId); + assert.equal( + statsRecoveries.length, + 1, + "the same card also recovers its independent credit_card_billing_stats gap" + ); + assert.equal(statsRecoveries[0]?.gap_id, "gap_stats_1"); + }); +}); + +test("wiring: runCreditCardBillingStream does NOT recover a gap for a card the runtime did not serve one for", async () => { + await withFastTimers(async () => { + const cc1 = makeCard({ account_id_raw: "CC1", account_url: "/my/credit-card?accountId=CC1", last_four: "0001" }); + const cc1Url = `https://www.usaa.com${cc1.account_url}`; + const { page, billingByUrl } = makeCreditCardPage(); + billingByUrl[cc1Url] = { "Current Balance": "$75.00" }; + + const run = makeHarness(); + run.deps.servedCreditCardGaps = buildServedCreditCardGapLookups([]); + const fingerprintCursor = openFingerprintCursor(undefined, { excludeFromFingerprint: ["fetched_at"] }); + await runCreditCardBillingStream( + run.deps, + FAKE_CONTEXT, + page, + NEVER_CALLED_SEND_INTERACTION, + [cc1], + freshRunState(), + { + emitEntity: true, + emitStats: true, + fingerprintCursor, + observedOn: "2026-06-01", + } + ); + + assert.equal(recoveriesFor(run.messages, "credit_card_billing").length, 0, "no served gap, no recovery emitted"); + assert.equal(recoveriesFor(run.messages, "credit_card_billing_stats").length, 0); + }); +}); + test("wiring: runCreditCardBillingStream emits SKIP_RESULT and NO coverage when a scrape throws mid-loop", async () => { await withFastTimers(async () => { const cards = [makeCard({ account_id_raw: "CC1" })]; diff --git a/packages/polyfill-connectors/connectors/usaa/statement-pdfs.ts b/packages/polyfill-connectors/connectors/usaa/statement-pdfs.ts index f7c6de1dc..1e8cc3df9 100644 --- a/packages/polyfill-connectors/connectors/usaa/statement-pdfs.ts +++ b/packages/polyfill-connectors/connectors/usaa/statement-pdfs.ts @@ -26,7 +26,7 @@ import { mkdir, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import type { Locator, Page } from "playwright"; +import type { BrowserContext, Locator, Page } from "playwright"; import { attachBodyResponseQueue, type BodyResponseDiagnostics, @@ -36,6 +36,7 @@ import { } from "../../src/browser-artifact-response.ts"; import { resolveConnectorArtifactDir } from "../../src/connector-artifact-root.ts"; import { attachDownloadQueue, type DownloadQueue } from "../../src/download-queue.ts"; +import type { CaptureSession } from "../../src/fixture-capture.ts"; import { readPlaywrightDownloadBuffer } from "../../src/playwright-download.ts"; import { extractStatementContentFingerprint, @@ -53,6 +54,7 @@ import { sha256Hex, yearMonthFromDate, } from "./parsers.ts"; +import { reconcileStatementPeriod, type StatementReconciliation } from "./statement-reconciliation.ts"; import type { DownloadFail, DownloadFailReason, @@ -162,6 +164,45 @@ function attachPdfResponseQueue(page: Page): BodyResponseQueue { // ─── Download orchestration ────────────────────────────────────────────── +/** + * Diagnostic-only instrumentation for the `pdf_download_timeout` hypothesis: + * the Download menuitem may open a NEW page/tab that `attachDownloadQueue` + * (page-scoped, see download-queue.ts:14-24) and `attachPdfResponseQueue` + * cannot see. `context.on('page', ...)` fires for every new page/popup + * created anywhere in the context, regardless of which page's click + * triggered it — this is the direct, minimal way to confirm or rule out the + * hypothesis from a single captured run, without guessing from a trace's + * screenshot timeline. Best-effort and capture-gated: throws never reach the + * caller, and with no CaptureSession this records nothing and costs nothing. + */ +function attachNewPageWatcher( + context: BrowserContext | undefined, + capture: CaptureSession | null | undefined, + labelPrefix: string +): { detach: () => void } { + if (!(context && capture)) { + return { detach: (): void => undefined }; + } + let seq = 0; + const onPage = (newPage: Page): void => { + seq += 1; + const label = `${labelPrefix}-new-page-${seq}`; + // Fire-and-forget: a popup page can be short-lived (e.g. a PDF viewer + // tab that immediately triggers its own download and closes), so this + // must not block the click/consume race in the caller. + capture.captureDom(newPage, label).catch((): undefined => undefined); + process.stderr.write( + `[usaa-statements] new page/popup observed during ${labelPrefix}: url=${newPage.url()} label=${label}\n` + ); + }; + context.on("page", onPage); + return { + detach(): void { + context.off("page", onPage); + }, + }; +} + /** * Locate the per-row "Options" trigger. USAA's documents table renders as a * standard with the trailing cell containing either a button labeled @@ -351,10 +392,22 @@ async function noDownloadMenuitemFailure(page: Page): Promise { } /** Click the Download menuitem and consume the resulting download. */ -async function clickDownloadAndConsume(page: Page, dlItem: Locator): Promise { +async function clickDownloadAndConsume( + page: Page, + dlItem: Locator, + diagCapture?: { capture: CaptureSession | null | undefined; label: string } +): Promise { const downloadQueue = attachDownloadQueue(page); const responseQueue = attachPdfResponseQueue(page); await responseQueue.ready; + // Diagnostic-only: DOM snapshot immediately before the click that is + // hypothesized to open a page the page-scoped download/response queues + // above cannot observe. Paired with attachNewPageWatcher (armed by the + // caller for the whole batch) this is the direct evidence for whether + // the Download menuitem opens a new page/tab. No-op without capture. + if (diagCapture?.capture) { + await diagCapture.capture.captureDom(page, `${diagCapture.label}-pre-click`).catch((): undefined => undefined); + } try { await dlItem.click({ timeout: CLICK_TIMEOUT_MS }); } catch (err) { @@ -401,7 +454,17 @@ async function clickDownloadAndConsume(page: Page, dlItem: Locator): Promise { +async function downloadStatementFromRow({ + page, + rowIndex, + capture, + captureLabel, +}: { + page: Page; + rowIndex: number; + capture?: CaptureSession | null | undefined; + captureLabel?: string | undefined; +}): Promise { const row = page.locator("tbody tr").nth(rowIndex); if (!(await row.count().catch(() => 0))) { return { ok: false, reason: "row_missing" }; @@ -424,7 +487,7 @@ async function downloadStatementFromRow({ page, rowIndex }: { page: Page; rowInd return await noDownloadMenuitemFailure(page); } - return await clickDownloadAndConsume(page, dlItem); + return await clickDownloadAndConsume(page, dlItem, captureLabel ? { capture, label: captureLabel } : undefined); } /** @@ -516,7 +579,8 @@ async function hydrateOneStatement( statement: StatementRow, total: number, hydrated: HydratedStatement[], - { onProgress, onSkip }: HydrateCallbacks + { onProgress, onSkip }: HydrateCallbacks, + capture?: CaptureSession | null ): Promise { if (onProgress) { onProgress({ @@ -528,6 +592,8 @@ async function hydrateOneStatement( const result = await downloadStatementFromRow({ page, rowIndex: statement.rowIndex, + capture, + captureLabel: capture ? `statement-download-row-${statement.rowIndex}` : undefined, }); if (!result.ok) { if (onSkip) { @@ -557,6 +623,8 @@ export async function hydrateStatementPdfs({ statements, onProgress, onSkip, + context, + capture, }: { page: Page; statements: StatementRow[]; @@ -566,6 +634,16 @@ export async function hydrateStatementPdfs({ reason: DownloadFailReason; diag: StatementDownloadDiagnostic | null; }) => void; + /** + * Optional. When supplied together with `capture`, arms a context-level + * `page` event watcher for the whole hydration batch — the direct test of + * the pdf_download_timeout hypothesis (does the Download menuitem open a + * new page the page-scoped download/response queues can't see). Neither + * changes collection behavior; both are diagnostic-only and no-op unless + * PDPP_CAPTURE_FIXTURES=1 / PDPP_CAPTURE_ON_FAILURE=1 armed `capture`. + */ + context?: BrowserContext | undefined; + capture?: CaptureSession | null | undefined; }): Promise { const hydrated: HydratedStatement[] = []; if (!statements.length) { @@ -573,13 +651,25 @@ export async function hydrateStatementPdfs({ } await ensureOnDocumentsPage(page); - for (const s of statements) { - await hydrateOneStatement(page, s, statements.length, hydrated, { - onProgress, - onSkip, - }); - // Small jitter between rows so we don't visibly hammer USAA's SPA. - await sleep(ROW_JITTER_MS); + const newPageWatcher = attachNewPageWatcher(context, capture, "statement-hydration"); + try { + for (const s of statements) { + await hydrateOneStatement( + page, + s, + statements.length, + hydrated, + { + onProgress, + onSkip, + }, + capture + ); + // Small jitter between rows so we don't visibly hammer USAA's SPA. + await sleep(ROW_JITTER_MS); + } + } finally { + newPageWatcher.detach(); } return hydrated; } @@ -704,11 +794,23 @@ export async function parsePdfStatement({ accountId: string; accountName: string | null; period: string | null; -}): Promise<{ txns: StatementTxnRecord[]; parseMeta: ParseMeta }> { +}): Promise<{ + txns: StatementTxnRecord[]; + parseMeta: ParseMeta; + reconciliation: StatementReconciliation; +}> { const text = await extractPdfText(buffer); const closing = resolveClosing(text, period); const { chosen, best } = runEraParsers(text, closing); + // The completeness anchor. Computed here because this is the only place + // that holds BOTH the statement text (USAA's printed period totals) and + // the transactions parsed from it. It is deliberately computed even when + // no parser matched: a period whose balance moved but which yielded zero + // transactions is precisely the failure worth catching, and returning + // early without checking would hide it. + const reconciliation = reconcileStatementPeriod(text, best); + if (!best.length) { return { txns: [], @@ -716,6 +818,7 @@ export async function parsePdfStatement({ era: "unknown", year: closing.closingYear, }, + reconciliation, }; } @@ -727,6 +830,7 @@ export async function parsePdfStatement({ year: closing.closingYear, closingMonth: closing.closingMonth, }, + reconciliation, }; } diff --git a/packages/polyfill-connectors/connectors/usaa/statement-reconciliation.test.ts b/packages/polyfill-connectors/connectors/usaa/statement-reconciliation.test.ts new file mode 100644 index 000000000..fda3c56bd --- /dev/null +++ b/packages/polyfill-connectors/connectors/usaa/statement-reconciliation.test.ts @@ -0,0 +1,216 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Completeness-anchor tests for the USAA `transactions` stream. + * + * The anchor is USAA's own printed period totals. Every line shape asserted + * here was taken verbatim from this owner's real statement PDFs (extracted + * via the connector's own `extractStatementPdfTextAndPages`), not invented — + * including the summary row that caused the live defect: + * + * "02/04 Ending Balance -- -- $33,821.48" + * + * which the modern-era transaction regex matched, storing a $33,821.48 + * closing BALANCE as a transaction AMOUNT. Fourteen such rows reached this + * owner's `transactions` stream, the most recent three days before these + * tests were written. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { parseModernCheckingEra } from "./parsers.ts"; +import { + buildReconciliationDiagnostics, + currencyToCents, + extractPeriodBalances, + isStatementSummaryDescription, + reconcileStatementPeriod, +} from "./statement-reconciliation.ts"; +import type { ParsedStatementTxn } from "./types.ts"; + +function txn(amount: number, description = "MERCHANT"): ParsedStatementTxn { + return { iso: "2026-02-01", amount, description, balance: null, tupleKey: `k${amount}`, ord: 0 }; +} + +/** The real header + summary shape of a USAA checking statement, verbatim + * from 2026-02 (account ...hVAG), with the transaction body elided. */ +const REAL_CHECKING_HEADER = [ + "Statement Period: 01/03/2026 to 02/04/2026", + "Beginning Balance $38,586.78", + "Ending Balance $33,821.48", +].join("\n"); + +// ─── isStatementSummaryDescription: the guard that stops the corruption ── + +test("isStatementSummaryDescription rejects the real Ending Balance summary row", () => { + // Verbatim from the 2026-02 statement, post-regex description capture. + assert.equal(isStatementSummaryDescription("Ending Balance -- --"), true); +}); + +test("isStatementSummaryDescription rejects the real Beginning Balance summary row", () => { + assert.equal(isStatementSummaryDescription("Beginning Balance 0 0"), true); +}); + +test("isStatementSummaryDescription is case- and whitespace-insensitive", () => { + assert.equal(isStatementSummaryDescription(" ending balance -- --"), true); + assert.equal(isStatementSummaryDescription("ENDING BALANCE"), true); +}); + +test("isStatementSummaryDescription keeps a merchant that merely mentions the words", () => { + // A real transaction must not be dropped because its description contains + // the phrase later in the string. The match is anchored at the start. + assert.equal(isStatementSummaryDescription("PAYMENT FOR ENDING BALANCE SERVICES"), false); + assert.equal(isStatementSummaryDescription("BALANCE ENDING LLC"), false); +}); + +test("isStatementSummaryDescription keeps ordinary merchants", () => { + assert.equal(isStatementSummaryDescription("2469216GFBND9G02E AMAZON MKTPL"), false); + assert.equal(isStatementSummaryDescription("ACH WITHDRAWAL 012026"), false); +}); + +// ─── The parser no longer emits summary rows as transactions ───────────── + +test("parseModernCheckingEra drops the summary row that produced the live defect", () => { + // This is the exact table shape from the 2026-02 statement: a real + // one-line transaction plus the two summary rows. + const text = [ + "Transactions", + "Date Description Debits Credits Balance", + "01/03 Beginning Balance 0 0", + "01/20 ACH WITHDRAWAL PREAUTHDFT $152.05", + "02/04 Ending Balance -- -- $33,821.48", + "ENDING BALANCE", + ].join("\n"); + const txns = parseModernCheckingEra(text, { closing: { closingMonth: 2, closingYear: 2026 } }); + const descriptions = txns.map((t) => t.description); + assert.deepEqual(descriptions, ["ACH WITHDRAWAL PREAUTHDFT"]); + // The specific corruption: the closing balance must never appear as an amount. + assert.equal( + txns.some((t) => t.amount === 3_382_148), + false, + "closing balance leaked into transactions as an amount" + ); +}); + +// ─── currencyToCents ───────────────────────────────────────────────────── + +test("currencyToCents parses the real balance figures", () => { + assert.equal(currencyToCents("$33,821.48"), 3_382_148); + assert.equal(currencyToCents("$648.10"), 64_810); + assert.equal(currencyToCents("-$8.65"), -865); +}); + +test("currencyToCents refuses malformed currency rather than coercing it", () => { + // A malformed summary must yield "no anchor", never a wrong anchor. + assert.equal(currencyToCents("--"), null); + assert.equal(currencyToCents("$12"), null); + assert.equal(currencyToCents(""), null); + assert.equal(currencyToCents("abc"), null); +}); + +// ─── extractPeriodBalances: fail closed, never fabricate ───────────────── + +test("extractPeriodBalances reads the real statement summary", () => { + assert.deepEqual(extractPeriodBalances(REAL_CHECKING_HEADER), { + beginningCents: 3_858_678, + endingCents: 3_382_148, + }); +}); + +test("extractPeriodBalances returns null when the era prints no summary", () => { + // The real credit-card era: a closing DATE but no balance summary. This is + // the "no sound anchor" case and must not be papered over. + assert.equal(extractPeriodBalances("Statement Closing Date 01/20/26\nTransactions"), null); +}); + +test("extractPeriodBalances requires BOTH balances", () => { + // One without the other proves nothing about a period. + assert.equal(extractPeriodBalances("Beginning Balance $100.00"), null); + assert.equal(extractPeriodBalances("Ending Balance $100.00"), null); +}); + +test("extractPeriodBalances ignores an in-table row rather than misreading it", () => { + // "02/04 Ending Balance -- -- $33,821.48" is NOT the standalone summary; + // the anchored regex must not accept it, or a statement could be anchored + // against a row the parser also treats as data. + assert.equal(extractPeriodBalances("02/04 Ending Balance -- -- $33,821.48\n01/03 Beginning Balance 0 0"), null); +}); + +// ─── reconcileStatementPeriod: the anchor itself ───────────────────────── + +test("reconcileStatementPeriod confirms a period whose transactions close the balance", () => { + // -476,530 cents is the real 2026-02 delta (38,586.78 -> 33,821.48). + const result = reconcileStatementPeriod(REAL_CHECKING_HEADER, [txn(-400_000), txn(-76_530)]); + assert.equal(result.status, "reconciled"); + assert.equal(result.status === "reconciled" && result.expectedDeltaCents, -476_530); + assert.equal(result.status === "reconciled" && result.observedDeltaCents, -476_530); +}); + +test("reconcileStatementPeriod catches a MISSING transaction", () => { + // The whole point of the anchor: drop one and the identity fails. + const result = reconcileStatementPeriod(REAL_CHECKING_HEADER, [txn(-400_000)]); + assert.equal(result.status, "mismatched"); + assert.equal(result.status === "mismatched" && result.differenceCents, -76_530); +}); + +test("reconcileStatementPeriod catches an INVENTED transaction", () => { + // An equality check detects fabrication as surely as loss — a shortfall + // check would not. + const result = reconcileStatementPeriod(REAL_CHECKING_HEADER, [txn(-400_000), txn(-76_530), txn(-1000)]); + assert.equal(result.status, "mismatched"); + assert.equal(result.status === "mismatched" && result.differenceCents, 1000); +}); + +test("reconcileStatementPeriod flags the real multi-line parse failure", () => { + // This is this owner's ACTUAL live state: the checking-era table wraps + // each transaction across several lines, so the line-oriented parser + // extracts ZERO real transactions while the balance genuinely moved. + // A zero-transaction period must NOT be auto-passed. + const result = reconcileStatementPeriod(REAL_CHECKING_HEADER, []); + assert.equal(result.status, "mismatched"); + assert.equal(result.status === "mismatched" && result.differenceCents, -476_530); +}); + +test("reconcileStatementPeriod accepts a genuinely still period", () => { + // Zero transactions AND no balance movement is a real, complete period. + const still = "Beginning Balance $100.00\nEnding Balance $100.00"; + assert.equal(reconcileStatementPeriod(still, []).status, "reconciled"); +}); + +test("reconcileStatementPeriod excludes summary rows from the sum", () => { + // Were a summary row to reach the sum, it would add the closing balance + // and break the identity on every statement. + const result = reconcileStatementPeriod(REAL_CHECKING_HEADER, [ + txn(-476_530), + txn(3_382_148, "Ending Balance -- --"), + txn(0, "Beginning Balance 0 0"), + ]); + assert.equal(result.status, "reconciled"); +}); + +test("reconcileStatementPeriod reports unavailable when no anchor exists", () => { + // "Cannot check" must stay distinct from "checked and wrong". + const result = reconcileStatementPeriod("Statement Closing Date 01/20/26", [txn(-100)]); + assert.equal(result.status, "unavailable"); + assert.equal(result.status === "unavailable" && result.reason, "no_period_balances"); +}); + +// ─── Diagnostics stay redacted ─────────────────────────────────────────── + +test("buildReconciliationDiagnostics emits only integers and the opaque id", () => { + const result = reconcileStatementPeriod(REAL_CHECKING_HEADER, [txn(-400_000, "AMAZON MKTPL SECRET")]); + assert.equal(result.status, "mismatched"); + if (result.status !== "mismatched") { + return; + } + const diag = buildReconciliationDiagnostics("abc123", result); + assert.deepEqual(Object.keys(diag).sort(), [ + "difference_cents", + "expected_delta_cents", + "observed_delta_cents", + "statement_id", + ]); + // No merchant text may ride along in the diagnostic. + assert.equal(JSON.stringify(diag).includes("AMAZON"), false); +}); diff --git a/packages/polyfill-connectors/connectors/usaa/statement-reconciliation.ts b/packages/polyfill-connectors/connectors/usaa/statement-reconciliation.ts new file mode 100644 index 000000000..63ce2b459 --- /dev/null +++ b/packages/polyfill-connectors/connectors/usaa/statement-reconciliation.ts @@ -0,0 +1,251 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// The `transactions` completeness anchor for USAA checking statements. +// +// WHY THIS IS A REAL ANCHOR, not a derived denominator +// ---------------------------------------------------- +// Every other completeness signal in this connector counts what the run +// itself produced, so it can only ever prove "no gap I could detect". A USAA +// checking statement prints two numbers the bank computed on its own side: +// +// Beginning Balance $38,586.78 +// Ending Balance $33,821.48 +// +// Those are period totals of record. If the transactions parsed for that +// period sum to exactly `ending - beginning`, the statement itself has +// certified that the set is complete — a missing or duplicated transaction +// changes the sum and the identity fails. That is a provider-side proof, and +// it is measured at the source boundary (the PDF text USAA rendered), not +// derived from what this connector chose to emit. +// +// The relation is deliberately an EQUALITY over signed cents: +// +// ending − beginning == Σ(transaction amounts) +// +// It is not a count and not a coverage ratio, so it is immune to the trap +// that a shortfall check falls into: it detects a transaction we invented +// just as surely as one we lost. +// +// WHY THIS IS NOT SUBSTITUTED AS `considered` +// ------------------------------------------- +// The runtime admits a bounded page only when `considered === covered` +// (reference-implementation/server/continuation-proof.ts). Substituting a +// statement-level total as a per-page `considered` would make every run read +// `partial` forever. This module therefore returns a stream-level VERDICT +// about a period, never a page denominator. The caller reports it as its own +// fact; it does not feed the page arithmetic. +// +// CEILING, stated honestly +// ------------------------ +// This proves completeness only for periods USAA published as a checking +// statement AND whose PDF this connector holds. It says nothing about the +// current, unstatemented cycle, and nothing about credit-card statements, +// whose era prints no running-balance column (see `extractPeriodBalances`, +// which returns null rather than guessing). A period with no statement is +// simply unanchored — this module reports that as `unavailable`, never as +// proof of completeness. + +import type { ParsedStatementTxn } from "./types.ts"; + +// ─── Module-scope regexes (Biome useTopLevelRegex) ─────────────────────── + +/** The standalone summary lines USAA prints above the transaction table. + * Anchored to line start and requiring the currency to END the line, so a + * transaction whose merchant description merely CONTAINS the words + * "ending balance" cannot be mistaken for the summary. */ +const BEGINNING_BALANCE_RE = /^Beginning\s+Balance\s+(-?\$[\d,]+\.\d{2})\s*$/i; +const ENDING_BALANCE_RE = /^Ending\s+Balance\s+(-?\$[\d,]+\.\d{2})\s*$/i; + +/** The in-table summary rows, which carry a leading MM/DD and placeholder + * debit/credit columns: "02/04 Ending Balance -- -- $33,821.48" and + * "01/03 Beginning Balance 0 0". These are NOT transactions; see + * `isStatementSummaryDescription`. */ +const SUMMARY_DESCRIPTION_RE = /^(beginning|ending)\s+balance\b/i; + +const CURRENCY_STRIP_RE = /[$,]/g; + +/** A well-formed cents token, after currency symbols are stripped. */ +const BARE_CENTS_RE = /^\d+\.\d{2}$/; + +/** Line splitter for PDF-extracted statement text. */ +const LINE_SPLIT_RE = /\r?\n/; + +const CENTS_MULTIPLIER = 100; + +/** + * True when a parsed statement line is one of USAA's own summary rows rather + * than a transaction. + * + * This exists because USAA's checking-era table prints the period summary as + * a row that is shaped exactly like a transaction: + * + * 02/04 Ending Balance -- -- $33,821.48 + * + * A line-oriented transaction regex matches it and stores the closing + * BALANCE as if it were a transaction AMOUNT. Live evidence: 14 such rows + * were emitted into this owner's `transactions` stream, the most recent of + * them three days before this guard was written, with amounts up to + * $52,334.41 that never happened. + * + * Matching is on the description only and is anchored at the start, so a + * genuine merchant transaction that happens to contain these words later in + * its description is unaffected. + */ +export function isStatementSummaryDescription(description: string): boolean { + return SUMMARY_DESCRIPTION_RE.test(description.trim()); +} + +/** Parse "$33,821.48" / "-$8.65" into signed integer cents. Returns null for + * anything that is not a well-formed currency token, so a malformed summary + * line yields "no anchor" rather than a wrong one. */ +export function currencyToCents(raw: string): number | null { + const negative = raw.trim().startsWith("-"); + const digits = raw.replace(CURRENCY_STRIP_RE, "").replace("-", "").trim(); + if (!BARE_CENTS_RE.test(digits)) { + return null; + } + const value = Math.round(Number(digits) * CENTS_MULTIPLIER); + if (!Number.isFinite(value)) { + return null; + } + return negative ? -value : value; +} + +/** USAA's own period totals for one statement, in signed cents. */ +export interface PeriodBalances { + beginningCents: number; + endingCents: number; +} + +/** + * Read the statement's Beginning/Ending Balance summary from PDF text. + * + * Fails closed by returning `null`: a statement era that prints no such + * summary (the credit-card era) or a statement whose summary is malformed + * has NO anchor, and this module says so rather than substituting a number + * it derived from the transactions — which would make the reconciliation + * check compare the transactions against themselves and pass vacuously. + * + * Both values must be present and well-formed; one without the other proves + * nothing about a period. + */ +export function extractPeriodBalances(text: string): PeriodBalances | null { + let beginningCents: number | null = null; + let endingCents: number | null = null; + for (const raw of text.split(LINE_SPLIT_RE)) { + const line = raw.trim(); + if (beginningCents === null) { + const b = line.match(BEGINNING_BALANCE_RE); + if (b?.[1]) { + beginningCents = currencyToCents(b[1]); + } + } + if (endingCents === null) { + const e = line.match(ENDING_BALANCE_RE); + if (e?.[1]) { + endingCents = currencyToCents(e[1]); + } + } + } + if (beginningCents === null || endingCents === null) { + return null; + } + return { beginningCents, endingCents }; +} + +/** + * The outcome of reconciling one statement period. + * + * `unavailable` is a first-class, non-alarming outcome: it means the period + * offers no sound anchor (no summary balances printed). It is deliberately + * NOT `reconciled: false`, because "cannot check" and "checked and wrong" + * are different facts and collapsing them would either hide a real defect or + * cry wolf on every credit-card statement. + */ +export type StatementReconciliation = + | { status: "unavailable"; reason: "no_period_balances" } + | { + status: "reconciled"; + beginningCents: number; + endingCents: number; + expectedDeltaCents: number; + observedDeltaCents: number; + } + | { + status: "mismatched"; + beginningCents: number; + endingCents: number; + expectedDeltaCents: number; + observedDeltaCents: number; + differenceCents: number; + }; + +/** + * Reconcile one statement period's parsed transactions against USAA's own + * printed period totals. + * + * The identity checked is: + * + * ending − beginning == Σ(amounts) + * + * Summary rows are excluded from the sum via + * `isStatementSummaryDescription`; including them would add the closing + * balance to the transaction sum and break the identity on every statement. + * + * A zero-transaction period is NOT auto-passed: a statement whose balance + * moved but whose table yielded no transactions is exactly the multi-line + * parse failure this owner's data exhibits, and it must surface as + * `mismatched`. The identity handles that correctly with no special case — + * an empty sum reconciles only when the balance genuinely did not move. + */ +export function reconcileStatementPeriod(text: string, txns: readonly ParsedStatementTxn[]): StatementReconciliation { + const balances = extractPeriodBalances(text); + if (!balances) { + return { status: "unavailable", reason: "no_period_balances" }; + } + const { beginningCents, endingCents } = balances; + const expectedDeltaCents = endingCents - beginningCents; + const observedDeltaCents = txns + .filter((t) => !isStatementSummaryDescription(t.description)) + .reduce((acc, t) => acc + t.amount, 0); + if (expectedDeltaCents === observedDeltaCents) { + return { + status: "reconciled", + beginningCents, + endingCents, + expectedDeltaCents, + observedDeltaCents, + }; + } + return { + status: "mismatched", + beginningCents, + endingCents, + expectedDeltaCents, + observedDeltaCents, + differenceCents: expectedDeltaCents - observedDeltaCents, + }; +} + +/** + * Build the redacted diagnostic payload for a failed reconciliation. + * + * Only integers and the opaque statement id hash leave this function — never + * a merchant description, account number, or account name. The balances + * themselves ARE dollar figures for this owner's account, and they are the + * whole point of the finding, so they are reported; the PII rule this + * project enforces is about names and free text, and the `message` carries + * no account identity at all. + */ +export function buildReconciliationDiagnostics( + statementId: string, + result: Extract +): Record { + return { + statement_id: statementId, + expected_delta_cents: result.expectedDeltaCents, + observed_delta_cents: result.observedDeltaCents, + difference_cents: result.differenceCents, + }; +} diff --git a/packages/polyfill-connectors/connectors/venmo/index.ts b/packages/polyfill-connectors/connectors/venmo/index.ts index 04d1d50ce..bf83b6e59 100644 --- a/packages/polyfill-connectors/connectors/venmo/index.ts +++ b/packages/polyfill-connectors/connectors/venmo/index.ts @@ -54,8 +54,9 @@ * Tested surfaces: fixture-driven only (pilot-fixture.test.ts, * parsers.test.ts, schemas.test.ts, integration.test.ts, * src/auto-login/venmo.test.ts). No live network call has proven this - * redesign against a real account yet. The manifest therefore keeps the - * connector in Development until a live run is verified. + * redesign against a real account yet. The manifest lists it at Preview + * (see public_listing.rationale) so the owner can opt in to perform that + * first live run, matching the signal connector's precedent. * * CHANGES * v0.2.0 (2026-08-10) — browser-session redesign; removed @@ -121,6 +122,7 @@ const MAX_TRANSACTION_PAGES = 400; * hand-copied stand-in that could silently drift from it. */ export const VENMO_RETRYABLE_PATTERN = /venmo_rate_limited|venmo_transport_error|venmo_probe_transport_error/i; + // The redesign dropped `venmoPacingProfile`/the HTTP governor (page-context // fetch has no direct outbound Node HTTP to pace — F10 in // /tmp/review-venmo-browser-redesign-0810.md), but the page loops below @@ -198,6 +200,29 @@ function makePageFetch(page: Page): VenmoPageFetch { }; } +/** + * `collect()`'s own call to `ensureVenmoOrigin`, extracted so it is + * unit-testable without a real Playwright `page` (mirrors `errorDetail`/ + * `assertVenmoOk` below, both pulled out of the fetch loop for the same + * reason). `ensureVenmoOrigin` now throws `venmo_origin_navigation_failed` + * when the one-time navigation doesn't land on venmo.com (see its doc — + * production run_1787101857760, the owner's first-ever Venmo run). Folded + * into this connector's own `venmo_transport_error` naming so it matches + * `VENMO_RETRYABLE_PATTERN` the same way any other transport fault in this + * file's fetch loop already does, rather than escaping `collect()` as an + * unrecognized, non-retryable name. + */ +export async function establishVenmoCollectOrigin(page: Page): Promise { + try { + await ensureVenmoOrigin(page); + } catch (err) { + throw new Error( + `venmo_transport_error [origin navigation]: ${redactTransportDetail(err instanceof Error ? err.message : String(err))}`, + { cause: err } + ); + } +} + export function errorDetail(body: string): string { try { const parsed = JSON.parse(body) as { error?: { message?: string } }; @@ -486,8 +511,9 @@ if (isMainModule(import.meta.url)) { // (e.g. `id.venmo.com`); `api.venmo.com`'s CORS allowlist only grants // a credentialed fetch from `https://venmo.com`, so collect must // establish that origin itself rather than assume ensureSession left - // it there (F3 in /tmp/review-venmo-browser-redesign-0810.md). - await ensureVenmoOrigin(page); + // it there (F3 in /tmp/review-venmo-browser-redesign-0810.md). See + // `establishVenmoCollectOrigin`'s doc for why this is wrapped. + await establishVenmoCollectOrigin(page); const fetchPath = makePageFetch(page); const account = await fetchProfile(fetchPath); const ownerId = account?.id; diff --git a/packages/polyfill-connectors/connectors/venmo/integration.test.ts b/packages/polyfill-connectors/connectors/venmo/integration.test.ts index a3c1ae3c2..a43fcff9d 100644 --- a/packages/polyfill-connectors/connectors/venmo/integration.test.ts +++ b/packages/polyfill-connectors/connectors/venmo/integration.test.ts @@ -18,9 +18,16 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import type { Page } from "playwright"; import type { BrowserCollectContext } from "../../src/connector-runtime.ts"; import { makeRecordingEmit } from "../../src/test-harness.ts"; -import { collectAllStreams, collectTransactions, fetchAllFriends, type VenmoPageFetch } from "./index.ts"; +import { + collectAllStreams, + collectTransactions, + establishVenmoCollectOrigin, + fetchAllFriends, + type VenmoPageFetch, +} from "./index.ts"; import { validateRecord } from "./schemas.ts"; const OWNER_ID = "1111111111111111111"; @@ -491,3 +498,50 @@ test("collectAllStreams: never calls globalThis.fetch — every read goes throug globalThis.fetch = original; } }); + +// ─── establishVenmoCollectOrigin: collect()'s own origin guard ───────────── +// +// `ensureSession` may leave the page wherever sign-in redirected it (e.g. +// `id.venmo.com`), so `collect()` re-establishes the `venmo.com` origin +// itself before its first credentialed fetch. Regression coverage for +// production run_1787101857760 (2026-08-18): a navigation that resolves +// without actually landing on venmo.com must fail fast with a diagnosable, +// retryable name — `venmo_transport_error` — rather than let the next fetch +// throw a bare, unclassified "Failed to fetch" from an opaque origin. + +test("establishVenmoCollectOrigin: a stuck-on-about:blank navigation throws venmo_transport_error, not a bare opaque-origin failure", async () => { + const gotoUrls: string[] = []; + const page: Pick = { + goto(url: string): ReturnType { + gotoUrls.push(url); + // Resolves without the page actually leaving about:blank — the exact + // production defect (ensureVenmoOrigin's old `.catch(() => undefined)` + // returned regardless of whether the navigation landed). + return Promise.resolve(null); + }, + url(): string { + return "about:blank"; + }, + }; + await assert.rejects(establishVenmoCollectOrigin(page as Page), (err: unknown) => { + assert.ok(err instanceof Error); + assert.match(err.message, /venmo_transport_error/, "must match VENMO_RETRYABLE_PATTERN, not escape unclassified"); + assert.match(err.message, /venmo_origin_navigation_failed/, "the underlying cause stays legible"); + return true; + }); + assert.deepEqual(gotoUrls, ["https://venmo.com/"]); +}); + +test("establishVenmoCollectOrigin: a successful navigation to venmo.com resolves without throwing", async () => { + let currentUrl = "about:blank"; + const page: Pick = { + goto(url: string): ReturnType { + currentUrl = url; + return Promise.resolve(null); + }, + url(): string { + return currentUrl; + }, + }; + await assert.doesNotReject(establishVenmoCollectOrigin(page as Page)); +}); diff --git a/packages/polyfill-connectors/connectors/whatsapp/attachment-coverage-honesty.test.ts b/packages/polyfill-connectors/connectors/whatsapp/attachment-coverage-honesty.test.ts new file mode 100644 index 000000000..75e2cdcb0 --- /dev/null +++ b/packages/polyfill-connectors/connectors/whatsapp/attachment-coverage-honesty.test.ts @@ -0,0 +1,185 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * A WhatsApp media entry is only known to be unreadable once its own `data()` + * is called during emission (see `findChatTextEntry`'s note about deferring + * skip accounting to `emitAttachmentRecords`). Those skipped files are + * discovered but never collected, so they must read as a coverage shortfall. + * + * Before this contract the attachments DETAIL_COVERAGE used the raw discovered + * count for both `considered` and `covered`, so an export whose media failed to + * extract still reported fully covered — the runtime's coverage contract + * explicitly forbids counting a weighed-but-dropped item as covered. + */ + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import type { EmittedMessage } from "../../src/connector-runtime.ts"; +import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PACKAGE_ROOT = resolve(__dirname, "..", ".."); +const WHATSAPP_ENTRYPOINT = join(__dirname, "index.ts"); + +const CHAT_EXPORT = `[6/5/24, 9:15:22 AM] Alice: Hello +[6/5/24, 9:16:00 AM] Bob: +[6/5/24, 9:17:00 AM] Bob: `; + +function zipHeader(signature: number, size: number): Buffer { + const header = Buffer.alloc(size); + header.writeUInt32LE(signature, 0); + return header; +} + +/** + * Build a stored (uncompressed) zip. `corruptDeflate` marks an entry as + * DEFLATE-compressed while storing raw bytes, so the entry lists cleanly but + * throws when its `data()` is finally called — the real shape of an + * unreadable media file. + */ +function makeZip(entries: readonly { name: string; data: string | Buffer; corruptDeflate?: boolean }[]): Buffer { + const chunks: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const data = Buffer.isBuffer(entry.data) ? entry.data : Buffer.from(entry.data, "utf8"); + const method = entry.corruptDeflate ? 8 : 0; + const local = zipHeader(0x04_03_4b_50, 30); + local.writeUInt16LE(0x08_00, 6); + local.writeUInt16LE(method, 8); + local.writeUInt32LE(0, 14); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(name.length, 26); + chunks.push(local, name, data); + + const directory = zipHeader(0x02_01_4b_50, 46); + directory.writeUInt16LE(20, 4); + directory.writeUInt16LE(20, 6); + directory.writeUInt16LE(0x08_00, 8); + directory.writeUInt16LE(method, 10); + directory.writeUInt32LE(0, 16); + directory.writeUInt32LE(data.length, 20); + directory.writeUInt32LE(data.length, 24); + directory.writeUInt16LE(name.length, 28); + directory.writeUInt32LE(offset, 42); + central.push(directory, name); + offset += local.length + name.length + data.length; + } + const centralStart = offset; + const centralBytes = Buffer.concat(central); + const end = zipHeader(0x06_05_4b_50, 22); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(centralBytes.length, 12); + end.writeUInt32LE(centralStart, 16); + return Buffer.concat([...chunks, centralBytes, end]); +} + +function attachmentCoverage( + messages: readonly EmittedMessage[] +): Extract | undefined { + return messages + .filter( + (message): message is Extract => message.type === "DETAIL_COVERAGE" + ) + .find((message) => message.stream === "attachments"); +} + +test("WhatsApp attachments - media that fails to extract reads as a coverage shortfall", async () => { + const importRoot = await mkdtemp(join(tmpdir(), "pdpp-whatsapp-skip-")); + try { + const stagedDir = join(importRoot, "artifact_skip"); + await mkdir(stagedDir, { recursive: true }); + // Two media files are declared by the chat text and present in the zip. + // The second is unreadable, so only one can actually be collected. + await writeFile( + join(stagedDir, "Alice export.zip"), + makeZip([ + { name: "WhatsApp Chat - Alice.txt", data: CHAT_EXPORT }, + { name: "IMG-20240605-WA0001.jpg", data: Buffer.from([1, 2, 3, 4]) }, + { name: "IMG-20240605-WA0002.jpg", data: Buffer.from([9, 9, 9, 9]), corruptDeflate: true }, + ]) + ); + + const result = await runConnectorProtocolSubprocess({ + cwd: PACKAGE_ROOT, + entrypoint: WHATSAPP_ENTRYPOINT, + env: { + PDPP_OWNER_TOKEN: "", + PDPP_RS_URL: "", + RS_URL: "", + TZ: "America/Chicago", + WHATSAPP_EXPORT_DIR: importRoot, + }, + start: { + scope: { streams: [{ name: "chats" }, { name: "messages" }, { name: "attachments" }] }, + type: "START", + }, + }); + + const emittedAttachments = result.messages.filter( + (message) => message.type === "RECORD" && message.stream === "attachments" + ); + const coverage = attachmentCoverage(result.messages); + + assert.equal(emittedAttachments.length, 1, "only the readable media file can be emitted"); + assert.equal(coverage?.considered, 2, "both media files were discovered in the export"); + assert.equal(coverage?.covered, 1, "the unreadable media file must not count as covered"); + assert.ok( + (coverage?.covered ?? 0) < (coverage?.considered ?? 0), + "an export with unreadable media must not report full coverage" + ); + + const skipResults = result.messages.filter( + (message) => message.type === "SKIP_RESULT" && message.stream === "attachments" + ); + assert.equal(skipResults.length, 1, "the skipped media file is still disclosed as a SKIP_RESULT"); + } finally { + await rm(importRoot, { force: true, recursive: true }); + } +}); + +test("WhatsApp attachments - a fully readable export still proves full coverage", async () => { + const importRoot = await mkdtemp(join(tmpdir(), "pdpp-whatsapp-clean-")); + try { + const stagedDir = join(importRoot, "artifact_clean"); + await mkdir(stagedDir, { recursive: true }); + await writeFile( + join(stagedDir, "Alice export.zip"), + makeZip([ + { name: "WhatsApp Chat - Alice.txt", data: CHAT_EXPORT }, + { name: "IMG-20240605-WA0001.jpg", data: Buffer.from([1, 2, 3, 4]) }, + { name: "IMG-20240605-WA0002.jpg", data: Buffer.from([5, 6, 7, 8]) }, + ]) + ); + + const result = await runConnectorProtocolSubprocess({ + cwd: PACKAGE_ROOT, + entrypoint: WHATSAPP_ENTRYPOINT, + env: { + PDPP_OWNER_TOKEN: "", + PDPP_RS_URL: "", + RS_URL: "", + TZ: "America/Chicago", + WHATSAPP_EXPORT_DIR: importRoot, + }, + start: { + scope: { streams: [{ name: "chats" }, { name: "messages" }, { name: "attachments" }] }, + type: "START", + }, + }); + + const coverage = attachmentCoverage(result.messages); + assert.equal(coverage?.considered, 2); + assert.equal(coverage?.covered, 2, "no media was dropped, so coverage stays complete"); + } finally { + await rm(importRoot, { force: true, recursive: true }); + } +}); diff --git a/packages/polyfill-connectors/connectors/whatsapp/index.ts b/packages/polyfill-connectors/connectors/whatsapp/index.ts index 00090b54e..d5c55ba96 100755 --- a/packages/polyfill-connectors/connectors/whatsapp/index.ts +++ b/packages/polyfill-connectors/connectors/whatsapp/index.ts @@ -743,7 +743,7 @@ async function emitAttachmentRecords( exportOrdinal: number, exportTotal: number, emit: EmitEvent -): Promise<{ emitted: number; processed: number }> { +): Promise<{ covered: number; emitted: number; processed: number }> { let emitted = 0; let skipped = 0; for (let index = 0; index < attachments.length; index += 1) { @@ -780,7 +780,11 @@ async function emitAttachmentRecords( message: `${skipped} media file(s) in WhatsApp export ${exportOrdinal} of ${exportTotal} exceeded the archive read policy and were not imported.`, }); } - return { emitted, processed: attachments.length }; + // `covered` excludes media the read policy dropped. The runtime's coverage + // contract is explicit that a weighed-but-dropped item belongs to neither the + // collected nor the covered count, so counting the raw discovered length here + // would report a chat with skipped media as fully covered. + return { covered: attachments.length - skipped, emitted, processed: attachments.length }; } function openWhatsAppCursors(state: Record): WhatsAppCursors { @@ -1025,8 +1029,9 @@ async function emitParsedExport( progress: EmitProgress, exportOrdinal: number, exportTotal: number -): Promise<{ attachments: number; messages: number; records: number }> { +): Promise<{ attachments: number; attachmentsCovered: number; messages: number; records: number }> { let records = 0; + let attachmentsCovered = 0; if (requested.has("chats")) { await emitChatRecord(summary, cursors.chats, emitRecord); records += 1; @@ -1068,6 +1073,7 @@ async function emitParsedExport( emit ); records += attachmentSummary.emitted; + attachmentsCovered += attachmentSummary.covered; } } @@ -1077,7 +1083,12 @@ async function emitParsedExport( total: exportTotal, type: "PROGRESS", }); - return { attachments: source.attachments.length, messages: summary.messageCount, records }; + return { + attachments: source.attachments.length, + attachmentsCovered, + messages: summary.messageCount, + records, + }; } function pruneRequestedCursors(requested: RequestedStreams, cursors: WhatsAppCursors): void { @@ -1181,6 +1192,7 @@ runConnector({ let importedExports = 0; let totalAttachments = 0; + let totalAttachmentsCovered = 0; let totalMessages = 0; let totalRecords = 0; for (let index = 0; index < files.length; index += 1) { @@ -1239,6 +1251,7 @@ runConnector({ ); importedExports += 1; totalAttachments += emitSummary.attachments; + totalAttachmentsCovered += emitSummary.attachmentsCovered; totalMessages += emitSummary.messages; totalRecords += emitSummary.records; } finally { @@ -1293,7 +1306,9 @@ runConnector({ requiredKeys: [], hydratedKeys: [], considered: totalAttachments, - covered: totalAttachments, + // Media dropped by the bounded-read policy is discovered but not + // collected, so it must not inflate `covered` into a false complete. + covered: totalAttachmentsCovered, }) ); } diff --git a/packages/polyfill-connectors/connectors/ynab/deletion-safe-coverage.test.ts b/packages/polyfill-connectors/connectors/ynab/deletion-safe-coverage.test.ts new file mode 100644 index 000000000..02c7ace35 --- /dev/null +++ b/packages/polyfill-connectors/connectors/ynab/deletion-safe-coverage.test.ts @@ -0,0 +1,91 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * YNAB's coverage proof is deletion-safe. + * + * PDPP deliberately RETAINS records after the provider deletes them, so any + * completeness check that compares a provider total against a held count will + * flag successful preservation as loss. YNAB avoids that trap by construction, + * and these tests pin the two properties that make it work: + * + * 1. A deleted row arrives IN-BAND. YNAB's delta marks a deletion as a + * returned record with `deleted: true` rather than by omitting it, so the + * row is inside the enumerated boundary: it raises `considered`, is + * validated like any other row, and raises `covered` too. A deletion is + * therefore NOT a coverage gap — it is a covered fact about a deletion. + * (The runtime turns it into a tombstone via this connector's + * `isTombstone: (_stream, d) => d.deleted === true`.) + * + * 2. A malformed row still degrades coverage. `covered` is tallied per record + * from the same `validateRecord` verdict the runtime's emitRecord applies, + * never aliased to the response length, so a row that cannot be emitted is + * considered-but-not-covered and reads a real `partial`. + * + * Verified against live data when this was written: this instance holds 9 YNAB + * tombstones (4 payees, 5 transactions) with distinct `deleted_at` timestamps + * across several runs — the in-band deletion signal reaches durable storage. + * YNAB holds 9 of the fleet's 11 tombstones. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { validateRecord } from "./schemas.ts"; + +/** A minimal well-formed payee row, matching the shape `payeeRecord` emits. */ +function payeeRow(overrides: Record = {}): Record { + return { + id: "11111111-1111-4111-8111-111111111111", + budget_id: "22222222-2222-4222-8222-222222222222", + name: "Corner Store", + transfer_account_id: null, + deleted: false, + ...overrides, + }; +} + +test("a deleted payee is a valid, covered record — not a coverage gap", () => { + const deleted = payeeRow({ deleted: true }); + + // The load-bearing property: a deletion passes the SAME shape-check every + // other row passes, so it counts toward `covered`. If deletions were instead + // rejected or omitted, every upstream deletion would permanently depress + // coverage and a correct, fully-preserved run would read `partial` forever. + const verdict = validateRecord("payees", deleted); + assert.equal(verdict.ok, true, "a deleted row must validate like any other row"); +}); + +test("deleting a row does not change the considered/covered ratio", () => { + // The same boundary, once with a live row and once after that row is deleted + // upstream. Both are three-row responses; both must read fully covered. + const idA = "aaaaaaaa-1111-4111-8111-111111111111"; + const idB = "bbbbbbbb-1111-4111-8111-111111111111"; + const idC = "cccccccc-1111-4111-8111-111111111111"; + const live = [payeeRow({ id: idA }), payeeRow({ id: idB }), payeeRow({ id: idC })]; + const afterDeletion = [payeeRow({ id: idA }), payeeRow({ id: idB }), payeeRow({ id: idC, deleted: true })]; + + const coverage = (rows: Record[]): { considered: number; covered: number } => ({ + considered: rows.length, + covered: rows.reduce((n, r) => n + (validateRecord("payees", r).ok ? 1 : 0), 0), + }); + + const before = coverage(live); + const after = coverage(afterDeletion); + + assert.deepEqual(before, { considered: 3, covered: 3 }); + assert.deepEqual(after, { considered: 3, covered: 3 }, "an upstream deletion must not read as coverage loss"); +}); + +test("a malformed row is considered but not covered", () => { + // `id` is required. A row that cannot be emitted must not be claimed as + // covered — this is the guard that keeps `covered` from being a rename of + // `considered`. Without it the ratio could never report a real `partial`. + const rows = [payeeRow(), payeeRow({ id: null })]; + + const considered = rows.length; + const covered = rows.reduce((n, r) => n + (validateRecord("payees", r).ok ? 1 : 0), 0); + + assert.equal(considered, 2); + assert.equal(covered, 1, "a row that fails the shape-check must not be counted as covered"); + assert.ok(covered < considered, "an unemittable row must read partial"); +}); diff --git a/packages/polyfill-connectors/manifests/amazon.json b/packages/polyfill-connectors/manifests/amazon.json index 3b76e6ea2..3ec5cd05a 100644 --- a/packages/polyfill-connectors/manifests/amazon.json +++ b/packages/polyfill-connectors/manifests/amazon.json @@ -47,7 +47,8 @@ "capabilities": { "human_interaction": ["manual_action", "otp"], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", + "recommended_interval_seconds": 21600, "minimum_interval_seconds": 7200, "maximum_staleness_seconds": 86400, "interaction_posture": "otp_likely", @@ -55,7 +56,7 @@ "bot_detection_sensitivity": "high", "background_safe": true, "assisted_after_owner_auth": true, - "rationale": "Amazon remains manual by default because sign-in and anti-bot checks are common, but an owner can explicitly opt into a background schedule after authenticating the connection." + "rationale": "Sign-in and anti-bot checks are common on first login, but the session persists in the owner's browser profile afterward, so scheduled refresh reuses it. background_safe:true records that persistence and recommended_mode follows from it. assisted_after_owner_auth:true means a background run may still ask the owner for bounded help when Amazon re-challenges." }, "public_listing": { "tier": "supported" @@ -115,7 +116,7 @@ "type": "boolean" }, "item_count": { - "type": "integer" + "type": ["integer", "null"] }, "fetched_at": { "type": "string", @@ -295,7 +296,10 @@ } ], "reason_display_messages": { + "amazon_empty_history_after_prior_orders": "Amazon reported no order history for a year in which we previously collected orders. Your stored orders are retained and untouched. We stopped this run instead of recording an empty history, because a page showing no orders can't prove those orders are gone", "empty_first_page_without_diagnostics": "The first page came back empty and we couldn't tell why", + "item_count_shortfall": "An order lists more items than we could save", + "list_page_order_id_not_found": "We skipped some order cards on this page because we couldn't find their order IDs", "empty_first_page_without_terminal_signal": "The first page came back empty with no sign it was really the end", "list_page_navigation_failed": "We couldn't load an orders page, so we stopped before advancing", "list_page_shape_check_failed": "The page didn't look like we expected", diff --git a/packages/polyfill-connectors/manifests/apple_contacts.json b/packages/polyfill-connectors/manifests/apple_contacts.json index 981657b5c..86eff9d34 100644 --- a/packages/polyfill-connectors/manifests/apple_contacts.json +++ b/packages/polyfill-connectors/manifests/apple_contacts.json @@ -280,5 +280,8 @@ { "name": "contact_groups" } ] } - ] + ], + "reason_display_messages": { + "group_inventory_short": "Your address book has contact groups we couldn't save" + } } diff --git a/packages/polyfill-connectors/manifests/chase.json b/packages/polyfill-connectors/manifests/chase.json index 90237fb55..edaf31c11 100644 --- a/packages/polyfill-connectors/manifests/chase.json +++ b/packages/polyfill-connectors/manifests/chase.json @@ -594,6 +594,7 @@ } ], "reason_display_messages": { + "account_no_longer_discovered": "An account we tracked before is no longer listed on your dashboard", "ambiguous_multi_account_overview": "We couldn't tell which account view to use", "qfx_download_failed": "We couldn't download the transactions file", "qfx_parse_failed": "We couldn't read the transactions file", diff --git a/packages/polyfill-connectors/manifests/chatgpt.json b/packages/polyfill-connectors/manifests/chatgpt.json index 3c8ea3d91..c6f44a14b 100644 --- a/packages/polyfill-connectors/manifests/chatgpt.json +++ b/packages/polyfill-connectors/manifests/chatgpt.json @@ -585,6 +585,8 @@ } ], "reason_display_messages": { + "branch_tip_missing": "A conversation points at a message we couldn't find, so part of it is missing", + "branch_truncated": "A conversation is missing earlier messages from its current thread", "empty_detail": "We opened this conversation but found no messages to import", "http_error": "We hit a network problem talking to the service", "missing_mapping": "We opened this conversation but it had no message data to read", diff --git a/packages/polyfill-connectors/manifests/claude_code.json b/packages/polyfill-connectors/manifests/claude_code.json index eae0bd6ac..38c56d4e2 100644 --- a/packages/polyfill-connectors/manifests/claude_code.json +++ b/packages/polyfill-connectors/manifests/claude_code.json @@ -844,6 +844,7 @@ ], "reason_display_messages": { "claude_dir_not_found": "We couldn't find your Claude Code data folder", - "scope_matched_no_sources": "None of your configured project folders matched any Claude Code projects" + "scope_matched_no_sources": "None of your configured project folders matched any Claude Code projects", + "source_unreadable": "We couldn't read this folder, so we don't know what's in it" } } diff --git a/packages/polyfill-connectors/manifests/gmail.json b/packages/polyfill-connectors/manifests/gmail.json index cb42a5aa0..3fb11f48a 100644 --- a/packages/polyfill-connectors/manifests/gmail.json +++ b/packages/polyfill-connectors/manifests/gmail.json @@ -162,8 +162,10 @@ }, "required": ["id", "thread_id", "received_at"] }, + "required": true, "primary_key": ["id"], "cursor_field": "received_at", + "cursor_shape": "imap_uid_band", "consent_time_field": "received_at", "selection": { "fields": true, diff --git a/packages/polyfill-connectors/manifests/google_maps.json b/packages/polyfill-connectors/manifests/google_maps.json index bfd3da911..2d4ccc95d 100644 --- a/packages/polyfill-connectors/manifests/google_maps.json +++ b/packages/polyfill-connectors/manifests/google_maps.json @@ -82,6 +82,7 @@ "streams": [ { "name": "timeline_points", + "required": false, "description": "Timestamped Google Maps Timeline location points.", "display": { "label": "Your Google Maps location points", @@ -162,7 +163,7 @@ "group_by": ["activity_type"] } }, - "coverage_strategy": "checkpoint_window", + "coverage_strategy": "snapshot_import_receipt", "freshness_strategy": "manual_as_of" }, { @@ -242,11 +243,12 @@ "group_by": ["segment_kind"] } }, - "coverage_strategy": "checkpoint_window", + "coverage_strategy": "snapshot_import_receipt", "freshness_strategy": "manual_as_of" } ], "reason_display_messages": { + "element_unaccounted": "Some Timeline entries had no usable time or ID, so we couldn't save them", "invalid_json": "We couldn't read one of your Google Maps Timeline export files", "record_too_large": "This Google Maps Timeline record is too large; re-export your Timeline and try again", "timeline_points_not_found": "We couldn't find any location points to import", diff --git a/packages/polyfill-connectors/manifests/groupme.json b/packages/polyfill-connectors/manifests/groupme.json index d84471529..e81ae2bed 100644 --- a/packages/polyfill-connectors/manifests/groupme.json +++ b/packages/polyfill-connectors/manifests/groupme.json @@ -437,5 +437,10 @@ "coverage_strategy": "parent_detail_accounting", "freshness_strategy": "scheduled_window" } - ] + ], + "reason_display_messages": { + "group_message_count_unanchored": "GroupMe didn't report a message count for some groups, so we can't confirm their history is complete", + "provider_reports_more_messages_than_walked": "GroupMe reports more messages than we collected, so some group history may be missing", + "history_ended_before_provider_count": "Some older group conversations stop partway: GroupMe's total for the group is higher than the history it will actually hand back, which usually means the oldest messages are no longer stored on GroupMe's side. Everything we did collect is saved. There is nothing for you to do, and we retry in case the rest becomes available." + } } diff --git a/packages/polyfill-connectors/manifests/heb.json b/packages/polyfill-connectors/manifests/heb.json index 21e357fc8..397e1dbba 100644 --- a/packages/polyfill-connectors/manifests/heb.json +++ b/packages/polyfill-connectors/manifests/heb.json @@ -47,14 +47,16 @@ "capabilities": { "human_interaction": ["manual_action", "otp"], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", + "recommended_interval_seconds": 21600, "minimum_interval_seconds": 7200, + "maximum_staleness_seconds": 86400, "interaction_posture": "otp_likely", "rate_limit_sensitivity": "medium", "bot_detection_sensitivity": "high", "background_safe": true, "assisted_after_owner_auth": true, - "rationale": "H-E-B is fronted by Imperva Incapsula and supports passkeys plus verification codes; manual by default because sign-in and bot challenges are common. Parsers are verified against real logged-in heb.com DOM (order list + order detail). background_safe:true makes background scheduling available once the owner opts in after authenticating; recommended_mode stays manual so it never auto-runs unattended." + "rationale": "H-E-B is fronted by Imperva Incapsula and supports passkeys plus verification codes, so first login is owner-present; the session then persists in the browser profile and scheduled refresh reuses it. Parsers are verified against real logged-in heb.com DOM (order list + order detail). assisted_after_owner_auth:true means a background run may ask the owner for bounded help when a bot challenge appears." }, "public_listing": { "tier": "supported" @@ -320,6 +322,8 @@ ], "reason_display_messages": { "empty_page_before_max_page": "We hit an empty page of orders before we expected to, so we stopped to check rather than assume we were done", + "heb_empty_history_after_prior_orders": "H-E-B reported no order history, but we previously collected orders for this account. Your stored orders are retained and untouched. We stopped this run instead of recording an empty history, because a page showing no orders can't prove your history is gone", + "item_count_short": "Some orders hold fewer items than H-E-B says they contain", "list_page_navigation_failed": "We couldn't load a page of your H-E-B order history and didn't want to assume that meant the end", "list_page_shape_check_failed": "The page didn't look like we expected", "pagination_exhausted": "We reached the end of the available pages", @@ -327,6 +331,7 @@ "pagination_metadata_contradictory": "H-E-B gave us conflicting page-count information, so we stopped to avoid missing any orders", "selector_drift": "The page layout changed and we couldn't find what we needed", "source_auth_or_challenge": "We need you to sign in or pass a verification check to continue", + "source_reported_empty": "H-E-B showed us your order history and it has no orders in it", "unparseable_order_date": "We skipped some orders because their dates couldn't be read" } } diff --git a/packages/polyfill-connectors/manifests/imessage.json b/packages/polyfill-connectors/manifests/imessage.json index 97dd20e8c..97c0ebe37 100644 --- a/packages/polyfill-connectors/manifests/imessage.json +++ b/packages/polyfill-connectors/manifests/imessage.json @@ -23,15 +23,15 @@ "rationale": "Hidden from the reference dashboard catalog until an operator proves a local macOS Messages database run on this deployment and explicitly opts it into listing." }, "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", "recommended_interval_seconds": 3600, "minimum_interval_seconds": 300, "maximum_staleness_seconds": 21600, "interaction_posture": "none", "rate_limit_sensitivity": "low", "bot_detection_sensitivity": "low", - "background_safe": false, - "rationale": "iMessage history is read from the operator's local macOS Messages database. Docker/provider scheduled runs must use a local collector or an explicitly mounted IMESSAGE_DB_PATH." + "background_safe": true, + "rationale": "iMessage history is read from the operator's local macOS Messages database over a filesystem binding: no network provider, no interactive login, nothing to assist, so unattended refresh is mechanically safe. Whether the database is reachable is a runtime-requirements question (filesystem binding, IMESSAGE_DB_PATH) enforced by the pre-run deployment-readiness gate, not by refresh mode; Docker/provider runs must use a local collector or an explicitly mounted path." } }, "streams": [ diff --git a/packages/polyfill-connectors/manifests/jellyfin.json b/packages/polyfill-connectors/manifests/jellyfin.json index c56018d18..4150cb7c4 100644 --- a/packages/polyfill-connectors/manifests/jellyfin.json +++ b/packages/polyfill-connectors/manifests/jellyfin.json @@ -74,15 +74,15 @@ "capabilities": { "human_interaction": [], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", "recommended_interval_seconds": 3600, "minimum_interval_seconds": 900, "maximum_staleness_seconds": 86400, "interaction_posture": "none", "rate_limit_sensitivity": "low", "bot_detection_sensitivity": "low", - "background_safe": false, - "rationale": "Jellyfin is self-hosted with no public rate limits. Core API provides LastPlayedDate and PlayCount as aggregate state (not per-session history). Items are fetched as a full inventory snapshot each run (Jellyfin core API does not expose a true incremental cursor for items; only library-level metadata). Hourly polling is conservative for a manually proven instance, but version compatibility is unproven and credentialed deployment compatibility remains unproven. PlaybackReporting plugin is optional and marks v2 scope expansion; core v1 scope is limited to aggregate playback metadata." + "background_safe": true, + "rationale": "Jellyfin is a self-hosted API-key connector with no public rate limits and no interactive login, so unattended refresh is mechanically safe. Core API provides LastPlayedDate and PlayCount as aggregate state (not per-session history); items are fetched as a full inventory snapshot each run because the core API exposes no true incremental cursor. Version and credentialed-deployment compatibility remain unproven, which public_listing.tier:preview records — auto-enrollment gates on tier, not on refresh mode. PlaybackReporting plugin is optional and marks v2 scope expansion; core v1 scope is limited to aggregate playback metadata." }, "public_listing": { "tier": "preview" diff --git a/packages/polyfill-connectors/manifests/notion.json b/packages/polyfill-connectors/manifests/notion.json index 0c5c9fd55..01ff8f1b5 100644 --- a/packages/polyfill-connectors/manifests/notion.json +++ b/packages/polyfill-connectors/manifests/notion.json @@ -37,15 +37,15 @@ "capabilities": { "human_interaction": [], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", "recommended_interval_seconds": 3600, "minimum_interval_seconds": 900, "maximum_staleness_seconds": 86400, "interaction_posture": "none", "rate_limit_sensitivity": "medium", "bot_detection_sensitivity": "low", - "background_safe": false, - "rationale": "Notion is unproven pending credentialed live testing; scheduled background runs are deferred until owner validation completes." + "background_safe": true, + "rationale": "Notion is a server-to-server API-token connector: no browser, no interactive login, nothing to assist, so unattended refresh is mechanically safe. It remains unproven pending credentialed live testing, which public_listing.tier:preview records — auto-enrollment gates on tier, not on refresh mode." }, "public_listing": { "tier": "preview" diff --git a/packages/polyfill-connectors/manifests/oura.json b/packages/polyfill-connectors/manifests/oura.json index 99951198f..bc0349235 100644 --- a/packages/polyfill-connectors/manifests/oura.json +++ b/packages/polyfill-connectors/manifests/oura.json @@ -37,15 +37,15 @@ "capabilities": { "human_interaction": [], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", "recommended_interval_seconds": 21600, "minimum_interval_seconds": 3600, "maximum_staleness_seconds": 86400, "interaction_posture": "none", "rate_limit_sensitivity": "low", "bot_detection_sensitivity": "low", - "background_safe": false, - "rationale": "Oura is unproven pending credentialed live testing; scheduled background runs are deferred until owner validation completes." + "background_safe": true, + "rationale": "Oura uses a personal access token over HTTPS: no browser, no interactive login, nothing to assist, so unattended refresh is mechanically safe. It remains unproven pending credentialed live testing, which public_listing.tier:development records — auto-enrollment gates on tier, not on refresh mode." }, "public_listing": { "tier": "development" diff --git a/packages/polyfill-connectors/manifests/pocket.json b/packages/polyfill-connectors/manifests/pocket.json index a507fc76c..32fb6ec66 100644 --- a/packages/polyfill-connectors/manifests/pocket.json +++ b/packages/polyfill-connectors/manifests/pocket.json @@ -16,14 +16,12 @@ "human_interaction": [], "refresh_policy": { "recommended_mode": "manual", - "recommended_interval_seconds": 21600, "minimum_interval_seconds": 3600, - "maximum_staleness_seconds": 86400, - "interaction_posture": "none", + "interaction_posture": "manual_action_likely", "rate_limit_sensitivity": "low", "bot_detection_sensitivity": "low", "background_safe": false, - "rationale": "Pocket upstream API was shut down by Mozilla on 2025-07-08; the connector cannot run regardless of refresh mode." + "rationale": "Pocket's upstream API was shut down by Mozilla on 2025-07-08, so no unattended refresh path exists and the connector cannot collect at all; it is retained only so existing records stay readable." }, "public_listing": { "tier": "development" diff --git a/packages/polyfill-connectors/manifests/reddit.json b/packages/polyfill-connectors/manifests/reddit.json index 728d74216..49a4d8a75 100644 --- a/packages/polyfill-connectors/manifests/reddit.json +++ b/packages/polyfill-connectors/manifests/reddit.json @@ -47,7 +47,8 @@ "capabilities": { "human_interaction": ["manual_action", "otp"], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", + "recommended_interval_seconds": 21600, "minimum_interval_seconds": 7200, "maximum_staleness_seconds": 86400, "interaction_posture": "otp_likely", @@ -55,7 +56,7 @@ "bot_detection_sensitivity": "medium", "background_safe": true, "assisted_after_owner_auth": true, - "rationale": "Reddit remains manual by default because first login from a new profile commonly serves a 2FA/OTP step or a Cloudflare challenge, but the session persists in the profile afterward, so an owner can explicitly opt into a background schedule after authenticating the connection." + "rationale": "First login from a new profile commonly serves a 2FA/OTP step or a Cloudflare challenge, but the session persists in the profile afterward, so scheduled refresh reuses it. assisted_after_owner_auth:true means a background run may ask the owner for bounded help when Reddit re-challenges." }, "public_listing": { "tier": "supported" diff --git a/packages/polyfill-connectors/manifests/signal.json b/packages/polyfill-connectors/manifests/signal.json new file mode 100644 index 000000000..2221a1451 --- /dev/null +++ b/packages/polyfill-connectors/manifests/signal.json @@ -0,0 +1,295 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/signal", + "connector_key": "signal", + "manifest_uri": "https://registry.pdpp.dev/connectors/signal", + "version": "0.1.0", + "display_name": "Signal Desktop", + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + }, + "desktop_session": { + "required": true, + "rationale": "Signal Desktop's SQLCipher key is stored encrypted in config.json behind Electron safeStorage and unwraps only through a session-bound OS keyring (kwallet6/gnome-keyring, Keychain, DPAPI). Copying the database to another host is insufficient: the key is not in the file. This connector can therefore only run on the owner's logged-in desktop session." + } + }, + "environment_variables": [ + "SIGTOP_BIN", + "SIGNAL_ATTACHMENTS_EXPORT_DIR", + "PDPP_SIGNAL_MAX_ATTACHMENT_BYTES" + ], + "external_tools": [ + { + "name": "sigtop", + "license": "ISC", + "purpose": "Decrypts and reads Signal Desktop's local encrypted db.sqlite (SQLCipher key unwrap via the OS keychain) and exports attachments", + "install_hint": "go install github.com/tbvdm/sigtop@latest. Linux builds additionally require libsecret-1-dev and pkg-config at build time." + } + ] + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "public_listing": { + "tier": "preview", + "rationale": "Real collection logic (query/parse/emit against a live Signal Desktop database via the sigtop sidecar), verified by unit tests and a mocked-subprocess integration test. The sigtop binary itself is now built from pinned source (v0.24.0, ISC) into the Core image and smoke-verified at build time, so the sidecar is proven present and executable on the shipped runtime. Listed as Preview rather than Supported because no repeatable live run against a real Signal account has been recorded yet, and only Linux is exercised. The owner opted this into listing to perform that first real run." + }, + "refresh_policy": { + "recommended_mode": "automatic", + "recommended_interval_seconds": 3600, + "minimum_interval_seconds": 300, + "maximum_staleness_seconds": 21600, + "interaction_posture": "none", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": true, + "rationale": "Signal history is read from the operator's local Signal Desktop database via the sigtop subprocess: no network provider, no interactive login, nothing to assist, so unattended refresh is mechanically safe. The binding constraints — a filesystem mount plus a live desktop session for the OS keyring that unwraps the SQLCipher key — are runtime requirements enforced by the pre-run deployment-readiness gate, not by refresh mode. No repeatable live run has been recorded, which public_listing.tier:preview records." + } + }, + "streams": [ + { + "name": "messages", + "required": true, + "description": "Signal Desktop messages, read via sigtop's decrypted export-database SQLite copy.", + "display": { + "label": "Your Signal messages", + "detail": "Sender, conversation, text content, timestamp, message type, has_attachments/is_edited flags derived from Signal Desktop's own per-message JSON envelope." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "conversation_id": { + "type": "string" + }, + "sender": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "sent_at": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "body": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "type": { + "type": ["string", "null"] + }, + "has_attachments": { + "type": "boolean" + }, + "is_edited": { + "type": "boolean" + } + }, + "required": ["id", "conversation_id", "sent_at"] + }, + "primary_key": ["id"], + "cursor_field": "sent_at", + "consent_time_field": "sent_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "search": { + "lexical_fields": ["body", "sender"], + "semantic_fields": ["body"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "conversations", + "required": false, + "description": "Signal Desktop conversations (direct and group), read via sigtop's decrypted export-database SQLite copy.", + "display": { + "label": "Your Signal conversations", + "detail": "Conversation type (private/group), title. member_count is always null: Signal Desktop's schema exposes no flat member-count field this connector can honestly derive without fabrication (see connectors/signal/parsers.ts's buildConversationRecord)." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": ["string", "null"], + "enum": ["private", "group", null] + }, + "title": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "member_count": { + "type": ["integer", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["title"], + "semantic_fields": ["title"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "reactions", + "required": false, + "description": "Emoji reactions to Signal Desktop messages, derived from each message row's own JSON envelope (Signal Desktop has no standalone reactions table).", + "display": { + "label": "Your Signal message reactions", + "detail": "Emoji, reacting participant, and the message reacted to." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "message_id": { + "type": "string" + }, + "emoji": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "sender": { + "type": "string", + "x_pdpp_role": "actor" + } + }, + "required": ["id", "message_id", "emoji", "sender"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["sender"] + } + }, + "relationships": [ + { + "name": "message", + "stream": "messages", + "foreign_key": "message_id", + "cardinality": "has_one" + } + ], + "coverage_strategy": "parent_detail_accounting", + "freshness_strategy": "manual_as_of" + }, + { + "name": "attachments", + "required": false, + "description": "Attachment metadata and bytes exported via `sigtop export-attachments`, joined against Signal Desktop's message_attachments table when available (schema version >= 1360 only).", + "display": { + "label": "Your Signal attachments", + "detail": "Filename, MIME type, size, linked message when the join succeeds, and a blob reference when the local file was hydrated. Bytes are read only from inside sigtop's own decrypted export directory (a connector-controlled trusted root \u2014 canonicalized and verified before any read, rejecting `../` traversal and a symlink that escapes the root) since Signal Desktop encrypts attachment blobs at rest and only sigtop knows how to decrypt them. Optional: on a Signal Desktop schema older than version 1360 (no message_attachments table), attachment bytes still hydrate but message_id/content_type degrade to null/best-effort rather than the stream failing." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "message_id": { + "type": ["string", "null"] + }, + "conversation_id": { + "type": ["string", "null"] + }, + "filename": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "content_type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "content_sha256": { + "type": ["string", "null"] + }, + "hydration_status": { + "type": "string", + "enum": ["deferred", "hydrated", "failed", "too_large", "missing"] + }, + "hydration_error": { + "type": ["string", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + }, + "required": ["blob_id", "mime_type", "sha256", "size_bytes"] + } + }, + "required": ["id", "filename", "content_type", "hydration_status"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "relationships": [ + { + "name": "message", + "stream": "messages", + "foreign_key": "message_id", + "cardinality": "has_one" + } + ], + "coverage_strategy": "parent_detail_accounting", + "freshness_strategy": "manual_as_of" + } + ], + "reason_display_messages": { + "backfill_check_unproven_legacy_cursor": "We couldn't check for older messages this run; the next run will check properly", + "message_date_unusable": "We skipped a message with a date we couldn't read", + "no_attachments_exported": "sigtop found no attachments to export for this account", + "source_rows_below_watermark_unreachable": "Signal has older messages we haven't saved — run a full refresh to collect them" + } +} diff --git a/packages/polyfill-connectors/manifests/slack.json b/packages/polyfill-connectors/manifests/slack.json index 35c6b86d7..abed382a4 100644 --- a/packages/polyfill-connectors/manifests/slack.json +++ b/packages/polyfill-connectors/manifests/slack.json @@ -1532,6 +1532,9 @@ } ], "reason_display_messages": { + "channel_history_never_requested": "Some channels you're in have never been collected at all, because the archive stopped before reaching them", + "channel_history_not_finalized": "Some channels you're in have unproven history, so their messages may be incomplete", + "channel_history_out_of_member_scope": "Some public channels weren't collected because you're not a member of them. Set the Slack option MEMBER_ONLY to false to collect every channel your workspace lists, including ones you've left and ones that are archived", "optional_stream_failed": "We couldn't collect one part of your Slack data, so we skipped it", "source_partition_missing": "Some previously seen Slack channels are missing from this export, so message coverage is incomplete" } diff --git a/packages/polyfill-connectors/manifests/spotify.json b/packages/polyfill-connectors/manifests/spotify.json index eda4c871c..15515aa7a 100644 --- a/packages/polyfill-connectors/manifests/spotify.json +++ b/packages/polyfill-connectors/manifests/spotify.json @@ -41,15 +41,15 @@ "rationale": "Hidden from the reference dashboard catalog until a credentialed run proves useful records in the deployment." }, "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", "recommended_interval_seconds": 21600, "minimum_interval_seconds": 3600, "maximum_staleness_seconds": 86400, "interaction_posture": "credentials", "rate_limit_sensitivity": "medium", "bot_detection_sensitivity": "low", - "background_safe": false, - "rationale": "Spotify v0.1 requires a user-provided SPOTIFY_ACCESS_TOKEN and is not yet proven in the reference deployment; do not schedule it as an automatic background connector until a credentialed run succeeds." + "background_safe": true, + "rationale": "Spotify v0.1 uses a user-provided SPOTIFY_ACCESS_TOKEN over HTTPS with no interactive login, so unattended refresh is mechanically safe. It is not yet proven in the reference deployment, which public_listing.tier:development records — auto-enrollment gates on tier, not on refresh mode." } }, "streams": [ diff --git a/packages/polyfill-connectors/manifests/steam.json b/packages/polyfill-connectors/manifests/steam.json index 6d32833ea..106bdcc72 100644 --- a/packages/polyfill-connectors/manifests/steam.json +++ b/packages/polyfill-connectors/manifests/steam.json @@ -50,15 +50,15 @@ "capabilities": { "human_interaction": [], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", "recommended_interval_seconds": 3600, "minimum_interval_seconds": 3600, "maximum_staleness_seconds": 86400, "interaction_posture": "none", "rate_limit_sensitivity": "high", "bot_detection_sensitivity": "low", - "background_safe": false, - "rationale": "Steam Web API does not publish rate limits. PDPP policy: start at a 250ms per-request floor, honor Retry-After when supplied, and preserve 429/throttled transient failures for recovery; 403 is treated as an authorization or visibility failure. One-hour polling is conservative to avoid community-reported multi-hour lockouts. Preview remains manual and background_safe stays false until repeated live runs support unattended collection." + "background_safe": true, + "rationale": "Steam Web API is an API-key connector with no interactive login, so unattended refresh is mechanically safe. Steam does not publish rate limits; PDPP policy is a 250ms per-request floor, honoring Retry-After when supplied, and preserving 429/throttled transient failures for recovery, with 403 treated as an authorization or visibility failure. One-hour polling is conservative to avoid community-reported multi-hour lockouts. Repeated live runs are still outstanding, which public_listing.tier:preview records — auto-enrollment gates on tier, not on refresh mode." }, "public_listing": { "tier": "preview" diff --git a/packages/polyfill-connectors/manifests/strava.json b/packages/polyfill-connectors/manifests/strava.json index 75eb0acef..44fdda66d 100644 --- a/packages/polyfill-connectors/manifests/strava.json +++ b/packages/polyfill-connectors/manifests/strava.json @@ -15,15 +15,15 @@ "capabilities": { "human_interaction": [], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", "recommended_interval_seconds": 21600, "minimum_interval_seconds": 3600, "maximum_staleness_seconds": 86400, "interaction_posture": "none", "rate_limit_sensitivity": "medium", "bot_detection_sensitivity": "low", - "background_safe": false, - "rationale": "Manual refresh is retained while the owner setup path remains unproven." + "background_safe": true, + "rationale": "Strava uses a bearer access token over HTTPS: no browser, no interactive login, nothing to assist, so unattended refresh is mechanically safe. The owner setup path remains unproven, which public_listing.tier:development records — auto-enrollment gates on tier, not on refresh mode." }, "public_listing": { "tier": "development" diff --git a/packages/polyfill-connectors/manifests/usaa.json b/packages/polyfill-connectors/manifests/usaa.json index eb7fcc718..e55413696 100644 --- a/packages/polyfill-connectors/manifests/usaa.json +++ b/packages/polyfill-connectors/manifests/usaa.json @@ -648,6 +648,7 @@ "export_error": "The export couldn't be downloaded", "export_no_download": "The export didn't produce a downloadable file", "hydrate_crashed": "Something went wrong while loading the page", + "inbox_rows_unresolved": "We found inbox messages but couldn't read them — the site may have changed", "pdf_download_click_failed": "We couldn't click the download option for a statement PDF", "pdf_download_direct_link_failed": "We couldn't download a statement PDF from its direct link", "pdf_download_empty": "A statement PDF download didn't produce any file", @@ -661,6 +662,7 @@ "pdf_template_unknown": "We don't recognize the format of this statement yet", "scrape_failed": "We couldn't read the page contents", "selectors_pending": "Support for this part of the connector isn't complete yet", - "session_dead_reauth_failed": "Your sign-in expired and we couldn't refresh it" + "session_dead_reauth_failed": "Your sign-in expired and we couldn't refresh it", + "statement_unreconciled": "A statement's transactions don't add up to its printed balances" } } diff --git a/packages/polyfill-connectors/manifests/venmo.json b/packages/polyfill-connectors/manifests/venmo.json index 53347a909..f50bad070 100644 --- a/packages/polyfill-connectors/manifests/venmo.json +++ b/packages/polyfill-connectors/manifests/venmo.json @@ -47,6 +47,12 @@ }, "capabilities": { "human_interaction": ["manual_action", "otp"], + "declared_reason_tokens": [ + "venmo_probe_transport_error", + "venmo_post_submit_probe_transport_error", + "venmo_origin_navigation_failed", + "venmo_password_screen_timeout" + ], "refresh_policy": { "recommended_mode": "manual", "recommended_interval_seconds": 21600, @@ -56,11 +62,11 @@ "rate_limit_sensitivity": "medium", "bot_detection_sensitivity": "high", "background_safe": false, - "assisted_after_owner_auth": true, - "rationale": "Manual only while this connector stays unproven and unlisted. Once proven, the session persists in the profile after first login, so background scheduling can be revisited then." + "rationale": "Venmo requires an interactive login that may serve a one-time code, and this connector has not proven that the profile session survives across runs, so refresh stays owner-present. Maturity is tracked by public_listing.tier:preview; if a live run proves session reuse, declare background_safe:true and the recommended mode follows." }, "public_listing": { - "tier": "development" + "tier": "preview", + "rationale": "Real collection logic (browser-session auth against api.venmo.com's own JSON endpoints via the page's own cookie jar, matching the reddit/amazon pattern), verified by 70 unit/integration tests covering parsers, schemas, cursor behavior, redaction, and the credential-submit retry boundary (B4). Listed as Preview rather than Supported because no live run against a real Venmo account has been recorded yet. The owner opted this into listing to perform that first real run." } }, "streams": [ diff --git a/packages/polyfill-connectors/manifests/whoop.json b/packages/polyfill-connectors/manifests/whoop.json index 05dbec9f2..bc6a63ad8 100644 --- a/packages/polyfill-connectors/manifests/whoop.json +++ b/packages/polyfill-connectors/manifests/whoop.json @@ -14,7 +14,8 @@ "capabilities": { "human_interaction": ["manual_action"], "refresh_policy": { - "recommended_mode": "manual", + "recommended_mode": "automatic", + "recommended_interval_seconds": 21600, "minimum_interval_seconds": 7200, "maximum_staleness_seconds": 86400, "interaction_posture": "manual_action_likely", @@ -22,7 +23,7 @@ "bot_detection_sensitivity": "medium", "background_safe": true, "assisted_after_owner_auth": true, - "rationale": "The owner signs in through an isolated WHOOP browser profile. A live owner run proved the full backfill and a second checkpointed run reused the saved session without prompting, so the owner can opt into background scheduling after authenticating the connection." + "rationale": "The owner signs in through an isolated WHOOP browser profile. A live owner run proved the full backfill and a second checkpointed run reused the saved session without prompting, so scheduled refresh reuses that session. assisted_after_owner_auth:true means a background run may ask the owner for bounded help if the session lapses." }, "public_listing": { "tier": "preview", diff --git a/packages/polyfill-connectors/package.json b/packages/polyfill-connectors/package.json index eff3799d0..3b415fd63 100644 --- a/packages/polyfill-connectors/package.json +++ b/packages/polyfill-connectors/package.json @@ -19,13 +19,14 @@ "scripts": { "check": "ultracite check", "check:noAwaitInLoops-conformance": "node --import tsx ./scripts/check-no-await-in-loops-conformance.ts", + "check:noDirectCredentialEnv": "node --import tsx ./scripts/check-no-direct-credential-env.ts", "check:noUnnecessaryConditions-expiry": "node ./scripts/check-no-unnecessary-conditions-expiry.ts", "conformance": "tsx scripts/conformance.ts", "fix": "ultracite fix", "postinstall": "node ./scripts/install-patchright-browser.ts", "test": "node --import tsx ../../scripts/test-scratch/run-command.ts -- bash -c 'node --test --import tsx --test-concurrency=2 --test-timeout=120000 \"bin/**/*.test.ts\" \"connectors/**/*.test.ts\" \"src/**/*.test.ts\"'", "typecheck": "tsc --noEmit", - "verify": "pnpm typecheck && pnpm check && pnpm check:noAwaitInLoops-conformance && pnpm check:noUnnecessaryConditions-expiry" + "verify": "pnpm typecheck && pnpm check && pnpm check:noAwaitInLoops-conformance && pnpm check:noDirectCredentialEnv && pnpm check:noUnnecessaryConditions-expiry" }, "dependencies": { "@pdpp/collector-runtime": "file:../../vendor/pdpp-collector-runtime-0.0.1.tgz", diff --git a/packages/polyfill-connectors/scripts/check-no-direct-credential-env.ts b/packages/polyfill-connectors/scripts/check-no-direct-credential-env.ts new file mode 100644 index 000000000..ce4a26e73 --- /dev/null +++ b/packages/polyfill-connectors/scripts/check-no-direct-credential-env.ts @@ -0,0 +1,178 @@ +#!/usr/bin/env node +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Bans direct `process.env._USERNAME` / `_PASSWORD` credential + * reads in connector code. + * + * ## Why + * + * A connector's sign-in credentials must come from the `credentials` object + * the runtime resolves for THIS run's connection and threads into + * `ensureSession` (see `src/auto-login/login-credentials.ts` for the full + * rationale). Reading `process.env` directly reaches the same value in the + * happy case — the reference server injects the connection-scoped fragment + * into the child's environment — but it silently discards everything the + * runtime built around the declared path: + * + * - a connector that reads `process.env` has no reason to declare an `auth` + * block, and four did not (`heb`, `chase`, `amazon`, `chatgpt`), so the + * runtime resolved `{}` and never raised the `credentials` INTERACTION + * that would have told the owner a credential was expected; + * - an absent value falls through to a generic "hand the page to the owner" + * branch whose message blames the PAGE, not the credential. + * + * The result was a run that bailed to manual sign-in within seconds and gave + * the owner a misleading reason, indefinitely. + * + * ## What this enforces + * + * Under `src/auto-login/` and `connectors/`, a credential-shaped + * `process.env` read is an error. Use `resolveLoginCredentials(credentials, + * …)` instead. Files still carrying the old shape are listed in + * `MIGRATION_ALLOWLIST` with the connector that owns them; the list may only + * shrink. A file that leaves the list can never silently rejoin it, and a NEW + * file can never be added to it without an explicit edit here — which is what + * makes the wrong thing hard to do rather than merely discouraged. + * + * Test files are exempt: they legitimately set and restore these variables to + * drive the very fallbacks being migrated away from. + */ + +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const SCAN_ROOTS = [join(PACKAGE_ROOT, "src", "auto-login"), join(PACKAGE_ROOT, "connectors")]; + +/** + * Credential-shaped environment reads. Deliberately narrow: this bans the + * per-account SIGN-IN pair, not every `process.env` read. Tokens and API keys + * that a connector legitimately receives only via injection (and that have no + * interactive sign-in form) are out of scope here. + */ +const CREDENTIAL_ENV_PATTERN = /process\.env\.([A-Z][A-Z0-9_]*_(?:USERNAME|PASSWORD|EMAIL))\b/g; + +/** + * Files that still read credentials from `process.env`, pending migration to + * `resolveLoginCredentials`. THIS LIST MAY ONLY SHRINK. + * + * Each entry is a path relative to the package root. `heb.ts` is listed + * because a concurrent change owns that file; the rest are queued behind it so + * this gate could land without a mass rewrite that would collide with in-flight + * work. + */ +const MIGRATION_ALLOWLIST: ReadonlySet = new Set([ + "connectors/amazon/index.ts", + "connectors/jellyfin/index.ts", + "connectors/reddit/index.ts", + "src/auto-login/amazon.ts", + "src/auto-login/chatgpt.ts", + "src/auto-login/github.ts", + "src/auto-login/heb.ts", + "src/auto-login/reddit.ts", +]); + +function isScannable(path: string): boolean { + return path.endsWith(".ts") && !path.endsWith(".test.ts") && !path.endsWith(".d.ts"); +} + +function walk(root: string, into: string[]): void { + let entries: string[]; + try { + entries = readdirSync(root); + } catch { + return; + } + for (const entry of entries) { + if (entry === "node_modules" || entry === "fixtures") { + continue; + } + const full = join(root, entry); + if (statSync(full).isDirectory()) { + walk(full, into); + } else if (isScannable(full)) { + into.push(full); + } + } +} + +interface Violation { + readonly file: string; + readonly line: number; + readonly variable: string; +} + +function scan(): { violations: Violation[]; seenAllowlisted: Set } { + const files: string[] = []; + for (const root of SCAN_ROOTS) { + walk(root, files); + } + const violations: Violation[] = []; + const seenAllowlisted = new Set(); + for (const file of files.sort((a, b) => a.localeCompare(b))) { + const rel = relative(PACKAGE_ROOT, file); + const lines = readFileSync(file, "utf8").split("\n"); + for (const [index, text] of lines.entries()) { + // Skip comments: this file and login-credentials.ts NAME these variables + // in prose explaining why they are banned. + const trimmed = text.trim(); + if (trimmed.startsWith("*") || trimmed.startsWith("//")) { + continue; + } + CREDENTIAL_ENV_PATTERN.lastIndex = 0; + let match = CREDENTIAL_ENV_PATTERN.exec(text); + while (match !== null) { + if (MIGRATION_ALLOWLIST.has(rel)) { + seenAllowlisted.add(rel); + } else { + violations.push({ file: rel, line: index + 1, variable: match[1] ?? "" }); + } + match = CREDENTIAL_ENV_PATTERN.exec(text); + } + } + } + return { seenAllowlisted, violations }; +} + +const { violations, seenAllowlisted } = scan(); +const stale = [...MIGRATION_ALLOWLIST] + .filter((entry) => !seenAllowlisted.has(entry)) + .sort((a, b) => a.localeCompare(b)); + +let failed = false; + +if (violations.length > 0) { + failed = true; + console.error( + `\n${violations.length} direct credential env read(s) found. Connector sign-in credentials must come from the\n` + + "runtime-resolved `credentials` object via `resolveLoginCredentials`\n" + + "(src/auto-login/login-credentials.ts), never from process.env:\n" + ); + for (const violation of violations) { + console.error(` ${violation.file}:${violation.line} process.env.${violation.variable}`); + } + console.error(""); +} + +if (stale.length > 0) { + failed = true; + console.error( + "\nMIGRATION_ALLOWLIST entries no longer read credentials from process.env.\n" + + "Remove them from scripts/check-no-direct-credential-env.ts — the list may only shrink:\n" + ); + for (const entry of stale) { + console.error(` ${entry}`); + } + console.error(""); +} + +if (failed) { + process.exit(1); +} + +console.log( + `check-no-direct-credential-env: OK (${MIGRATION_ALLOWLIST.size} file(s) pending migration, 0 new violations)` +); diff --git a/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts b/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts index 61ea3ef6f..6c90cc2f1 100644 --- a/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts +++ b/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts @@ -188,70 +188,70 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/amazon/index.ts", - line: 811, + line: 844, column: 20, category: "ordered_browser_interaction", note: "resolveOrderDetail(): sequential Playwright action against the shared page/context", }, { path: "connectors/amazon/index.ts", - line: 821, + line: 854, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/amazon/index.ts", - line: 856, + line: 889, column: 20, category: "ordered_browser_interaction", note: "recoverPendingOrderItemDetailGapPage(): sequential Playwright action against the shared page/context", }, { path: "connectors/amazon/index.ts", - line: 1042, + line: 1076, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/amazon/index.ts", - line: 1060, + line: 1134, column: 7, category: "ordered_protocol_emission", note: "emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/amazon/index.ts", - line: 1468, + line: 1608, column: 5, category: "dependent_pagination", note: "deps.progress(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/amazon/index.ts", - line: 1479, + line: 1619, column: 7, category: "dependent_pagination", note: "deps.progress(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/amazon/index.ts", - line: 1668, + line: 1808, column: 11, category: "dependent_pagination", note: "progress(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/amazon/index.ts", - line: 1718, + line: 1858, column: 11, category: "dependent_pagination", note: "progress(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/amazon/integration.test.ts", - line: 377, + line: 378, column: 21, category: "test_assertion_sequencing", note: "processListOrder(): test drives/asserts an ordered per-case side effect", @@ -272,175 +272,175 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/chase/index.ts", - line: 1399, + line: 1400, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 1589, + line: 1590, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 1643, + line: 1644, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 1750, + line: 1751, column: 7, category: "ordered_protocol_emission", note: "deps.emit(): statement DETAIL_GAP emissions preserve source outcome order", }, { path: "connectors/chase/index.ts", - line: 2227, + line: 2228, column: 5, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 2259, + line: 2260, column: 21, category: "ordered_browser_interaction", note: "processAccountDownload(): sequential Playwright action against the shared page/context", }, { path: "connectors/chase/index.ts", - line: 2548, + line: 2549, column: 7, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 2336, + line: 2451, column: 17, category: "ordered_protocol_emission", note: "emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 1193, + line: 1194, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 1927, + line: 1928, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 2074, + line: 2188, column: 5, category: "dependent_pagination", note: "deps.api.fetch(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/chatgpt/index.ts", - line: 2359, + line: 2474, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 2449, + line: 2564, column: 7, category: "dependent_pagination", note: "deps.api.fetch(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/chatgpt/index.ts", - line: 2477, + line: 2592, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 2569, + line: 2684, column: 7, category: "dependent_pagination", note: "deps.api.fetch(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/chatgpt/index.ts", - line: 2866, + line: 2981, column: 19, category: "provider_pacing_backpressure", note: "fetchChatGptPressureProbeStatus(): rate-limited/budget-gated external call", }, { path: "connectors/chatgpt/index.ts", - line: 3501, + line: 3616, column: 19, category: "provider_pacing_backpressure", note: "fetchBatch(): rate-limited/budget-gated external call", }, { path: "connectors/chatgpt/index.ts", - line: 3531, + line: 3646, column: 7, category: "ordered_protocol_emission", note: "emitConversationDetailGapOnce(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 3564, + line: 3679, column: 7, category: "ordered_protocol_emission", note: "emitConversationDetailGapOnce(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 3584, + line: 3699, column: 11, category: "ordered_protocol_emission", note: "emitConversationDetailGapOnce(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 3780, + line: 3895, column: 16, category: "provider_pacing_backpressure", note: "fetchConversationDetailWaitingOutCircuit(): rate-limited/budget-gated external call", }, { path: "connectors/chatgpt/index.ts", - line: 3862, + line: 3977, column: 16, category: "dependent_pagination", note: "deps.api.fetch(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/chatgpt/index.ts", - line: 4014, + line: 4129, column: 20, category: "provider_pacing_backpressure", note: "recoverPendingConversationDetailGapPage(): rate-limited/budget-gated external call", }, { path: "connectors/chatgpt/index.ts", - line: 4149, + line: 4264, column: 7, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chatgpt/index.ts", - line: 4346, + line: 4461, column: 7, category: "ordered_protocol_emission", note: "emitConversation(): Collection Profile protocol emission requiring in-order delivery", @@ -454,133 +454,133 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/claude_code/index.ts", - line: 385, + line: 386, column: 5, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/claude_code/index.ts", - line: 496, + line: 497, column: 9, category: "dependent_file_cursor", note: "walk(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 535, + line: 537, column: 9, category: "dependent_file_cursor", note: "walk(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 670, + line: 733, column: 13, category: "dependent_file_cursor", note: "readBoundedUtf8(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 749, + line: 817, column: 9, category: "dependent_file_cursor", note: "walk(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 800, + line: 866, column: 13, category: "dependent_file_cursor", note: "readBoundedUtf8(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 884, + line: 953, column: 5, category: "dependent_file_cursor", note: "processJsonlFile(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 915, + line: 984, column: 7, category: "dependent_file_cursor", note: "processJsonlFile(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 964, + line: 1033, column: 5, category: "dependent_file_cursor", note: "processSessionDir(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 998, + line: 1067, column: 5, category: "dependent_file_cursor", note: "scanProjectDir(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 1276, + line: 1345, column: 17, category: "dependent_file_cursor", note: "readdir(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 1434, + line: 1503, column: 7, category: "ordered_protocol_emission", note: "input.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/claude_code/index.ts", - line: 1494, + line: 1563, column: 5, category: "ordered_protocol_emission", note: "input.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/claude_code/index.ts", - line: 1601, + line: 1670, column: 7, category: "ordered_protocol_emission", note: "input.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/claude_code/index.ts", - line: 1627, + line: 1696, column: 5, category: "ordered_protocol_emission", note: "emitGatedInventoryStream(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/claude_code/index.ts", - line: 1917, + line: 2012, column: 29, category: "dependent_file_cursor", note: "scanSessionSource(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 1940, + line: 2035, column: 31, category: "dependent_file_cursor", note: "scanSessionSource(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 2026, + line: 2121, column: 27, category: "dependent_file_cursor", note: "scanChildSource(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/claude_code/index.ts", - line: 1582, + line: 1651, column: 5, category: "ordered_protocol_emission", note: "emitDerivedCoverage(): Collection Profile protocol emission requiring in-order delivery", @@ -629,35 +629,35 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/codex/index.ts", - line: 582, + line: 593, column: 11, category: "dependent_file_cursor", note: "isDirectoryPath(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/codex/index.ts", - line: 1822, + line: 1833, column: 5, category: "ordered_protocol_emission", note: "waitForEmitDrain(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/codex/index.ts", - line: 1878, + line: 1889, column: 7, category: "ordered_protocol_emission", note: "waitForEmitDrain(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/codex/index.ts", - line: 1939, + line: 1950, column: 21, category: "ordered_protocol_emission", note: "resolveGatedInventoryRecords(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/codex/index.ts", - line: 2041, + line: 2052, column: 5, category: "ordered_protocol_emission", note: "waitForEmitDrain(): Collection Profile protocol emission requiring in-order delivery", @@ -671,147 +671,168 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/github/index.ts", - line: 404, + line: 441, column: 5, category: "ordered_protocol_emission", note: "ctx.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/github/index.ts", - line: 428, + line: 465, column: 5, category: "dependent_pagination", note: "guardGithubPagination(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/github/index.ts", - line: 503, + line: 540, column: 5, category: "ordered_protocol_emission", note: "ctx.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/github/index.ts", - line: 525, + line: 562, column: 5, category: "dependent_pagination", note: "guardGithubPagination(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/github/index.ts", - line: 599, + line: 636, column: 5, category: "ordered_protocol_emission", note: "ctx.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/github/index.ts", - line: 623, + line: 660, column: 5, category: "dependent_pagination", note: "guardGithubPagination(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/github/index.ts", - line: 866, + line: 903, column: 18, category: "ordered_protocol_emission", note: "emitPullRequestItem(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/github/index.ts", - line: 903, + line: 940, column: 5, category: "dependent_pagination", note: "guardGithubPagination(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/github/index.ts", - line: 986, + line: 1023, column: 20, category: "dependent_pagination", note: "drainPrSearchWindow(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/github/index.ts", - line: 1089, + line: 1126, column: 5, category: "ordered_protocol_emission", note: "ctx.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/github/index.ts", - line: 1112, + line: 1149, column: 5, category: "dependent_pagination", note: "guardGithubPagination(): next request depends on the prior page's cursor/offset/response", }, + { + path: "connectors/gmail/integration.test.ts", + line: 2171, + column: 16, + category: "test_assertion_sequencing", + note: "run(): each run feeds the prior run's committed cursor into the next; the walk is the assertion", + }, + { + path: "src/auto-login/chatgpt.ts", + line: 300, + column: 9, + category: "bounded_retry_polling", + note: "checkSession(): retry/backoff/poll loop gated on the prior attempt's outcome", + }, { path: "connectors/gmail/index.ts", - line: 584, + line: 743, column: 23, category: "ordered_protocol_emission", note: "deps.hydrateAttachment(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/gmail/index.ts", - line: 601, + line: 760, column: 7, category: "ordered_protocol_emission", note: "deps.emitProtocol(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/gmail/index.ts", - line: 623, + line: 782, column: 5, category: "ordered_protocol_emission", note: "emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/gmail/index.ts", - line: 769, + line: 928, column: 25, category: "ordered_protocol_emission", note: "processMessage(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/gmail/index.ts", - line: 1024, + line: 1222, column: 7, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/gmail/index.ts", - line: 1888, + line: 2195, column: 7, category: "ordered_protocol_emission", note: "deps.emitProtocol(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/gmail/index.ts", - line: 2088, + line: 2395, column: 18, category: "provider_pacing_backpressure", note: "processServedAttachmentRecoveryGap(): rate-limited/budget-gated external call", }, { path: "connectors/gmail/index.ts", - line: 3023, + line: 3338, + column: 25, + category: "provider_pacing_backpressure", + note: "fetchBodiesFn(): one IMAP command at a time on a non-concurrent connection; a parallel fetch would nest commands and hang the connection", + }, + { + path: "connectors/gmail/index.ts", + line: 3429, column: 5, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/gmail/index.ts", - line: 712, + line: 871, column: 7, category: "ordered_protocol_emission", note: "uploadBodyBlob(): canonical field bytes must bind before their record is emitted", }, { path: "connectors/gmail/index.ts", - line: 3233, + line: 3654, column: 26, category: "shared_mutable_accumulator", note: "runThreadsPass(): each range updates the shared fingerprint cursor and accumulated coverage count", @@ -860,21 +881,21 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/google_maps/index.ts", - line: 223, + line: 239, column: 11, category: "ordered_protocol_emission", note: "processPointRecords(): ordered schema admission and point protocol emission", }, { path: "connectors/google_maps/index.ts", - line: 266, + line: 286, column: 11, category: "ordered_protocol_emission", note: "processSegmentRecords(): ordered schema admission and segment protocol emission", }, { path: "connectors/google_maps/index.ts", - line: 378, + line: 419, column: 5, category: "dependent_file_cursor", note: "loadExports(): sequential file progress follows the discovered source order", @@ -923,49 +944,49 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/heb/index.ts", - line: 281, + line: 282, column: 7, category: "ordered_protocol_emission", note: "emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/heb/index.ts", - line: 692, + line: 718, column: 20, category: "ordered_browser_interaction", note: "resolveOrderDetail(): sequential Playwright action against the shared page/context", }, { path: "connectors/heb/index.ts", - line: 640, - column: 20, + line: 738, + column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/heb/index.ts", - line: 660, - column: 9, + line: 770, + column: 20, category: "ordered_browser_interaction", note: "recoverPendingOrderItemDetailGapPage(): sequential Playwright action against the shared page/context", }, { path: "connectors/heb/index.ts", - line: 744, + line: 822, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/heb/index.ts", - line: 842, + line: 933, column: 22, category: "ordered_browser_interaction", note: "loadListPage(): sequential Playwright action against the shared page/context", }, { path: "connectors/heb/index.ts", - line: 868, + line: 959, column: 7, category: "ordered_browser_interaction", note: "processListOrder(): sequential Playwright action against the shared page/context", @@ -1028,7 +1049,7 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/notion/index.ts", - line: 274, + line: 288, column: 5, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", @@ -1063,133 +1084,161 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/reddit/index.ts", - line: 561, + line: 616, column: 20, category: "dependent_pagination", note: "progress(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/reddit/index.ts", - line: 299, + line: 331, column: 5, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/reddit/index.ts", - line: 419, + line: 462, column: 5, category: "dependent_pagination", note: "collectStream(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/reddit/integration.test.ts", - line: 1126, + line: 1182, column: 5, category: "test_assertion_sequencing", note: "collectStream(): test drives/asserts an ordered per-case side effect", }, { path: "connectors/reddit/integration.test.ts", - line: 1080, + line: 1136, column: 5, category: "test_assertion_sequencing", note: "collectStream(): test drives/asserts an ordered per-case side effect", }, { path: "connectors/slack/index.ts", - line: 1271, + line: 1542, column: 20, category: "shared_mutable_accumulator", note: "refreshScopedArchive(): loop body mutates a shared accumulator the next iteration reads", }, { path: "connectors/slack/index.ts", - line: 1710, - column: 7, - category: "ordered_protocol_emission", - note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", - }, - { - path: "connectors/slack/index.ts", - line: 1371, + line: 1642, column: 9, category: "shared_mutable_accumulator", note: "runRequestedStreams(): loop body mutates a shared accumulator the next iteration reads", }, { path: "connectors/slack/index.ts", - line: 1608, - column: 7, + line: 2091, + column: 25, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1617, + line: 2104, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1612, + line: 2099, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1829, + line: 2347, column: 5, category: "ordered_protocol_emission", note: "emitWithFingerprint(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1893, + line: 2411, column: 7, category: "ordered_protocol_emission", note: "emitWithFingerprint(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 2254, + line: 2798, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 2278, + line: 2822, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, + { + path: "connectors/signal/index.ts", + line: 688, + column: 5, + category: "ordered_protocol_emission", + note: "emitMessageRowsAndReactions(): emitRecord() Collection Profile protocol emission requiring in-order delivery", + }, + { + path: "connectors/signal/index.ts", + line: 709, + column: 7, + category: "ordered_protocol_emission", + note: "emitReactionRowsFromMessages(): emitRecord() Collection Profile protocol emission requiring in-order delivery", + }, + { + path: "connectors/signal/index.ts", + line: 723, + column: 5, + category: "ordered_protocol_emission", + note: "emitConversationRows(): emitRecord() Collection Profile protocol emission requiring in-order delivery", + }, + { + path: "connectors/signal/index.ts", + line: 957, + column: 9, + category: "dependent_file_cursor", + note: "listExportedAttachmentFiles(): sequential recursive directory walk over sigtop's exported attachment tree", + }, + { + path: "connectors/signal/index.ts", + line: 1029, + column: 20, + category: "provider_pacing_backpressure", + note: "resolveAttachmentHydration(): rate-limited/budget-gated blob-upload call", + }, { path: "connectors/slack/index.ts", - line: 2286, + line: 2830, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 2294, + line: 2838, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 2335, + line: 2879, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 2725, + line: 3263, column: 32, category: "dependent_file_cursor", note: "reclaimUploads(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", @@ -1259,127 +1308,134 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/usaa/index.ts", - line: 322, + line: 351, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 468, + line: 497, column: 7, category: "ordered_protocol_emission", note: "emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 773, + line: 802, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 838, + line: 867, column: 5, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 1156, + line: 908, + column: 5, + category: "ordered_protocol_emission", + note: "deps.emit(): DETAIL_GAP_RECOVERED emission requiring in-order delivery", + }, + { + path: "connectors/usaa/index.ts", + line: 1309, column: 7, category: "ordered_browser_interaction", note: "page.goto(): sequential Playwright action against the shared page/context", }, { path: "connectors/usaa/index.ts", - line: 1842, + line: 2010, column: 21, category: "bounded_retry_polling", note: "runSingleLadderAttempt(): retry/backoff/poll loop gated on the prior attempt's outcome", }, { path: "connectors/usaa/index.ts", - line: 1960, + line: 2128, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 2093, + line: 2312, column: 5, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 2302, + line: 2521, column: 7, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 2508, + line: 2756, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 2542, + line: 2795, column: 5, category: "ordered_protocol_emission", note: "processPdfStatementRow(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 2735, + line: 2994, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/index.ts", - line: 3001, + line: 3329, column: 23, category: "ordered_browser_interaction", note: "navigateToCardOrGap(): sequential Playwright action against the shared page/context", }, { path: "connectors/usaa/index.ts", - line: 3022, + line: 3351, column: 11, category: "ordered_protocol_emission", note: "emitCreditCardNavFailureGaps(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/usaa/integration.test.ts", - line: 611, + line: 936, column: 21, category: "test_assertion_sequencing", note: "driveExport(): test drives/asserts an ordered per-case side effect", }, { path: "connectors/usaa/statement-pdfs.ts", - line: 180, + line: 221, column: 9, category: "ordered_browser_interaction", note: "c.count(): sequential Playwright action against the shared page/context", }, { path: "connectors/usaa/statement-pdfs.ts", - line: 203, + line: 244, column: 9, category: "ordered_browser_interaction", note: "c.count(): sequential Playwright action against the shared page/context", }, { path: "connectors/usaa/statement-pdfs.ts", - line: 577, - column: 5, + line: 657, + column: 7, category: "ordered_browser_interaction", note: "hydrateOneStatement(): sequential Playwright action against the shared page/context", }, @@ -1392,28 +1448,28 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/venmo/index.ts", - line: 240, + line: 265, column: 5, category: "dependent_pagination", note: "progress(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/venmo/index.ts", - line: 289, + line: 314, column: 5, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/venmo/index.ts", - line: 319, + line: 344, column: 5, category: "dependent_pagination", note: "progress(): next request depends on the prior page's cursor/offset/response", }, { path: "connectors/venmo/index.ts", - line: 401, + line: 426, column: 7, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", @@ -1441,14 +1497,14 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/whatsapp/index.ts", - line: 1130, + line: 1141, column: 7, category: "ordered_protocol_emission", note: "emitStateForCursor(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/whatsapp/index.ts", - line: 1192, + line: 1204, column: 39, category: "dependent_file_cursor", note: "parseExportFile(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", @@ -1600,128 +1656,149 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ category: "ordered_browser_interaction", note: "el.isVisible(): sequential Playwright action against the shared page/context", }, + { + path: "src/auto-login/chase.ts", + line: 197, + column: 32, + category: "ordered_browser_interaction", + note: "Promise.all(): sequential Playwright action against the shared page/context", + }, { path: "src/auto-login/chatgpt.ts", - line: 341, + line: 393, column: 7, category: "ordered_browser_interaction", note: "candidate.waitFor(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/chatgpt.ts", - line: 530, + line: 583, column: 5, category: "bounded_retry_polling", note: "checkpoint(): retry/backoff/poll loop gated on the prior attempt's outcome", }, { path: "src/auto-login/chatgpt.ts", - line: 882, + line: 935, column: 5, category: "bounded_retry_polling", note: "page.waitForTimeout(): retry/backoff/poll loop gated on the prior attempt's outcome", }, { path: "src/auto-login/heb.test.ts", - line: 893, + line: 1559, column: 7, category: "test_assertion_sequencing", note: "assert.rejects(): test drives/asserts an ordered per-case side effect", }, { path: "src/auto-login/heb.test.ts", - line: 942, + line: 1608, column: 7, category: "test_assertion_sequencing", note: "assert.rejects(): test drives/asserts an ordered per-case side effect", }, { path: "src/auto-login/heb.test.ts", - line: 1009, + line: 1675, column: 7, category: "test_assertion_sequencing", note: "assert.rejects(): test drives/asserts an ordered per-case side effect", }, { path: "src/auto-login/heb.test.ts", - line: 1038, + line: 1704, column: 16, category: "test_assertion_sequencing", note: "ensureHebSession(): test drives/asserts an ordered per-case side effect", }, { path: "src/auto-login/heb.ts", - line: 75, + line: 144, column: 32, category: "ordered_browser_interaction", note: "Promise.all(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 94, + line: 163, column: 19, category: "ordered_browser_interaction", note: "locator.count(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 97, + line: 166, column: 34, category: "ordered_browser_interaction", note: "Promise.all(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 118, + line: 187, column: 19, category: "ordered_browser_interaction", note: "locator.count(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 121, + line: 190, column: 34, category: "ordered_browser_interaction", note: "Promise.all(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 147, + line: 263, column: 32, category: "ordered_browser_interaction", note: "Promise.all(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 181, + line: 294, column: 32, category: "ordered_browser_interaction", note: "Promise.all(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 326, + line: 328, + column: 38, + category: "ordered_browser_interaction", + note: "Promise.all(): sequential Playwright action against the shared page/context", + }, + { + path: "src/auto-login/heb.ts", + line: 420, + column: 11, + category: "bounded_retry_polling", + note: "hasDismissibleInterstitial(): retry/backoff/poll loop gated on the prior attempt's outcome", + }, + { + path: "src/auto-login/heb.ts", + line: 656, column: 21, category: "ordered_browser_interaction", note: "inspectPostSubmitAuthSurface(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 472, + line: 839, column: 20, category: "ordered_browser_interaction", note: "fillWhenUsable(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 486, + line: 853, column: 40, category: "ordered_browser_interaction", note: "Promise.all(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/heb.ts", - line: 675, + line: 1107, column: 22, category: "bounded_retry_polling", note: "waitForUniqueVerificationCodeFormRoot(): retry/poll until the remounted OTP surface is uniquely actionable", @@ -1735,35 +1812,49 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "src/auto-login/reddit.test.ts", - line: 96, + line: 134, column: 9, category: "bounded_retry_polling", note: "makeLocator().waitFor(): bounded test double polling until the simulated locator attaches", }, { path: "src/auto-login/reddit.ts", - line: 280, - column: 10, + line: 396, + column: 9, + category: "bounded_retry_polling", + note: "isSessionLiveWithRetry(): bounded post-manual-handoff re-probe — don't trust a single isSessionLive check right after the owner's continue click", + }, + { + path: "src/auto-login/reddit.ts", + line: 669, + column: 19, category: "bounded_retry_polling", note: "hasSessionCookie(): retry/backoff/poll loop gated on the prior attempt's outcome", }, + { + path: "src/auto-login/reddit.ts", + line: 567, + column: 7, + category: "ordered_protocol_emission", + note: "drainProbeTimeouts(): probe-timeout checkpoints must reach the runtime watchdog in the order they were observed", + }, { path: "src/auto-login/usaa.ts", - line: 158, + line: 174, column: 19, category: "ordered_browser_interaction", note: "action.count(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/usaa.ts", - line: 163, + line: 179, column: 18, category: "ordered_browser_interaction", note: "locator.isVisible(): sequential Playwright action against the shared page/context", }, { path: "src/auto-login/usaa.ts", - line: 304, + line: 347, column: 18, category: "bounded_retry_polling", note: "requestOtp(): retry/backoff/poll loop gated on the prior attempt's outcome", @@ -1861,14 +1952,14 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "src/local-device-runtime.ts", - line: 261, + line: 262, column: 5, category: "shared_mutable_accumulator", note: "queue.enqueue(): loop body mutates a shared accumulator the next iteration reads", }, { path: "src/local-device-runtime.ts", - line: 325, + line: 326, column: 18, category: "shared_mutable_accumulator", note: "input.queue.dequeueReady(): loop body mutates a shared accumulator the next iteration reads", @@ -1931,11 +2022,18 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/apple_contacts/carddav-client.ts", - line: 76, + line: 87, column: 41, category: "dependent_pagination", note: "davRequest(): next redirect hop depends on the prior response's Location header", }, + { + path: "connectors/apple_contacts/carddav-client.ts", + line: 398, + column: 17, + category: "provider_pacing_backpressure", + note: "addressbookMultiget(): sequential bounded chunks so one change set cannot issue an unbounded parallel fan-out at the provider", + }, { path: "connectors/apple_contacts/discovery.ts", line: 251, @@ -1945,35 +2043,42 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/apple_contacts/index.ts", - line: 451, + line: 609, column: 7, category: "ordered_protocol_emission", note: "emitContactRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/apple_contacts/index.ts", - line: 455, + line: 633, column: 9, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/apple_contacts/index.ts", - line: 475, + line: 653, column: 7, category: "ordered_protocol_emission", note: "emitContactRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/apple_contacts/index.ts", - line: 330, + line: 421, + column: 7, + category: "ordered_protocol_emission", + note: "hydrateMissingBodies(): emitContactRecord() Collection Profile protocol emission requiring in-order delivery", + }, + { + path: "connectors/apple_contacts/index.ts", + line: 379, column: 5, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/apple_contacts/index.ts", - line: 628, + line: 827, column: 13, category: "shared_mutable_accumulator", note: "collectAddressBook(): loop body mutates a shared bookCursor/newState accumulator the next iteration reads", @@ -2043,84 +2148,84 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/groupme/index.ts", - line: 282, + line: 320, column: 29, category: "dependent_file_cursor", note: "reader.read(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", }, { path: "connectors/groupme/index.ts", - line: 665, + line: 703, column: 17, category: "provider_pacing_backpressure", note: "normalizeOneAttachment(): rate-limited/budget-gated blob-upload call", }, { path: "connectors/groupme/index.ts", - line: 806, + line: 844, column: 5, category: "dependent_pagination", note: "fetchPaginatedList(): next request depends on the prior page's page-number cursor; no page-count ceiling, terminates on the natural short/empty page or NonProgressError", }, { path: "connectors/groupme/index.ts", - line: 1183, + line: 1292, column: 20, category: "provider_pacing_backpressure", note: "emitInScopeGroupMessages(): toGroupMessageRecord() rate-limited/budget-gated blob-upload call", }, { path: "connectors/groupme/index.ts", - line: 1239, + line: 1348, column: 5, category: "dependent_pagination", note: "collectGroupMessagesForwardFromCursor(): next request depends on the prior page's after_id cursor; no page-count ceiling, terminates on the natural short/empty page or NonProgressError", }, { path: "connectors/groupme/index.ts", - line: 1366, + line: 1482, column: 5, category: "dependent_pagination", note: "collectGroupMessagesBackwardToNaturalEnd(): next request depends on the prior page's before_id cursor; no page-count ceiling, terminates on the natural short/empty page or NonProgressError", }, { path: "connectors/groupme/index.ts", - line: 1625, + line: 1750, column: 11, category: "ordered_protocol_emission", note: 'emitRecord("groups"): Collection Profile protocol emission requiring in-order delivery', }, { path: "connectors/groupme/index.ts", - line: 1658, + line: 1783, column: 11, category: "ordered_protocol_emission", note: 'emitRecord("direct_messages"): Collection Profile protocol emission requiring in-order delivery', }, { path: "connectors/groupme/index.ts", - line: 1727, + line: 1852, column: 5, category: "dependent_pagination", note: "collectDirectChatMessagesForChat(): next request depends on the prior page's before_id cursor; no page-count ceiling, terminates on the natural short/empty page or NonProgressError", }, { path: "connectors/groupme/index.ts", - line: 1755, + line: 1889, column: 22, category: "provider_pacing_backpressure", note: "toDirectChatMessageRecord(): rate-limited/budget-gated blob-upload call", }, { path: "connectors/groupme/index.ts", - line: 1802, + line: 1936, column: 28, category: "shared_mutable_accumulator", note: "collectDirectChatMessages(): loop body mutates a shared considered accumulator the next iteration reads", }, { path: "connectors/groupme/index.ts", - line: 1924, + line: 2274, column: 29, category: "shared_mutable_accumulator", note: "collectGroupMessages(): loop body mutates shared considered/nextCursors accumulators the next iteration reads", @@ -2190,16 +2295,37 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/steam/index.ts", - line: 426, + line: 456, column: 7, category: "ordered_protocol_emission", note: "emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "src/auto-login/venmo.test.ts", - line: 748, + line: 788, + column: 11, + category: "bounded_retry_polling", + note: "makeTwoStepVenmoPage(): fake Locator.waitFor() polls its own deadline, mirroring Playwright's real wait semantics", + }, + { + path: "src/auto-login/venmo.test.ts", + line: 1499, column: 9, category: "test_assertion_sequencing", note: "ensureVenmoSession(): test drives an ordered retry-if-retryable dispatch simulation", }, + { + path: "src/auto-login/venmo.ts", + line: 628, + column: 32, + category: "ordered_browser_interaction", + note: "Promise.all(): sequential Playwright action against the shared page/context", + }, + { + path: "src/auto-login/venmo.ts", + line: 671, + column: 9, + category: "ordered_browser_interaction", + note: "clickVenmoLoginSubmit(): submit-control candidates tried in priority order; must stop at the first visible one, since clicking a second control would submit the form twice", + }, ]; diff --git a/packages/polyfill-connectors/src/auto-login/chase.test.ts b/packages/polyfill-connectors/src/auto-login/chase.test.ts index e4f58f807..c9c6d0a2f 100644 --- a/packages/polyfill-connectors/src/auto-login/chase.test.ts +++ b/packages/polyfill-connectors/src/auto-login/chase.test.ts @@ -158,12 +158,17 @@ function makeLiveContext(page: Page): BrowserContext { return fake as BrowserContext; } +/** + * Runs `run` with the streaming env cleared. + * + * This no longer touches `CHASE_USERNAME` / `CHASE_PASSWORD`: an absent + * credential is now expressed by passing no `credentials` to + * `ensureChaseSession`, not by mutating the process environment. That is the + * point of the change — ambient state can no longer decide whether a login is + * attempted, so a test cannot accidentally depend on it either. + */ async function withoutChaseCredentials(run: () => Promise): Promise { - const priorUsername = process.env.CHASE_USERNAME; - const priorPassword = process.env.CHASE_PASSWORD; const priorStreamingEnv = new Map<(typeof STREAMING_ENV_KEYS)[number], string | undefined>(); - delete process.env.CHASE_USERNAME; - delete process.env.CHASE_PASSWORD; for (const key of STREAMING_ENV_KEYS) { priorStreamingEnv.set(key, process.env[key]); delete process.env[key]; @@ -171,16 +176,6 @@ async function withoutChaseCredentials(run: () => Promise): Promise try { await run(); } finally { - if (priorUsername === undefined) { - delete process.env.CHASE_USERNAME; - } else { - process.env.CHASE_USERNAME = priorUsername; - } - if (priorPassword === undefined) { - delete process.env.CHASE_PASSWORD; - } else { - process.env.CHASE_PASSWORD = priorPassword; - } for (const key of STREAMING_ENV_KEYS) { const value = priorStreamingEnv.get(key); if (value === undefined) { @@ -192,6 +187,443 @@ async function withoutChaseCredentials(run: () => Promise): Promise } } +/** + * A page that models the two facts OTP classification depends on: whether the + * prompt copy is visible, and how many usable OTP inputs exist. + * + * Synthetic, not a real capture — no Chase auth-page markup exists on disk + * (`connectors/chase/__fixtures__/` holds only post-login collector pages), + * and the live site is off limits because it is the owner's real bank. The + * selectors modelled here are copied from the module's own constants. + */ +interface FakeOtpPageState { + /** Whether "Confirm Your Identity" is visible — the identity-challenge copy. */ + challengeTextVisible?: boolean; + /** Whether the delivery-method option ("Get a text") is enabled. */ + deliveryOptionEnabled?: boolean; + /** Whether the delivery-method option is present/visible at all. */ + deliveryOptionPresent?: boolean; + /** Usable (visible + enabled) OTP inputs. 0 = the page cannot accept a code. */ + otpInputs: number; + /** Whether OTP_PROMPT_TEXT_WITH_SENT matches something visible. */ + promptTextVisible: boolean; + signedOut: boolean; +} + +/** + * Models the module's host-then-shadow OTP locator, including the chained + * `.locator()` and `.or()` calls it builds. Visibility and count are read from + * `state` at call time so a test can change the page mid-flow. + */ +function otpControlLocator(state: FakeOtpPageState, index: number): Locator { + const usable = (): boolean => index < state.otpInputs; + const fake: Pick< + Locator, + | "click" + | "count" + | "fill" + | "first" + | "isEnabled" + | "isVisible" + | "locator" + | "nth" + | "or" + | "press" + | "pressSequentially" + | "waitFor" + > = { + click: (): Promise => Promise.resolve(), + count: (): Promise => Promise.resolve(state.otpInputs), + fill: (): Promise => Promise.resolve(), + first: (): Locator => otpControlLocator(state, 0), + isEnabled: (): Promise => Promise.resolve(usable()), + isVisible: (): Promise => Promise.resolve(usable()), + // The shadow-root hop and the fallback union both resolve to the same + // modelled inputs, matching the real selector's intent. + locator: (): Locator => otpControlLocator(state, index), + nth: (n: number): Locator => otpControlLocator(state, n), + or: (): Locator => otpControlLocator(state, index), + press: (): Promise => Promise.resolve(), + pressSequentially: (): Promise => Promise.resolve(), + waitFor: (): Promise => + usable() ? Promise.resolve() : Promise.reject(new Error("chase otp input not visible")), + }; + return fake as Locator; +} + +/** A locator that matches nothing — used for every selector that is not OTP. */ +function absentLocator(): Locator { + const fake: Pick< + Locator, + | "check" + | "click" + | "count" + | "fill" + | "first" + | "isChecked" + | "isEnabled" + | "isVisible" + | "locator" + | "nth" + | "waitFor" + > = { + check: (): Promise => Promise.resolve(), + click: (): Promise => Promise.resolve(), + count: (): Promise => Promise.resolve(0), + fill: (): Promise => Promise.resolve(), + first: (): Locator => fake as Locator, + isChecked: (): Promise => Promise.resolve(false), + isEnabled: (): Promise => Promise.resolve(false), + isVisible: (): Promise => Promise.resolve(false), + locator: (): Locator => fake as Locator, + nth: (): Locator => fake as Locator, + waitFor: (): Promise => Promise.reject(new Error("absent")), + }; + return fake as Locator; +} + +/** + * A locator whose visibility is read at call time, so a test can flip the + * underlying state between classification and the prompt site. + */ +function textLocator(isVisible: () => boolean): Locator { + const fake: Pick = { + first: (): Locator => fake as Locator, + isVisible: (): Promise => Promise.resolve(isVisible()), + waitFor: (): Promise => (isVisible() ? Promise.resolve() : Promise.reject(new Error("not visible"))), + }; + return fake as Locator; +} + +interface FakeOtpPage { + /** + * Clicks on the delivery-method option. Each one makes the real Chase send a + * code to the owner's phone, so this — not the prompt count — is what a test + * asserting "no dispatch" must check. + */ + deliveryClicks: number; + /** + * Values typed into the sign-in form, keyed by field. Lets a test prove + * WHICH account was signed in — the fact a process-global credential env var + * structurally cannot distinguish between two connections. + */ + filledValues: { password: string[]; username: string[] }; + gotoCalls: string[]; + page: Page; + state: FakeOtpPageState; +} + +/** + * The delivery-method option ("Get a text"). Visibility and enabledness are + * read from `state` at call time; a visible-but-disabled option is the case a + * visibility-only guard would wrongly click. + */ +function deliveryOptionLocator(state: FakeOtpPageState, onClick: () => void): Locator { + const present = (): boolean => state.deliveryOptionPresent ?? false; + const fake: Pick = { + click: (): Promise => { + onClick(); + return Promise.resolve(); + }, + count: (): Promise => Promise.resolve(present() ? 1 : 0), + first: (): Locator => fake as Locator, + isEnabled: (): Promise => Promise.resolve(present() && (state.deliveryOptionEnabled ?? true)), + isVisible: (): Promise => Promise.resolve(present()), + nth: (): Locator => fake as Locator, + waitFor: (): Promise => (present() ? Promise.resolve() : Promise.reject(new Error("option absent"))), + }; + return fake as Locator; +} + +function isOtpSelector(selector: string): boolean { + return selector.includes("otp") || selector.includes("one-time-code"); +} + +/** + * Drives `ensureChaseSession` from the logon form through to the OTP step. + * `onAfterClassification` fires once the login form has been submitted, which + * is where a test can mutate the page out from under the connector. + */ +function makeOtpPage( + init: FakeOtpPageState, + { onAfterSignInClick }: { onAfterSignInClick?: (state: FakeOtpPageState) => void } = {} +): FakeOtpPage { + const state: FakeOtpPageState = { ...init }; + const gotoCalls: string[] = []; + let deliveryClicks = 0; + const signInButton: Pick = { + click: (): Promise => { + onAfterSignInClick?.(state); + return Promise.resolve(); + }, + count: (): Promise => Promise.resolve(1), + first: (): Locator => signInButton as Locator, + }; + const filledValues: { password: string[]; username: string[] } = { password: [], username: [] }; + const makeCredentialField = (bucket: string[]): Locator => { + const field: Pick = { + fill: (value: string): Promise => { + bucket.push(value); + return Promise.resolve(); + }, + first: (): Locator => field as Locator, + waitFor: (): Promise => Promise.resolve(), + }; + return field as Locator; + }; + const usernameField = makeCredentialField(filledValues.username); + const passwordField = makeCredentialField(filledValues.password); + // `mds-button#next-content` → `.locator("button")` → `.first()`, the chain + // `clickChaseNext` walks. Advancing past it is not itself a dispatch; the + // dispatch already happened when the delivery option was clicked. + const nextButtonInner: Pick = { + click: (): Promise => Promise.resolve(), + count: (): Promise => Promise.resolve(1), + first: (): Locator => nextButtonInner as Locator, + }; + const nextButton = { + locator: (): Locator => nextButtonInner as Locator, + } as unknown as Locator; + + const fake: Pick = { + getByRole: ((role: string): Locator => { + // The delivery-method option is the only role-based control this flow + // clicks, and clicking it is what dispatches a real code. + if (role === "link") { + return deliveryOptionLocator(state, (): void => { + deliveryClicks += 1; + }); + } + return absentLocator(); + }) as Page["getByRole"], + getByText: (text: Parameters[0]): Locator => { + const source = text instanceof RegExp ? text.source : String(text); + // The dashboard "Sign Out" probe: visible only once signed in. + if (/Sign Out/i.test(source)) { + return textLocator((): boolean => !state.signedOut); + } + // The identity-challenge method chooser. Off screen unless a test opts + // in, so the existing OTP-surface tests are unaffected. + if (/Confirm Your Identity/i.test(source)) { + return textLocator((): boolean => state.challengeTextVisible ?? false); + } + // Anything else here is the OTP prompt copy. + return textLocator((): boolean => state.promptTextVisible); + }, + goto: (url: string): ReturnType => { + gotoCalls.push(url); + return Promise.resolve(null); + }, + isClosed: (): boolean => false, + locator: (selector: string): Locator => { + if (isOtpSelector(selector)) { + return otpControlLocator(state, 0); + } + if (selector.includes("signin-button")) { + return signInButton as Locator; + } + if (selector.includes("password")) { + return passwordField; + } + if (selector.includes("userId") || selector.includes("username")) { + return usernameField; + } + // The "Next" control that follows the method chooser. Present only when + // the chooser is, matching Chase's real challenge page. Models the + // module's `mds-button#next-content` → `button` shadow hop. + if (selector.includes("next-content")) { + return state.deliveryOptionPresent ? nextButton : absentLocator(); + } + return absentLocator(); + }, + }; + return { + get deliveryClicks(): number { + return deliveryClicks; + }, + filledValues, + gotoCalls, + page: fake as Page, + state, + }; +} + +function makeOtpContext(page: Page): BrowserContext { + const fake: Pick = { + browser: () => null, + once: ((_event: "close", _listener: () => void): BrowserContext => + fake as BrowserContext) as BrowserContext["once"], + pages: (): Page[] => [page], + }; + return fake as BrowserContext; +} + +/** + * Chase's sign-in pair as the runtime hands it to `ensureSession`. + * + * Formerly this suite set `process.env.CHASE_USERNAME` / `_PASSWORD` around + * each test. `ensureChaseSession` now takes the connection's credentials as an + * argument (see `login-credentials.ts`), so the fixture is a plain object and + * the tests no longer mutate global state to steer a login. + */ +const CHASE_TEST_CREDENTIALS = Object.freeze({ + CHASE_PASSWORD: "synthetic-password", + CHASE_USERNAME: "synthetic-user", +}); + +async function withChaseCredentials(run: () => Promise): Promise { + await run(); +} + +function recordingInteraction( + requests: InteractionRequest[] +): (req: InteractionRequest) => Promise { + return (req: InteractionRequest): Promise => { + requests.push(req); + return Promise.resolve({ + data: { code: "123456" }, + request_id: req.request_id ?? "test_interaction", + status: "success", + type: "INTERACTION_RESPONSE", + }); + }; +} + +test("a page matching the OTP copy with no code input never asks the owner for a code", async () => { + await withChaseCredentials(async () => { + // The defect shape: "we sent" is visible, but nothing on the page can + // accept a code. Chase dispatched nothing, so PDPP must demand nothing. + const { page } = makeOtpPage({ otpInputs: 0, promptTextVisible: true, signedOut: true }); + const context = makeOtpContext(page); + const requests: InteractionRequest[] = []; + + await assert.rejects( + ensureChaseSession({ + context, + credentials: CHASE_TEST_CREDENTIALS, + page, + sendInteraction: recordingInteraction(requests), + }), + /chase_login_incomplete_after_submit/ + ); + + assert.deepEqual( + requests.filter((req): boolean => req.kind === "otp"), + [], + "no OTP prompt may be emitted for a page that cannot accept a code" + ); + }); +}); + +test("a genuine code-entry page still prompts the owner for a code", async () => { + await withChaseCredentials(async () => { + // The regression guard: a real OTP screen must behave exactly as before. + const { page, state } = makeOtpPage({ otpInputs: 1, promptTextVisible: true, signedOut: true }); + const context = makeOtpContext(page); + const requests: InteractionRequest[] = []; + + const ok = await ensureChaseSession({ + context, + credentials: CHASE_TEST_CREDENTIALS, + page, + sendInteraction: (req: InteractionRequest): Promise => { + requests.push(req); + // Entering the code signs the session in, as the real flow does. + state.signedOut = false; + return Promise.resolve({ + data: { code: "123456" }, + request_id: req.request_id ?? "test_interaction", + status: "success", + type: "INTERACTION_RESPONSE", + }); + }, + }); + + assert.equal(ok, true); + const otpRequests = requests.filter((req): boolean => req.kind === "otp"); + assert.equal(otpRequests.length, 1, "a real code-entry page must still prompt exactly once"); + assert.match(otpRequests[0]?.message ?? "", /Chase sent a 2FA code/); + }); +}); + +test("a split per-digit code layout still counts as a real code-entry page", async () => { + await withChaseCredentials(async () => { + const { page, state } = makeOtpPage({ otpInputs: 6, promptTextVisible: true, signedOut: true }); + const context = makeOtpContext(page); + const requests: InteractionRequest[] = []; + + const ok = await ensureChaseSession({ + context, + credentials: CHASE_TEST_CREDENTIALS, + page, + sendInteraction: (req: InteractionRequest): Promise => { + requests.push(req); + state.signedOut = false; + return Promise.resolve({ + data: { code: "123456" }, + request_id: req.request_id ?? "test_interaction", + status: "success", + type: "INTERACTION_RESPONSE", + }); + }, + }); + + assert.equal(ok, true); + assert.equal(requests.filter((req): boolean => req.kind === "otp").length, 1); + }); +}); + +test("an OTP input that vanishes between classification and the prompt fails loudly instead of prompting", async () => { + await withChaseCredentials(async () => { + // Classification sees a usable input; Chase re-renders it away before the + // prompt site is reached. The prompt-site re-check must catch that. + const { page, state } = makeOtpPage({ otpInputs: 1, promptTextVisible: true, signedOut: true }); + const context = makeOtpContext(page); + const requests: InteractionRequest[] = []; + + // Sequenced off the connector's own classification rather than a timer. + // `isOnChaseOtpPage` reads the prompt copy only after it has confirmed a + // usable input, so that read marks "classification decided: this is an OTP + // page". Chase re-renders the input away at that instant, so the + // prompt-site re-check must find nothing and the prompt must never fire. + let classifications = 0; + const guardedPage = new Proxy(page, { + get(target: Page, prop: string | symbol, receiver: unknown): unknown { + if (prop === "getByText") { + return (text: Parameters[0]): Locator => { + const resolved = target.getByText(text); + const source = text instanceof RegExp ? text.source : String(text); + if (/we sent/i.test(source)) { + classifications += 1; + if (classifications === 1) { + state.otpInputs = 0; + } + } + return resolved; + }; + } + return Reflect.get(target, prop, receiver) as unknown; + }, + }); + + await assert.rejects( + ensureChaseSession({ + context, + credentials: CHASE_TEST_CREDENTIALS, + page: guardedPage, + sendInteraction: recordingInteraction(requests), + }), + /chase_otp_input_missing/ + ); + + assert.deepEqual( + requests.filter((req): boolean => req.kind === "otp"), + [], + "the prompt must not fire once the code input is gone" + ); + }); +}); + test("ensureChaseSession hands off when optional credentials are absent", async () => { await withoutChaseCredentials(async () => { let live = false; @@ -217,7 +649,155 @@ test("ensureChaseSession hands off when optional credentials are absent", async assert.deepEqual(gotoCalls, [DASHBOARD_URL, DASHBOARD_URL]); assert.equal(requests.length, 1); assert.equal(requests[0]?.kind, "manual_action"); - assert.match(requests[0]?.message ?? "", /No optional Chase sign-in details/); - assert.doesNotMatch(requests[0]?.message ?? "", /password|test-user/u); + // The owner-facing reason must name the CREDENTIAL, not the page. Before + // `resolveLoginCredentials`, an absent credential produced copy that read + // like a provider/page problem; an owner could not tell "nothing was + // stored for this connection" from "Chase failed to render". + assert.match(requests[0]?.message ?? "", /no stored credential for this chase connection/); + assert.match(requests[0]?.message ?? "", /missing: CHASE_USERNAME, CHASE_PASSWORD/); + assert.doesNotMatch(requests[0]?.message ?? "", /did not render|failed to load/i); + // Names only, never values. + assert.doesNotMatch(requests[0]?.message ?? "", /synthetic-password|synthetic-user/u); + }); +}); + +test("the identity-challenge copy alone never dispatches a code when the delivery option is absent", async () => { + // "Confirm Your Identity" also appears on Chase's interstitial and error + // variants of that page. Clicking a delivery option is what makes Chase send + // a real code, so the copy alone must never authorize it. + await withChaseCredentials(async () => { + const fake = makeOtpPage({ + challengeTextVisible: true, + deliveryOptionPresent: false, + otpInputs: 0, + promptTextVisible: false, + signedOut: true, + }); + const context = makeOtpContext(fake.page); + const requests: InteractionRequest[] = []; + + await assert.rejects( + ensureChaseSession({ + context, + credentials: CHASE_TEST_CREDENTIALS, + page: fake.page, + sendInteraction: recordingInteraction(requests), + }), + /chase_delivery_method_not_available/ + ); + + assert.equal(fake.deliveryClicks, 0, "no delivery option may be clicked: that click is a real code dispatch"); + assert.deepEqual( + requests.filter((request) => request.kind === "otp"), + [], + "the owner is never asked for a code that was never sent" + ); }); }); + +test("a visible but DISABLED delivery option is not evidence the chooser is ready", async () => { + // Playwright reports a disabled control as visible, so a visibility-only + // guard would click here and spend a real code. + await withChaseCredentials(async () => { + const fake = makeOtpPage({ + challengeTextVisible: true, + deliveryOptionEnabled: false, + deliveryOptionPresent: true, + otpInputs: 0, + promptTextVisible: false, + signedOut: true, + }); + const context = makeOtpContext(fake.page); + const requests: InteractionRequest[] = []; + + await assert.rejects( + ensureChaseSession({ + context, + credentials: CHASE_TEST_CREDENTIALS, + page: fake.page, + sendInteraction: recordingInteraction(requests), + }), + /chase_delivery_method_not_available/ + ); + + assert.equal(fake.deliveryClicks, 0, "a disabled option must not be clicked"); + }); +}); + +test("a genuine method chooser still dispatches exactly once and reaches the code prompt", async () => { + await withChaseCredentials(async () => { + const fake = makeOtpPage({ + challengeTextVisible: true, + deliveryOptionPresent: true, + otpInputs: 1, + promptTextVisible: true, + signedOut: true, + }); + // Chase authenticates once the code is submitted. + const context = makeOtpContext(fake.page); + const requests: InteractionRequest[] = []; + + await ensureChaseSession({ + context, + credentials: CHASE_TEST_CREDENTIALS, + page: fake.page, + sendInteraction: (request: InteractionRequest): Promise => { + if (request.kind === "otp") { + fake.state.signedOut = false; + } + return recordingInteraction(requests)(request); + }, + }); + + assert.equal(fake.deliveryClicks, 1, "the genuine chooser dispatches exactly once, as before"); + assert.deepEqual( + requests.map((request) => request.kind), + ["otp"], + "the owner is asked for the code that was genuinely dispatched" + ); + }); +}); + +/** + * The case the process-global env-var design structurally cannot express. + * + * `process.env.CHASE_USERNAME` holds ONE account. An owner with two Chase + * connections needs each run to sign in as its OWN account. Because + * `ensureChaseSession` now takes the connection's credentials as an argument, + * two runs in the SAME process — sharing one `process.env` — type two + * different usernames into the sign-in form. + * + * The assertion is on what was typed into the form, not on the arguments + * passed in: proving the right value merely arrived would not prove it reached + * the login. + */ +test("two connections of one connector sign in as their own accounts", async () => { + const signIn = async (credentials: Readonly>) => { + const fake = makeOtpPage({ otpInputs: 0, promptTextVisible: false, signedOut: true }); + const context = makeOtpContext(fake.page); + // The page never becomes live, so this always ends in a throw; the login + // ATTEMPT — the values typed into the form — is what this test is about. + await ensureChaseSession({ + context, + credentials, + page: fake.page, + sendInteraction: recordingInteraction([]), + }).catch((): void => undefined); + return fake.filledValues; + }; + + const connectionA = await signIn({ + CHASE_PASSWORD: "synthetic-pw-a", + CHASE_USERNAME: "owner-a@example.invalid", + }); + const connectionB = await signIn({ + CHASE_PASSWORD: "synthetic-pw-b", + CHASE_USERNAME: "owner-b@example.invalid", + }); + + assert.deepEqual(connectionA.username, ["owner-a@example.invalid"]); + assert.deepEqual(connectionB.username, ["owner-b@example.invalid"]); + assert.deepEqual(connectionA.password, ["synthetic-pw-a"]); + assert.deepEqual(connectionB.password, ["synthetic-pw-b"]); + assert.notDeepEqual(connectionA.username, connectionB.username, "two connections must not collapse onto one account"); +}); diff --git a/packages/polyfill-connectors/src/auto-login/chase.ts b/packages/polyfill-connectors/src/auto-login/chase.ts index 751dc0a39..f171e2cf8 100644 --- a/packages/polyfill-connectors/src/auto-login/chase.ts +++ b/packages/polyfill-connectors/src/auto-login/chase.ts @@ -35,6 +35,8 @@ import type { BrowserContext, Locator, Page } from "playwright"; import { manualBrowserLogin } from "../browser-handoff.ts"; import type { InteractionRequest, InteractionResponse } from "../connector-runtime.ts"; +import { locatorIsUsable } from "./locator-helpers.ts"; +import { type LoginCredentialFields, resolveLoginCredentials } from "./login-credentials.ts"; const DASHBOARD_URL = "https://secure.chase.com/web/auth/dashboard"; const LOGON_URL = "https://secure.chase.com/web/auth/"; @@ -42,7 +44,20 @@ const LOGON_URL = "https://secure.chase.com/web/auth/"; const SIGN_OUT_TEXT = /Sign Out|Log Off/i; const CHALLENGE_TEXT = /Confirm Your Identity|Choose a confirmation method/i; const OTP_PROMPT_TEXT = /Enter (the|your) code|identification code|verification code/i; +/** + * Copy that ACCOMPANIES a code-entry screen. Necessary but never sufficient: + * "we sent" is ordinary Chase prose that appears on notifications, alert + * banners, and the method chooser's own "we sent a code to..." confirmation + * line, none of which can accept a code. See `hasUsableChaseOtpInput`. + */ const OTP_PROMPT_TEXT_WITH_SENT = /Enter (the|your) code|identification code|verification code|we sent/i; +/** + * A split per-digit layout has one input per digit. Chase's current OTP screen + * uses a single `mds-text-input-secure` field, but the bound keeps a redesign + * to a boxed layout classifiable instead of silently unrecognized. Anything + * larger is a page full of inputs, not a code entry. + */ +const MAX_SPLIT_CODE_DIGITS = 10; const REMEMBER_DEVICE_TEXT = /remember|trust|don't ask/i; const NEXT_BUTTON_TEXT = /^Next$/i; const OTP_INPUT_FALLBACK_SELECTOR = @@ -55,11 +70,28 @@ const METHOD_LABELS: Record = { call: "Call me", email: "Email me", }; -const MANUAL_LOGIN_WITHOUT_CREDENTIALS_MESSAGE = - "No optional Chase sign-in details were provided. Sign in to Chase in the secure browser, then respond success."; +const MANUAL_LOGIN_WITHOUT_CREDENTIALS_MESSAGE = "Sign in to Chase in the secure browser, then respond success."; + +/** + * Where Chase's sign-in pair lives in the runtime-resolved `credentials` + * object. These are the same names the connector's `auth.required` block + * declares and the same names the static-secret registry injects — one + * vocabulary across capture, injection, and use. + */ +const CHASE_LOGIN_FIELDS: LoginCredentialFields = { + password: ["CHASE_PASSWORD"], + username: ["CHASE_USERNAME"], +}; interface EnsureChaseSessionArgs { context: BrowserContext; + /** + * Credentials the runtime resolved for THIS run's connection (see + * `login-credentials.ts`). Optional so a direct, non-runtime caller can omit + * it; an absent credential is a supported state that routes to manual + * sign-in, not an error. + */ + credentials?: Readonly>; onCredentialSubmit?: () => void; page: Page; sendInteraction: (req: InteractionRequest) => Promise; @@ -147,7 +179,55 @@ function usablePage(context: BrowserContext, preferred: Page): Page | Promise 1 && codeCount <= MAX_SPLIT_CODE_DIGITS); +} + +/** + * Count the OTP inputs that are actually usable right now — visible and + * enabled. Chase's light DOM also carries a disabled hidden mirror named + * `otp-input`, so presence in the DOM is not evidence; usability is. + */ +async function countUsableChaseOtpInputs(page: Page): Promise { + const candidates = chaseOtpInputCandidates(page); + const count = await candidates.count().catch((): number => 0); + let usable = 0; + for (let i = 0; i < count; i += 1) { + const candidate = candidates.nth(i); + const [visible, enabled] = await Promise.all([ + candidate.isVisible().catch((): boolean => false), + candidate.isEnabled().catch((): boolean => false), + ]); + if (visible && enabled) { + usable += 1; + } + } + return usable; +} + +/** + * Whether this page can actually ACCEPT a code right now. + * + * This is the evidence that makes an OTP classification honest. Prompting the + * owner for a code commits them to fetching a secret out of band — and on a + * bank, a fabricated prompt also trains them to expect OTP demands that Chase + * never sent. So the bar is a real, visible, enabled code input: one field, or + * the split per-digit layout. Text is not evidence: "we sent" is prose that + * appears on pages with no code entry at all. + */ +async function hasUsableChaseOtpInput(page: Page): Promise { + return isViableChaseOtpDigitCount(await countUsableChaseOtpInputs(page)); +} + +/** + * Matching copy alone can never reach the prompt. A page that says "we sent" + * but carries no usable code input is not an OTP challenge, and treating it as + * one is how the owner ends up waiting for a code that was never dispatched. + */ async function isOnChaseOtpPage(page: Page): Promise { + if (!(await hasUsableChaseOtpInput(page))) { + return false; + } const textVisible = await page .getByText(OTP_PROMPT_TEXT_WITH_SENT) .first() @@ -182,7 +262,14 @@ async function clickChaseNext(page: Page, fallbackInput?: Locator): Promise