add a source →
@@ -329,6 +400,22 @@ function InstanceListItem({
panel, while the list shows only the owner label, retained facts, and
health. */}
+ {/*
+ * The fused status line: what it is, when it last updated, and whether
+ * it is syncing right now — one string, worst honest axis first. Before
+ * this, the row showed only a colored glyph (label sr-only), so an owner
+ * could not tell a fresh source from a stale one, or a syncing source
+ * from a stuck one, without opening the detail panel. See
+ * design-notes/fused-source-status-2026-08-22.md.
+ */}
+
) : null}
- {/* The dot is a decorative reinforcement of the status; the textual
- label is announced via the sr-only span so color is never the sole
- signal and the glyph itself carries no a11y burden. */}
+ {/* The dot is a decorative reinforcement of the status. The textual
+ status now renders visibly in `.rr-s-item__status`, so color is
+ never the sole signal and this glyph carries no a11y burden — a
+ second sr-only copy here would make screen readers announce the
+ status twice per row. */}
{instance.status.dot}
- {instance.status.label}
>
);
@@ -440,7 +528,22 @@ function InstancePassport({
{instance.nextAction ? (
) : null}
- {instance.revoked ? (
+ {instance.setupFailed ? (
+
+ {/* Server-owned sentence so a TTL-expired attempt says so plainly
+ ("expired while waiting for you to finish signing in")
+ instead of collapsing into the same generic sentence an
+ owner-abandoned or connector-failed attempt gets — see
+ `archiveRenderedVerdict` in `server/ref-control.ts`. The
+ fallback (pre-existing generic copy, own CTA appended) applies
+ only when the mirror predates `rendered_verdict` — never for a
+ live reference, whose forward_statement always names the next
+ step itself. */}
+ {instance.setupFailedForwardStatement ??
+ "This connection attempt never finished setup. No records were collected — start a new attempt from Add a source."}
+
+ ) : null}
+ {instance.revoked && !instance.setupFailed ? (
Future collection is stopped. Already-collected records stay visible and searchable; revoke does not erase
anything.
@@ -607,7 +710,11 @@ function PassportActions({
) : null}
- {interactive && reactivateAction && instance.connectionId && instance.revoked ? (
+ {/* A setup-failed row has no credential, schedule, or prior collection
+ to reactivate — it is a spent enrollment shell, not a paused live
+ connection. Offering "Reactivate" here would be the same dishonest
+ promise archived sources already refuse to make. */}
+ {interactive && reactivateAction && instance.connectionId && instance.revoked && !instance.setupFailed ? (
setConfirmingReactivate((v) => !v)}
@@ -944,9 +1051,9 @@ function StreamManifestRow({
- Unknown
+ Not measured
)}
{collection?.dispositionLabel ? (
diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts
index 0828019ec..de4ba79ab 100644
--- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts
+++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.test.ts
@@ -42,6 +42,29 @@ test("410 with companion_unavailable code classifies as companion_unavailable",
assert.equal(reason, "companion_unavailable");
});
+test("503 with streaming_companion_unavailable code classifies as companion_unavailable", () => {
+ // Regression: the reference server raises StreamingCompanionUnavailableError
+ // as HTTP 503 with body code `streaming_companion_unavailable`
+ // (reference-implementation/server/streaming/routes.ts). That fell through
+ // to `unknown`, whose copy ("Couldn't reach the browser stream after
+ // several tries.") reads as a network/reachability problem when the server
+ // in fact answered with a specific, actionable reason.
+ const { reason, troubleMessage } = classifyStreamReachFailure({
+ probeCode: "streaming_companion_unavailable",
+ probeStatus: 503,
+ });
+ assert.equal(reason, "companion_unavailable");
+ assert.equal(troubleMessage, "The browser session is no longer running on the server. Start the browser step again.");
+});
+
+test("503 with an unrelated or absent code still classifies as unknown", () => {
+ // Other 503s (e.g. the managed n.eko window-settle probe's
+ // managed_surface_window_settle_unavailable) are a distinct condition;
+ // only the companion-unavailable body code should map to companion_unavailable.
+ const { reason } = classifyStreamReachFailure({ probeCode: null, probeStatus: 503 });
+ assert.equal(reason, "unknown");
+});
+
test("a thrown probe (no HTTP response) classifies as unreachable_origin", () => {
const { reason } = classifyStreamReachFailure({ probeError: true, probeStatus: null });
assert.equal(reason, "unreachable_origin");
@@ -59,9 +82,12 @@ test("an answered-but-unrecognized status (5xx) classifies as unknown", () => {
assert.equal(reason, "unknown");
});
-test("unknown preserves the prior generic give-up message verbatim (no regression)", () => {
+test("unknown with a known status appends the status code to the generic give-up message", () => {
+ // A future unmapped status must not hide behind identical generic copy the
+ // way 503/streaming_companion_unavailable did before this fix — the status
+ // is now visible in the message even though the reason stays `unknown`.
const { troubleMessage } = classifyStreamReachFailure({ probeStatus: 500 });
- assert.equal(troubleMessage, "Couldn't reach the browser stream after several tries.");
+ assert.equal(troubleMessage, "Couldn't reach the browser stream after several tries. (server responded 500)");
});
test("every reason yields a non-empty operator message that never claims success", () => {
diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts
index 527da137b..8901ac32e 100644
--- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts
+++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-reach-diagnostics.ts
@@ -39,8 +39,12 @@ const STREAM_REACH_REASON_SET = new Set(STREAM_REACH_REASONS);
/**
* Operator-facing copy per reason. Operator-console voice: address the operator
* running their own instance, name the failure class, point at the next action.
- * Never claim the stream connected or recovered. `unknown` preserves the prior
- * generic give-up message verbatim so no occurrence regresses.
+ * Never claim the stream connected or recovered. `unknown`'s entry here is the
+ * fallback used only when no status was available at all (see
+ * {@link unknownTroubleMessage}); when a status IS known it is appended so an
+ * unmapped status is visible rather than hidden behind identical generic copy
+ * — that silence is exactly what let the 503/streaming_companion_unavailable
+ * case go unnoticed before this reason was mapped.
*/
const STREAM_REACH_MESSAGE: Record = {
companion_unavailable: "The browser session is no longer running on the server. Start the browser step again.",
@@ -52,6 +56,11 @@ const STREAM_REACH_MESSAGE: Record = {
"Couldn't reach the browser stream. Check that the reference server is reachable, then try again.",
};
+function unknownTroubleMessage(probeStatus: number | null): string {
+ const base = STREAM_REACH_MESSAGE.unknown;
+ return probeStatus === null ? base : `${base} (server responded ${probeStatus})`;
+}
+
export interface StreamReachProbeResult {
/**
* `error.code` parsed from the probe response body, when present. Used to
@@ -93,7 +102,8 @@ export function sanitizeStreamReachReason(value: unknown): StreamReachReason {
*/
export function classifyStreamReachFailure(probe: StreamReachProbeResult): StreamReachClassification {
const reason = classifyReason(probe);
- return { reason, troubleMessage: STREAM_REACH_MESSAGE[reason] };
+ const troubleMessage = reason === "unknown" ? unknownTroubleMessage(probe.probeStatus) : STREAM_REACH_MESSAGE[reason];
+ return { reason, troubleMessage };
}
function classifyReason(probe: StreamReachProbeResult): StreamReachReason {
@@ -113,9 +123,27 @@ function classifyReason(probe: StreamReachProbeResult): StreamReachReason {
// default to the more common expiry case rather than fabricating
// companion loss.
return probe.probeCode === "companion_unavailable" ? "companion_unavailable" : "session_expired";
+ case 503:
+ // The reference server raises StreamingCompanionUnavailableError as 503
+ // ("a browser-control interaction is current, but no ready browser
+ // surface is registered for this run") with body code
+ // `streaming_companion_unavailable`. That fell through to `unknown`,
+ // whose copy read as a network/proxy fault and sent the owner looking
+ // in the wrong place while the server had in fact answered with a
+ // specific, actionable reason. Verified 2026-08-18: the single
+ // STREAMING_COMPANION_UNAVAILABLE occurrence in the production log is
+ // recorded with statusCode 503. Other 503 codes (e.g. the managed n.eko
+ // window-settle probe's `managed_surface_window_settle_unavailable`)
+ // are a different, currently-unclassified condition — only match the
+ // companion-unavailable body code here rather than collapsing every
+ // 503 into this one reason.
+ return probe.probeCode === "streaming_companion_unavailable" ? "companion_unavailable" : "unknown";
default:
- // 5xx, proxy errors, or any other answered status: real but unclassified.
- // Preserve the prior generic give-up rather than guessing.
+ // 5xx, proxy errors, or any other answered status: real but
+ // unclassified. Keep the reason `unknown` (the spine vocabulary stays
+ // closed), but the message includes the status code so the next
+ // unmapped status is at least visible instead of hiding behind
+ // identical generic copy the way this one did.
return "unknown";
}
}
diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.test.ts
new file mode 100644
index 000000000..bbad26cb6
--- /dev/null
+++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.test.ts
@@ -0,0 +1,89 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import { normalizedPointerButton, readablePointerInput } from "./stream-viewer-pointer-input.ts";
+
+test("a primary-contact touch tap is forwarded, matching real touch pointerdown/pointerup semantics", () => {
+ // Per the Pointer Events spec, `button` is 0 for the primary contact on a
+ // touch pointerdown/pointerup. A regression that gates touch on `button`
+ // drops every real-world tap while leaving synthetic-event tests (which
+ // often omit `button` or set it to a mouse-like value) green.
+ assert.deepEqual(readablePointerInput({ buttons: 1, pointerType: "touch", type: "pointerdown" }), {
+ pointerType: "touch",
+ type: "pointerdown",
+ });
+ assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointerup" }), {
+ pointerType: "touch",
+ type: "pointerup",
+ });
+});
+
+test("touch pointermove is always forwarded regardless of the mouse-only hover-move gate", () => {
+ assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointermove" }), {
+ pointerType: "touch",
+ type: "pointermove",
+ });
+});
+
+test("a mouse hover move with no button held is dropped to avoid flooding the wire", () => {
+ assert.equal(readablePointerInput({ buttons: 0, pointerType: "mouse", type: "pointermove" }), null);
+});
+
+test("a mouse drag move with a button held is forwarded", () => {
+ assert.deepEqual(readablePointerInput({ buttons: 1, pointerType: "mouse", type: "pointermove" }), {
+ pointerType: "mouse",
+ type: "pointermove",
+ });
+});
+
+test("pointercancel is always forwarded for every pointer type", () => {
+ assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointercancel" }), {
+ pointerType: "touch",
+ type: "pointercancel",
+ });
+ assert.deepEqual(readablePointerInput({ buttons: 0, pointerType: "mouse", type: "pointercancel" }), {
+ pointerType: "mouse",
+ type: "pointercancel",
+ });
+});
+
+test("an unrecognized DOM event type is dropped", () => {
+ assert.equal(readablePointerInput({ buttons: 0, pointerType: "touch", type: "pointerenter" }), null);
+});
+
+// `normalizedPointerButton` exists because the forwarded `button` is not a
+// filter downstream — it is arithmetic. neko computes the X11 button as
+// `(button ?? 0) + 1`, and X11 button 1 is primary, so a touch pointerdown
+// reporting the non-spec `button === -1` becomes X11 button 0 (no button at
+// all) and the remote page never sees a press. This is the residual half of
+// the owner's "can't tap the captcha on mobile" report: an earlier fix stopped
+// DROPPING such events but still forwarded the raw value into that arithmetic.
+
+test("a touch pointerdown reporting button -1 is normalized to primary contact", () => {
+ // The exact value the sibling gate's own doc comment documents as real on
+ // touch input paths. Left raw, it silently disarms the tap.
+ assert.equal(normalizedPointerButton(-1, "touch"), 0);
+});
+
+test("a pen pointerdown reporting button -1 is normalized to primary contact", () => {
+ assert.equal(normalizedPointerButton(-1, "pen"), 0);
+});
+
+test("an ordinary primary-contact touch button is left alone", () => {
+ assert.equal(normalizedPointerButton(0, "touch"), 0);
+});
+
+test("mouse buttons are passed through so middle/right/back/forward survive", () => {
+ // Touch and pen have no secondary button, but a mouse does — normalizing it
+ // would turn every right-click into a left-click.
+ assert.equal(normalizedPointerButton(0, "mouse"), 0);
+ assert.equal(normalizedPointerButton(1, "mouse"), 1);
+ assert.equal(normalizedPointerButton(2, "mouse"), 2);
+ assert.equal(normalizedPointerButton(3, "mouse"), 3);
+ assert.equal(normalizedPointerButton(4, "mouse"), 4);
+ // A mouse never legitimately reports -1 on a press, but if it does the value
+ // is preserved rather than invented — mouse `button` is meaningful.
+ assert.equal(normalizedPointerButton(-1, "mouse"), -1);
+});
diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts
new file mode 100644
index 000000000..2f38543af
--- /dev/null
+++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts
@@ -0,0 +1,102 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Extracted out of stream-viewer.tsx (a "use client" React component file)
+ * into a plain module so this gate can be unit-tested under plain
+ * `node --test` without a DOM/PointerEvent runtime. This function decides
+ * whether a captured DOM pointer event becomes a forwarded remote-surface
+ * intent at all; a wrong decision here silently drops input with no visible
+ * symptom other than "taps don't do anything."
+ */
+
+export type RemotePointerActionType = "pointercancel" | "pointerdown" | "pointermove" | "pointerup";
+export type RemotePointerType = "mouse" | "pen" | "touch";
+
+export interface ReadablePointerEventLike {
+ buttons: number;
+ pointerType: string;
+ type: string;
+}
+
+export interface ReadablePointerInput {
+ pointerType: RemotePointerType;
+ type: RemotePointerActionType;
+}
+
+function remoteTypeFor(type: string): RemotePointerActionType | null {
+ switch (type) {
+ case "pointerdown":
+ return "pointerdown";
+ case "pointermove":
+ return "pointermove";
+ case "pointerup":
+ return "pointerup";
+ case "pointercancel":
+ return "pointercancel";
+ default:
+ return null;
+ }
+}
+
+/**
+ * Gates a raw DOM pointer event down to a forwardable remote-surface intent,
+ * or `null` if it should be dropped.
+ *
+ * Per the W3C Pointer Events spec, `button`/`buttons` describe *mouse-style*
+ * button state and are largely meaningless for touch: `button` is 0 (primary)
+ * for a touch pointerdown/pointerup, but some touch input paths (stylus
+ * palm-rejection proxies, certain WebViews) report `button === -1` on a
+ * touch pointerup even though the contact itself is legitimate. Gating touch
+ * on `event.button` at all is the wrong signal — touch has no secondary
+ * button, so every primary-contact touch event must pass through regardless
+ * of `button`. Only `buttons` (a bitmask) is meaningful across pointer types,
+ * and it is only used here to suppress hover-only mouse moves.
+ */
+export function readablePointerInput(event: ReadablePointerEventLike): ReadablePointerInput | null {
+ const type = remoteTypeFor(event.type);
+ if (!type) {
+ return null;
+ }
+ const pointerType: RemotePointerType =
+ event.pointerType === "touch" || event.pointerType === "pen" ? event.pointerType : "mouse";
+ // Hover-move gate: suppress mouse moves with no button held to prevent
+ // hover floods. Touch/pen have no hover-move concept at this layer.
+ if (type === "pointermove" && pointerType === "mouse" && event.buttons === 0) {
+ return null;
+ }
+ return { pointerType, type };
+}
+
+/**
+ * Normalizes `PointerEvent.button` into a primary-contact button index that is
+ * safe to forward, for touch and pen only.
+ *
+ * The gate above stopped *dropping* touch events with a non-zero `button`, but
+ * the raw value is still forwarded in the wire payload — and downstream it is
+ * arithmetic, not a filter. `NekoPointerController.handle` (remote-surface
+ * 1.5.2, `controllers/neko-pointer-controller.js`) computes the X11 button as
+ * `(event.button ?? 0) + 1`, where X11 button 1 is primary. A touch
+ * `pointerdown` reporting `button === -1` — the same non-spec value this
+ * module's own comment above documents as real on touch input paths —
+ * therefore becomes X11 button **0**, which is not a button at all, so neko
+ * presses nothing and the tap never clicks.
+ *
+ * Verified against the installed controller by replaying the exact client
+ * payload shape: `button: 0` on down yields `buttonDown(1)`, while
+ * `button: -1` on down yields `buttonDown(0)`.
+ *
+ * `pointerup` was already safe by luck — the controller prefers the remembered
+ * press button over the event's own — but `pointerdown` has nothing to fall
+ * back to, so it must be normalized here at the payload boundary.
+ *
+ * Touch and pen have no secondary button, so pinning them to primary loses no
+ * information. Mouse is passed through untouched: its `button` is meaningful
+ * (middle/right/back/forward) and must survive.
+ */
+export function normalizedPointerButton(button: number, pointerType: RemotePointerType): number {
+ if (pointerType === "mouse") {
+ return button;
+ }
+ return button < 0 ? 0 : button;
+}
diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts
index 65f1c6126..19aa1a8d0 100644
--- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts
+++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-session-mechanisms.test.ts
@@ -51,7 +51,7 @@ function readViewerSource(): Promise {
test("trusted-touch keyboard focus reaches the viewer session synchronously", async () => {
const src = await readViewerSource();
const tapStart = src.indexOf("const handleMobileKeyboardPointer =");
- const tapEnd = src.indexOf("const remoteTypeFor =", tapStart);
+ const tapEnd = src.indexOf("const dispatchPointerIntent =", tapStart);
const trustedTap = src.slice(tapStart, tapEnd);
assert.notEqual(tapStart, -1, "the production trusted-touch handler is present");
diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-touch-tap-oracle.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-touch-tap-oracle.test.ts
new file mode 100644
index 000000000..09b75e018
--- /dev/null
+++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-touch-tap-oracle.test.ts
@@ -0,0 +1,133 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * End-to-end oracle for the owner's "can't tap the captcha on mobile" report.
+ *
+ * The unit tests beside this file assert `normalizedPointerButton` in
+ * isolation, which proves only that it does what it says. This file replays
+ * the client's real wire payload through the REAL installed
+ * `NekoPointerController` from `@opendatalabs/remote-surface` and asserts on
+ * the X11 button the remote actually receives. That is the property the owner
+ * cares about — "the tap presses a button on the remote page" — and it stays
+ * honest if the dependency changes its mapping, because the oracle is the
+ * dependency itself rather than a restatement of our own arithmetic.
+ *
+ * A captcha checkbox is not special here: the remote input path is
+ * coordinate-based (neko X11 / CDP `Input.dispatchMouseEvent`), so it crosses
+ * cross-origin iframe boundaries like reCAPTCHA's by construction. Nothing in
+ * the path hit-tests the top document. What actually broke the tap was the
+ * button index, which fails identically inside or outside an iframe — it just
+ * gets noticed on a captcha because that is where a tap is unavoidable.
+ */
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import { normalizedPointerButton } from "./stream-viewer-pointer-input.ts";
+
+// X11 button codes, as used by neko's `control.buttonDown`/`buttonUp`.
+const X11_PRIMARY = 1;
+const X11_NO_BUTTON = 0;
+
+interface ControlCall {
+ button: number;
+ kind: "buttonDown" | "buttonUp";
+}
+
+type NekoPointerControllerCtor = new (deps: {
+ control: Record;
+ mapToRemote: (x: number, y: number) => { x: number; y: number };
+}) => { handle: (event: Record) => void };
+
+/** Loads the real controller, or `null` when the dependency isn't installed. */
+async function loadNekoPointerController(): Promise {
+ try {
+ const mod = await import("@opendatalabs/remote-surface");
+ return (mod.NekoPointerController as unknown as NekoPointerControllerCtor | undefined) ?? null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Replays a touch tap the way the client sends it, returning the X11 button
+ * calls neko would make. `rawButton` is what the browser put on the
+ * PointerEvent; `normalize` mirrors whether the client sanitizes it.
+ */
+function tapThroughController(
+ Controller: NonNullable>>,
+ rawButton: number,
+ normalize: boolean
+): ControlCall[] {
+ const calls: ControlCall[] = [];
+ const controller = new Controller({
+ control: {
+ buttonDown: (pressed: number) => calls.push({ button: pressed, kind: "buttonDown" }),
+ buttonUp: (released: number) => calls.push({ button: released, kind: "buttonUp" }),
+ move: () => undefined,
+ scroll: () => undefined,
+ },
+ mapToRemote: (x: number, y: number) => ({ x, y }),
+ });
+ const button = normalize ? normalizedPointerButton(rawButton, "touch") : rawButton;
+ // The payload shape built in stream-viewer.tsx's `dispatchPointerIntent`.
+ controller.handle({ button, pointerId: 1, pointerType: "touch", type: "pointerdown", x: 100, y: 200 });
+ controller.handle({ button, pointerId: 1, pointerType: "touch", type: "pointerup", x: 100, y: 200 });
+ return calls;
+}
+
+test("a touch tap reporting button -1 presses a real button on the remote page", async (t) => {
+ const Controller = await loadNekoPointerController();
+ if (!Controller) {
+ t.skip("@opendatalabs/remote-surface is not installed in this workspace");
+ return;
+ }
+
+ const calls = tapThroughController(Controller, -1, true);
+
+ assert.deepEqual(
+ calls,
+ [
+ { button: X11_PRIMARY, kind: "buttonDown" },
+ { button: X11_PRIMARY, kind: "buttonUp" },
+ ],
+ "a touch tap must press and release X11 primary, or the remote page sees no click at all"
+ );
+});
+
+test("the unnormalized payload is what silently disarmed the tap", async (t) => {
+ const Controller = await loadNekoPointerController();
+ if (!Controller) {
+ t.skip("@opendatalabs/remote-surface is not installed in this workspace");
+ return;
+ }
+
+ // Characterizes the defect rather than asserting the fix: forwarding the raw
+ // -1 makes neko press X11 button 0, which is not a button. If a future
+ // dependency bump makes the raw value safe on its own, this test fails and
+ // says so, instead of leaving the normalization as unexplained cargo.
+ const calls = tapThroughController(Controller, -1, false);
+
+ assert.equal(
+ calls[0]?.button,
+ X11_NO_BUTTON,
+ "raw button -1 maps to X11 button 0 (no button) — this is why the captcha tap did nothing"
+ );
+});
+
+test("an ordinary touch tap reporting button 0 was never broken", async (t) => {
+ const Controller = await loadNekoPointerController();
+ if (!Controller) {
+ t.skip("@opendatalabs/remote-surface is not installed in this workspace");
+ return;
+ }
+
+ // Bounds the blast radius of the report: taps on engines that report a spec
+ // -compliant 0 always worked, which is why this reproduced only on some
+ // devices and why "taps are broken" was never reproducible on desktop.
+ const normalized = tapThroughController(Controller, 0, true);
+ const raw = tapThroughController(Controller, 0, false);
+
+ assert.deepEqual(normalized, raw, "normalization must be a no-op for a spec-compliant primary contact");
+ assert.equal(normalized[0]?.button, X11_PRIMARY);
+});
diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx
index 72de60fbc..adb47cba4 100644
--- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx
+++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx
@@ -141,6 +141,7 @@ import {
readViewerViewport,
viewportLayoutFromInfo,
} from "./stream-viewer-geometry.ts";
+import { normalizedPointerButton, readablePointerInput } from "./stream-viewer-pointer-input.ts";
import { parseAttachedMessage } from "./stream-viewer-protocol.ts";
import {
createPdppRemoteSurfaceTransport,
@@ -4180,42 +4181,6 @@ function NekoSurface({
userActivationActive: navigator.userActivation?.isActive ?? null,
});
};
- const remoteTypeFor = (type: string): "pointerdown" | "pointermove" | "pointerup" | "pointercancel" | null => {
- switch (type) {
- case "pointerdown":
- return "pointerdown";
- case "pointermove":
- return "pointermove";
- case "pointerup":
- return "pointerup";
- case "pointercancel":
- return "pointercancel";
- default:
- return null;
- }
- };
- const readablePointerInput = (
- event: PointerEvent
- ): {
- pointerType: "mouse" | "touch" | "pen";
- type: "pointercancel" | "pointerdown" | "pointermove" | "pointerup";
- } | null => {
- const type = remoteTypeFor(event.type);
- if (!type) {
- return null;
- }
- const pointerType: "mouse" | "touch" | "pen" =
- event.pointerType === "touch" || event.pointerType === "pen" ? event.pointerType : "mouse";
- if ((type === "pointerdown" || type === "pointerup") && pointerType === "touch" && event.button !== 0) {
- return null;
- }
- // Hover-move gate (step 2 open question #2): suppress mouse moves
- // with no button held to prevent hover floods.
- if (type === "pointermove" && event.pointerType === "mouse" && event.buttons === 0) {
- return null;
- }
- return { pointerType, type };
- };
const dispatchPointerIntent = (
event: PointerEvent,
type: "pointercancel" | "pointerdown" | "pointermove" | "pointerup",
@@ -4227,7 +4192,11 @@ function NekoSurface({
}
const pointerIntent: Extract = {
action: type,
- button: event.button,
+ // Touch/pen `button` is normalized to primary before it leaves the
+ // client: neko turns this into an X11 button by adding 1, so a raw
+ // `-1` from a touch pointerdown would press X11 button 0 (no button)
+ // and the tap would never click. See `normalizedPointerButton`.
+ button: normalizedPointerButton(event.button, pointerType),
buttons: event.buttons,
clickCount: event.detail,
pointerId: event.pointerId,
diff --git a/apps/console/src/app/(console)/syncs/error.tsx b/apps/console/src/app/(console)/syncs/error.tsx
index 5bf1d76cb..cd8fc17e1 100644
--- a/apps/console/src/app/(console)/syncs/error.tsx
+++ b/apps/console/src/app/(console)/syncs/error.tsx
@@ -3,38 +3,63 @@
// Copyright The PDP-Connect Contributors
// SPDX-License-Identifier: Apache-2.0
-import { buttonVariants } from "@pdpp/brand-react";
import { useEffect } from "react";
+import { createRetryCounter, nextRetryDelayMs } from "../components/read-resilient-retry.ts";
+import { ListLoadingSkeleton } from "../components/route-loading.tsx";
/**
- * Runs-segment error boundary (App Router convention).
+ * Syncs-segment error boundary (App Router convention) — SLVP bar: Stripe,
+ * Linear, Vercel, and Plaid never tell an owner "we hit a transient read
+ * interruption, retrying." The page renders, or it quietly shows last-known
+ * state. The owner never learns the backend hiccuped.
*
- * Scopes a runs-area failure to the runs area instead of the dashboard-wide
- * `Something went wrong`. A run that failed unexpectedly should not also crash
- * the surrounding page to a contextless boundary. Self-contained on purpose
- * (mirrors `dashboard/error.tsx`): no server-only imports.
+ * Root cause of the throw this boundary catches (`Error: The destination
+ * stream closed early`): the read itself is fine — React's Flight/RSC
+ * streaming writer reacting to the HTTP response closing before the stream
+ * finished flushing (a poll tick from `run-detail-poller.tsx`/`LivePoller`
+ * firing `router.refresh()` while a prior refresh's stream is still in
+ * flight, or the tab backgrounding mid-render). It is a client-transport race
+ * below the data layer, not a backend outage — see `sources/error.tsx` for
+ * the full original writeup of this pattern.
+ *
+ * `/syncs` has no client-cached last-known-read marker (unlike
+ * `sources/last-known-read.ts`), so this boundary shows the plain skeleton
+ * with no staleness caption rather than fabricate a timestamp.
+ *
+ * Self-contained on purpose: a `"use client"` boundary must not import
+ * server-only modules, since the dashboard shell transitively pulls in
+ * `lib/owner-token.ts` (`server-only`).
+ */
+
+/**
+ * Consecutive-failure counter, held at MODULE scope rather than component
+ * state — see `read-resilient-retry.ts` for why a `useState` counter would
+ * silently reset every catch and never actually back off.
*/
+const retryCounter = createRetryCounter();
+
export default function RunsError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) {
useEffect(() => {
+ // Logged for operator diagnostics only — never surfaced to the owner.
console.error(error);
}, [error]);
+ useEffect(() => {
+ // Unbounded, capped backoff: every mount (i.e. every failed attempt)
+ // schedules the next retry at a delay that grows with the module-scoped
+ // counter. There is deliberately no ceiling on the counter itself — a
+ // persistent failure degrades to a slow quiet heartbeat, never a dead end.
+ const delay = nextRetryDelayMs(retryCounter.attempts);
+ const id = setTimeout(() => {
+ retryCounter.attempts += 1;
+ reset();
+ }, delay);
+ return () => clearTimeout(id);
+ }, [reset]);
+
return (
-
- Read error
- Couldn't load syncs
-
- The Syncs view ran into an error while reading from your reference deployment. Your syncs are unaffected — this
- is a read failure, not a change. Try again, or check your reference deployment status.
-
-
-
+
+
+
);
}
diff --git a/apps/console/src/app/(console)/syncs/page.invariants.test.ts b/apps/console/src/app/(console)/syncs/page.invariants.test.ts
index 74a76d4d9..142715b11 100644
--- a/apps/console/src/app/(console)/syncs/page.invariants.test.ts
+++ b/apps/console/src/app/(console)/syncs/page.invariants.test.ts
@@ -222,7 +222,11 @@ test("recent syncs pager links to the next cursor the server returned, never a f
assert.match(src, RECENT_PAGER_NEXT_HREF_RE, "the older-syncs link must carry the real next_cursor from the server");
});
-const COVERAGE_HUMANIZER_IMPORT_RE = /formatCoverageAxis[\s\S]{0,40}from "\.\.\/lib\/connection-evidence\.ts"/;
+// The shared humanizer moved from the console-local `connection-evidence.ts`
+// into `@pdpp/display` so the headless `sources-report` CLI renders the SAME
+// coverage words this view does. The invariant is unchanged — coverage copy
+// must go through the shared humanizer, never interpolate the raw axis key.
+const COVERAGE_HUMANIZER_IMPORT_RE = /formatCoverageAxis[\s\S]{0,40}from "@pdpp\/display"/;
const COVERAGE_HUMANIZER_CALL_RE = /formatCoverageAxis\(condition\)\.value/;
const RAW_COVERAGE_INTERPOLATION_RE = /`\s*·\s*\$\{condition\}`/;
diff --git a/apps/console/src/app/(console)/syncs/read-resilience.invariants.test.ts b/apps/console/src/app/(console)/syncs/read-resilience.invariants.test.ts
new file mode 100644
index 000000000..320c6e988
--- /dev/null
+++ b/apps/console/src/app/(console)/syncs/read-resilience.invariants.test.ts
@@ -0,0 +1,104 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Read-resilience acceptance invariants for the syncs segment, mirroring
+ * `sources/read-resilience.invariants.test.ts`. The prior version of this
+ * boundary showed the owner "Couldn't load syncs" behind a manual "Try
+ * again" button on the FIRST catch, with no auto-retry at all — strictly
+ * worse than the original `sources/error.tsx` this pattern replaces.
+ *
+ * THE STANDARD (stated explicitly by the owner): Stripe, Linear, Vercel, and
+ * Plaid never show a user "we hit a transient read interruption, retrying."
+ * The page renders, or it shows last-known state. The user never learns the
+ * backend hiccuped.
+ *
+ * `/syncs` has no client-cached last-known-read marker (unlike
+ * `sources/last-known-read.ts`), so this boundary shows the plain skeleton
+ * with no staleness caption — see the file's own doc comment for why that is
+ * the correct choice rather than fabricating a timestamp.
+ */
+
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+const HERE = fileURLToPath(new URL(".", import.meta.url));
+const ERROR_FILE = `${HERE}error.tsx`;
+
+const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g;
+
+/**
+ * Strip `/* ... */` block comments before checking for retired owner-facing
+ * copy. The boundary's doc comment legitimately QUOTES the retired phrases
+ * (to explain what this pattern replaces and why) — that is documentation,
+ * not rendered JSX text, so it must not trip the ban.
+ */
+function withoutBlockComments(src: string): string {
+ return src.replace(BLOCK_COMMENT_RE, "");
+}
+
+// Regexes hoisted to module scope (project lint: useTopLevelRegex). The
+// owner-facing-string bans intentionally allow "error" as a JS identifier
+// (the boundary prop is literally named `error`) but forbid it in rendered
+// JSX text content, so we assert on specific retired phrases.
+const RETIRED_COULDNT_RE = /Couldn't/;
+const RETIRED_ERROR_HEADING_RE = /Read error/;
+const RETIRED_TRY_AGAIN_RE = /Try again/;
+const RETIRED_INTERRUPTION_COPY_RE = /transient read interruption/i;
+const RETIRED_READ_FAILURE_FRAMING_RE = /read failure/i;
+const RETIRED_BACK_LINK_RE = /Back to Syncs/;
+const RETIRED_SEGMENT_ERROR_IMPORT_RE = /from\s+["']\.\.\/components\/segment-error\.tsx["']/;
+
+const USES_LOADING_SKELETON_RE = / {
+ const rawSrc = await readFile(ERROR_FILE, "utf8");
+ const src = withoutBlockComments(rawSrc);
+ assert.doesNotMatch(src, RETIRED_COULDNT_RE);
+ assert.doesNotMatch(src, RETIRED_ERROR_HEADING_RE);
+ assert.doesNotMatch(src, RETIRED_TRY_AGAIN_RE);
+ assert.doesNotMatch(src, RETIRED_INTERRUPTION_COPY_RE);
+ assert.doesNotMatch(src, RETIRED_READ_FAILURE_FRAMING_RE);
+ assert.doesNotMatch(src, RETIRED_BACK_LINK_RE);
+ assert.doesNotMatch(rawSrc, RETIRED_SEGMENT_ERROR_IMPORT_RE);
+});
+
+test("the boundary renders the same loading skeleton the route's loading.tsx uses, not a bespoke banner", async () => {
+ const src = await readFile(ERROR_FILE, "utf8");
+ assert.match(src, IMPORTS_LOADING_SKELETON_RE);
+ assert.match(src, USES_LOADING_SKELETON_RE);
+ assert.match(src, RECOVERING_TESTID_RE);
+ // loading.tsx uses ListLoadingSkeleton label="Syncs" rows={8}; the boundary
+ // must match so a caught teardown is visually identical to a normal load.
+ assert.match(src, /ListLoadingSkeleton label="Syncs" rows=\{8\}/);
+});
+
+test("the boundary retries unbounded on a capped backoff held at module scope, with no manual-retry terminal state", async () => {
+ const src = await readFile(ERROR_FILE, "utf8");
+ assert.match(src, CALLS_RESET_RE);
+ assert.match(src, UNBOUNDED_RETRY_SCHEDULES_NEXT_RE);
+ assert.match(src, IMPORTS_SHARED_RETRY_RE);
+ assert.match(src, MODULE_SCOPE_COUNTER_RE);
+ assert.doesNotMatch(
+ src,
+ NO_REACT_STATE_COUNTER_RE,
+ "the retry counter must live at module scope, not React state, or backoff never grows across remounts"
+ );
+ assert.doesNotMatch(src, NO_TERMINAL_GIVE_UP_FLAG_RE, "no gated give-up state — retry must be unbounded");
+});
+
+test("the boundary is self-contained: no server-only import", async () => {
+ const src = await readFile(ERROR_FILE, "utf8");
+ assert.doesNotMatch(src, SERVER_ONLY_IMPORT_RE);
+});
diff --git a/apps/console/src/app/(console)/syncs/syncs-demo.ts b/apps/console/src/app/(console)/syncs/syncs-demo.ts
index 3c0850840..4bde1fdc0 100644
--- a/apps/console/src/app/(console)/syncs/syncs-demo.ts
+++ b/apps/console/src/app/(console)/syncs/syncs-demo.ts
@@ -105,7 +105,7 @@ const COOLING_VERDICT = {
channel: "advisory",
detail: {},
forward_statement: "The source is throttling this connection; it will retry automatically.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
progress: {
gaps_drained_last_run: null,
headline: "Waiting for the next attempt.",
diff --git a/apps/console/src/app/(console)/syncs/syncs-model.test.ts b/apps/console/src/app/(console)/syncs/syncs-model.test.ts
index 396c22b2f..1b632b0fe 100644
--- a/apps/console/src/app/(console)/syncs/syncs-model.test.ts
+++ b/apps/console/src/app/(console)/syncs/syncs-model.test.ts
@@ -203,7 +203,7 @@ test("source-pressure cooldown produces a WAIT card, never a reconnect prompt",
channel: "advisory",
forward_statement:
"The source is throttling this connection, so the scheduler is spacing out automatic attempts.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [
action({
audience: "none",
@@ -258,7 +258,7 @@ test("a blocked connection with a source-pressure backlog still gets the WAIT ca
channel: "advisory",
forward_statement:
"The source is throttling this connection, so the scheduler is spacing out automatic attempts.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [
action({
audience: "none",
@@ -749,7 +749,7 @@ test("a broken connector does not rewrite a successful last run into sync failed
required_actions: [
action({
audience: "maintainer",
- cta: "Connector code needs a fix",
+ cta: "Some data from this source can't be collected",
kind: "code_fix",
satisfied_when: { kind: "none" },
terminal: true,
@@ -794,7 +794,7 @@ test("failure cards bind terminal gaps to rendered verdict copy, never retryable
required_actions: [
action({
audience: "maintainer",
- cta: "Connector code needs a fix",
+ cta: "Some data from this source can't be collected",
kind: "code_fix",
satisfied_when: { kind: "none" },
terminal: true,
@@ -811,7 +811,7 @@ test("failure cards bind terminal gaps to rendered verdict copy, never retryable
// biome-ignore lint/suspicious/noUnnecessaryConditions: array/Map-lookup access under noUncheckedIndexedAccess is genuinely T | undefined; tsc rejects removing this guard (Biome does not honor that tsconfig flag here).
assert.equal(card?.summary.prose, "This connector needs a code fix before it can collect again.");
assert.equal(card.summary.cta, "wait");
- assert.equal(card.summary.actionLabel, "Connector code needs a fix");
+ assert.equal(card.summary.actionLabel, "Some data from this source can't be collected");
assert.equal(card.summary.ownerActionRequired, false);
// biome-ignore lint/suspicious/noUnnecessaryConditions: array/Map-lookup access under noUncheckedIndexedAccess is genuinely T | undefined; tsc rejects removing this guard (Biome does not honor that tsconfig flag here).
assert.doesNotMatch(card?.summary.prose, RESUME_FALSE_REASSURANCE_RE);
@@ -834,7 +834,7 @@ test("failure cards bind retryable gaps to the rendered Retry now action", () =>
rendered_verdict: renderedVerdict({
channel: "advisory",
forward_statement: "Retry now to give the recoverable gap another run.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [action()],
}),
}),
@@ -908,7 +908,7 @@ test("failure cards bind dead-letter backlog to collector action, not resume-nor
rendered_verdict: renderedVerdict({
channel: "attention",
forward_statement: "Check the collector before this source can make progress.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [
action({
cta: "Check the collector",
@@ -949,7 +949,7 @@ test("device-local recovery counts as need-your-hand while navigating to recover
rendered_verdict: renderedVerdict({
channel: "attention",
forward_statement: "The local collector has saved records on its host that did not upload to this server.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [
action({
cta: "Run local recovery",
@@ -1028,7 +1028,7 @@ test("failure cards carry shared source-work groups for Runs presentation", () =
rendered_verdict: renderedVerdict({
channel: "advisory",
forward_statement: "Latest collection completed with known coverage gaps.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [
action({
audience: "maintainer",
@@ -1062,12 +1062,12 @@ test("syncs ranking only treats attention plus primary owner action as need-your
source_work: "system_issue",
rendered_verdict: renderedVerdict({
channel: "attention",
- forward_statement: "Connector code needs a fix before this can collect again.",
+ forward_statement: "Some data from this source can't be collected.",
pill: { label: "Can't collect", tone: "red" },
required_actions: [
action({
audience: "maintainer",
- cta: "Connector code needs a fix",
+ cta: "Some data from this source can't be collected",
kind: "code_fix",
satisfied_when: { kind: "none" },
terminal: true,
@@ -1139,7 +1139,7 @@ test("syncs overview collapses repeated unnamed fallback sources", () => {
const amazonAdvisory = renderedVerdict({
channel: "advisory",
forward_statement: "Retry now to give the recoverable gap another run.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [action()],
});
const model = buildSyncsViewModel({
@@ -1279,7 +1279,7 @@ test("syncs overview shows ALL review cards (no cap) and the band counts the ful
const advisoryVerdict = renderedVerdict({
channel: "advisory",
forward_statement: "Run a refresh to bring this up to date.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [action({ cta: "Refresh now", kind: "retry_gap" })],
});
const connectors = Array.from({ length: 8 }, (_, index) =>
@@ -1326,7 +1326,7 @@ test("syncs cross-surface: rendered review-card count equals the failure cards b
const advisoryVerdict = renderedVerdict({
channel: "advisory",
forward_statement: "Run a refresh to bring this up to date.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
required_actions: [action({ cta: "Refresh now", kind: "retry_gap" })],
});
const model = buildSyncsViewModel({
@@ -1361,7 +1361,7 @@ test("syncs cross-surface: an inactive queued recovery card is passive progress,
const deferredRecoveryVerdict = renderedVerdict({
channel: "calm",
forward_statement: "The next run is expected to fill the remaining data.",
- pill: { label: "Degraded", tone: "amber" },
+ pill: { label: "Missing data", tone: "amber" },
progress: {
gaps_drained_last_run: null,
headline: "Collecting in the background.",
diff --git a/apps/console/src/app/(console)/syncs/syncs-view.tsx b/apps/console/src/app/(console)/syncs/syncs-view.tsx
index 1cbe05428..1805372f5 100644
--- a/apps/console/src/app/(console)/syncs/syncs-view.tsx
+++ b/apps/console/src/app/(console)/syncs/syncs-view.tsx
@@ -28,10 +28,10 @@ import {
TableHeader,
TableHeaderRow,
} from "@pdpp/brand-react";
-import { humanizeFieldLabel } from "@pdpp/display";
+import { formatCoverageAxis, humanizeFieldLabel } from "@pdpp/display";
import { dashboardRoutes } from "@pdpp/operator-ui/components/views/routes";
import Link from "next/link";
-import { formatCoverageAxis } from "../lib/connection-evidence.ts";
+
import { SOURCE_WORK_GROUP_COPY } from "../lib/source-actionability.ts";
import {
type DuplicateSyncGroup,
@@ -533,8 +533,10 @@ function formatCollectedThisRun(row: SyncRow): string | null {
}
/**
- * The coverage condition is shown only when it adds information: "complete" is
- * the expected baseline and "unknown" is noise, so both are suppressed.
+ * The coverage condition is shown only when it adds information: `complete` is
+ * the expected baseline and `unknown` (rendered "not measured") is noise, so
+ * both are suppressed. The comparison is against the raw AXIS KEY, not the
+ * owner-facing label, so renaming the label leaves this correct.
* Otherwise this renders the SAME owner-facing wording as the source detail
* page and connection diagnostics (`formatCoverageAxis`'s humanized `value`,
* e.g. "won't backfill" / "retryable gap") — never the raw internal axis key
diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml
index a578c8ee7..89d7694ef 100644
--- a/deploy/docker/docker-compose.yml
+++ b/deploy/docker/docker-compose.yml
@@ -89,6 +89,30 @@ services:
POSTGRES_USER: pdpp
POSTGRES_PASSWORD: ${PDPP_POSTGRES_PASSWORD:-pdpp}
POSTGRES_DB: pdpp
+ # Postgres ships defaults sized for a small dev database. A PDPP node is a
+ # write-heavy record store: a real instance reached 5.4M records / 76 GB,
+ # where the stock max_wal_size of 1GB forced a checkpoint every 16-22
+ # seconds. Postgres logged "checkpoints are occurring too frequently"
+ # 8,209 times, ingest returned intermittent 503
+ # (ingest_batch_storage_error), and two backend processes were killed
+ # under the write pressure. Raising the WAL ceiling and checkpoint window
+ # removed the warnings outright.
+ #
+ # These are deliberately modest -- safe on a 4 GB VPS, and far better than
+ # the defaults on anything larger. An operator with more RAM should raise
+ # shared_buffers (~25% of RAM) and effective_cache_size (~50-75%).
+ command:
+ - postgres
+ - -c
+ - max_wal_size=${PDPP_POSTGRES_MAX_WAL_SIZE:-8GB}
+ - -c
+ - min_wal_size=${PDPP_POSTGRES_MIN_WAL_SIZE:-1GB}
+ - -c
+ - checkpoint_timeout=${PDPP_POSTGRES_CHECKPOINT_TIMEOUT:-15min}
+ - -c
+ - shared_buffers=${PDPP_POSTGRES_SHARED_BUFFERS:-512MB}
+ - -c
+ - effective_cache_size=${PDPP_POSTGRES_EFFECTIVE_CACHE_SIZE:-1536MB}
# No published port: Postgres is reachable only from the compose network.
# If you publish one, change PDPP_POSTGRES_PASSWORD first.
volumes:
diff --git a/design-notes/browser-stream-status-honesty-2026-08-22.md b/design-notes/browser-stream-status-honesty-2026-08-22.md
new file mode 100644
index 000000000..4db61756d
--- /dev/null
+++ b/design-notes/browser-stream-status-honesty-2026-08-22.md
@@ -0,0 +1,183 @@
+# Browser-stream: the captcha tap, and the false "logged in" claim
+
+2026-08-22
+
+Two owner-reported defects on the interactive browser-stream path, both
+repeated across several days. They are unrelated in mechanism but share a
+theme: the console asserted things it could not prove.
+
+## 1. "Can't tap the captcha on mobile" — the residual half
+
+Reported at least three times from 2026-08-18. Scroll and rotate worked; taps
+did not.
+
+### The iframe theory is wrong, and worth retiring explicitly
+
+A reCAPTCHA checkbox lives in a cross-origin iframe, so the natural suspicion is
+that synthetic pointer events do not cross the iframe boundary, or that
+coordinates fail to resolve inside the iframe's document.
+
+**That is not what happens here.** The remote input path is coordinate-based all
+the way down — neko dispatches X11 pointer events, and the CDP backend calls
+`Input.dispatchMouseEvent` with raw `x`/`y`. Both operate at the browser
+compositor level, *below* the DOM, so they cross cross-origin iframe boundaries
+by construction. Nothing in the path hit-tests the top document: there is no
+`elementFromPoint`, no `querySelector` against remote DOM, no frame targeting.
+The only `elementFromPoint` in the streaming server is a calibration-beacon
+diagnostic, not the input path.
+
+The captcha was never special. It is simply the one control an owner *cannot*
+work around, so it is where a general tap defect gets noticed and reported.
+
+### What two prior commits fixed, and what they left
+
+- `a21a9a1be` stopped *dropping* touch taps that reported a non-zero
+ `event.button`. Correct: `button` is mouse-state and must not gate touch.
+- `5274dbd4c` rerouted touch press/release onto the CDP mouse path, because
+ `Input.dispatchTouchEvent` does not reliably synthesize a `click`.
+
+Both are real fixes. Neither covers the residual case, because the first
+commit stopped *filtering* on `button` but still **forwards the raw value**, and
+downstream that value is not a filter — it is arithmetic:
+
+```js
+// remote-surface 1.5.2, controllers/neko-pointer-controller.js
+const x11Button = (event.button ?? 0) + 1;
+```
+
+X11 button 1 is primary. A touch `pointerdown` reporting `button === -1` — the
+same non-spec value `stream-viewer-pointer-input.ts`'s own doc comment documents
+as real on touch input paths — therefore becomes **X11 button 0**, which is not
+a button. neko presses nothing, and the tap never clicks.
+
+Verified by replaying the exact client payload through the real installed
+controller:
+
+| `button` on pointerdown | X11 call emitted |
+|---|---|
+| `0` (spec-compliant) | `buttonDown(1)` — correct |
+| `-1` | `buttonDown(0)` — **no button pressed** |
+
+`pointerup` was already safe by luck: the controller prefers the *remembered*
+press button over the event's own. `pointerdown` has nothing to fall back to.
+
+This also explains the reporting pattern. Engines that report a spec-compliant
+`0` always worked, which is why the bug never reproduced on desktop and why it
+looked intermittent rather than universal.
+
+### The fix
+
+`normalizedPointerButton` in
+`apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer-pointer-input.ts`
+pins touch and pen to primary contact at the payload boundary. Touch and pen
+have no secondary button, so this loses no information. Mouse is passed through
+untouched — its `button` is meaningful, and normalizing it would turn every
+right-click into a left-click.
+
+Guarded by `stream-viewer-touch-tap-oracle.test.ts`, which asserts against the
+**real dependency** rather than restating our own arithmetic, so a future
+dependency bump that changes the mapping fails loudly instead of silently.
+
+### Honest limits
+
+**This is not confirmed against a real phone.** I do not have the owner's
+device, and I did not attempt a Reddit login (OTP/bot-detection sensitive, and
+the owner has been explicit about not burning accounts). What is proven is
+mechanical: the payload the client sends, replayed through the real controller,
+pressed no button before this change and presses primary after it. Whether that
+was the *only* remaining cause of the owner's symptom is unverified — it is a
+genuine defect on the exact reported path, not a confirmed end-to-end repro.
+
+## 2. The false "logged in" state — the more dangerous half
+
+The UI told the owner he was already logged in to Reddit when he was not.
+
+### The evidence the UI used was a proxy
+
+`deriveSetupState` in `reference-implementation/runtime/static-secret-setup-status.ts`
+reaches `first_sync_running` / `first_sync_pending` for a browser session from
+`hasDraftSetupProgress`, which is satisfied by run evidence alone:
+
+```ts
+return hasSetupMaterial || (setupKind === "browser_session" && hasRunEvidence(input));
+```
+
+The console then rendered, in `connect/status/[connectionId]/page.tsx`:
+
+> "Login is complete and the first sync is running."
+
+**For a `browser_session` connection, the run IS the login attempt.** It starts
+precisely so the owner can sign in inside the streamed browser. So a run row
+proves a sign-in was *attempted*, never that it completed. The projection's own
+comment stated the flawed inference out loud — "(the owner completed login and a
+first sync started)" — which nothing in the projection verifies. `lastRun` also
+satisfies it, so a *previous* failed run made the UI claim login was complete on
+the current attempt.
+
+The asymmetry makes the defect obvious once seen. For the other two setup kinds
+the same states carry claims that **are** backed by evidence:
+
+| Setup kind | Claim | Backed by |
+|---|---|---|
+| `static_secret` | "The provider credential is captured" | `setup_material.present === true` |
+| `manual_upload` | "The import file is captured" | `setup_material.present === true` |
+| `browser_session` | ~~"Login is complete"~~ | **nothing** — `defaultSetupMaterial` pins `present: false` |
+
+### The fix
+
+The browser-session copy now describes only what is observed — a sync is
+running — and points the owner back at the browser, admitting the page cannot
+confirm the login itself. The static-secret and manual-upload claims are
+deliberately left intact, and a test pins them, so this does not over-correct
+into scrubbing information that is genuinely proven.
+
+Guarded by `browser-session-login-honesty.invariants.test.ts`, which fails on
+any phrasing that asserts a completed sign-in.
+
+### What was NOT fixed, and should be
+
+The connector layer already holds **real proof** of Reddit login state.
+`isSessionLive` in `packages/polyfill-connectors/src/auto-login/reddit.ts`
+fetches an owner-only JSON endpoint and requires HTTP 200 — ground truth, not a
+heuristic, and deliberately so per its own doc comment.
+
+**That verdict is never plumbed into the console.** Wiring it through would let
+the UI make a positive, evidence-backed claim instead of merely declining to
+make a false one. That is a larger change across the RI projection boundary and
+is left for a follow-up.
+
+Two related weaknesses found and not fixed here:
+
+- `isSessionLive` degrades to counting a logout-link selector when
+ `REDDIT_USERNAME` is unset — which is exactly the credential-less manual
+ handoff path. The repo's own tests document that this selector is unreliable
+ in both directions.
+- An instance flips to `status = 'active'` on first successful ingest and is
+ **never re-validated**. A session that worked in June and has since been
+ logged out server-side still projects `active` → `healthy`. There is no
+ `last_verified_at` on the credential row, and for browser sessions the
+ credential row is bypassed entirely.
+
+Both mean "connection active" is a durable-row claim, not a live one. That is a
+real instance of the same defect class and deserves its own change.
+
+## 3. "Couldn't reach the browser stream" — assessed, largely already sound
+
+Root-caused but not re-engineered, because the existing design is mostly honest.
+The give-up message fires after 10 attempts with backoff, and is followed by a
+diagnostic probe (`stream-reach-diagnostics.ts`) that recovers the real HTTP
+status and error code — necessary because `EventSource` collapses every
+pre-attach failure into a payload-less error. Four of six reasons already get
+specific, actionable copy.
+
+Two genuine residual gaps, left for a follow-up rather than fixed blind:
+
+1. `managed_surface_window_settle_unavailable` (a 503 the server describes as a
+ transient restart worth retrying) is not in the classifier's match list, so
+ it falls through to the vague generic copy.
+2. The generic message is shown *before* the probe resolves, so there is a
+ window where the owner reads network-flavored copy even when the cause is
+ precisely known moments later.
+
+Neither is a fabricated-green defect — the classification rests on a real HTTP
+status — so they are accuracy gaps, not dishonesty.
diff --git a/design-notes/connector-coverage-findings-2026-08-22.md b/design-notes/connector-coverage-findings-2026-08-22.md
new file mode 100644
index 000000000..77b0cd2f6
--- /dev/null
+++ b/design-notes/connector-coverage-findings-2026-08-22.md
@@ -0,0 +1,214 @@
+# Connector coverage findings — USAA and Gmail (2026-08-22)
+
+Four owner-reported coverage gaps (ledger items D1–D4). Two of the four had a
+wrong premise in the report; the evidence is recorded here because each finding
+is the kind that gets rediscovered as something else entirely six months later.
+
+## D1 — USAA `transactions` stuck at 2 of 4: a marketing interstitial, NOT a UI change
+
+**The finding that will otherwise be rediscovered as "USAA changed their UI".**
+
+USAA serves a promotional interstitial in FRONT of a checking account page:
+
+```
+https://www.usaa.com/my/banking-offer/atm-deposit
+ ?accountId=…&accountType=checking&goto=https://www.usaa.com/inet/ent_home/CpHome
+title: "Find an ATM | USAA" ("Depositing cash just got more convenient!")
+```
+
+`locateExportPage` navigated to the account URL, was redirected here, found no
+Export button, and reported `source_structure_changed` — whose owner-facing
+meaning is "the source's UI changed; retrying is pointless until the connector's
+selectors are revisited". **The selectors were fine.** The connector was looking
+at a page that never claimed to have an Export button.
+
+Only `LOGON_REDIRECT_RE` was checked after navigation. `/my/banking-offer`
+matched neither that nor `USAA_ACCOUNT_DETAIL_ROUTE_RE`, so it classified as
+`unknown` and fell through to the no-affordance path.
+
+### The correlation that proves it
+
+The denominator for `transactions` is **accounts, not statements**
+(`emitTransactionsDetailCoverage`, `state_stream: "accounts"`): 5 accounts, 4
+transaction-eligible (Chase is `external-account`).
+
+| account | type | offered ATM deposit? | outcome |
+|---|---|---|---|
+| Checking | checking | yes → interstitial | **gap** |
+| Family Checking | checking | yes → interstitial | **gap** |
+| Signature Visa | credit-card | no | covered |
+| American Express | credit-card | no | covered |
+
+**The 2 covered accounts were precisely the 2 credit cards** — the accounts that
+get no ATM-deposit offer. Both checking accounts hit the interstitial; both
+credit cards exported normally. That is the whole of the 2-of-4 gap. Captured
+live 2026-07-14 in the connector's own page artifacts (URL + title).
+
+Note also: `pdf_template_unknown` is a *separate*, much smaller skip on the
+statements→PDF path. It is NOT the cause of the 2-of-4 transactions gap, and
+chasing it would have fixed nothing.
+
+**Fix:** navigate through the offer once, using the interstitial's own `goto`
+param when it names a USAA account-detail route (off-host or wrong-section
+`goto` values are refused, so a redirect chain cannot steer the connector).
+Session death is re-checked *after* the hop, because an offer page can itself
+bounce to logon — that must surface as session-dead (which triggers re-auth),
+never as a missing affordance.
+
+**Watch for:** other banks doing the same thing. A promotional interstitial in
+front of an account page is a generic pattern, and the failure mode it produces
+— "the source changed its UI" — is maximally misleading.
+
+## D2 — a gap-opening path with no gap-closing path (general defect class)
+
+USAA `statements` simultaneously reported:
+
+- stream fact: `covered 10 / considered 10, checkpoint: committed`
+- gap table: **4 rows still `pending`**, three with `attempt_count = 0`
+
+Both readings were current. They disagreed because the four "pending"
+statements were not missing at all — every one has a durable `pdf_sha256` on its
+record. They had been downloaded.
+
+### The class
+
+A pending detail gap leaves `pending` **only** on an explicit
+`DETAIL_GAP_RECOVERED` (`connector-detail-gap-store.ts`). Nothing closes one
+implicitly: not a later success, not a full-coverage `DETAIL_COVERAGE`, not a
+committed checkpoint.
+
+USAA emitted that message for `transactions`
+(`recoverServedAccountTransactionGaps`) and for both credit-card streams —
+**but never for `statements`**. So the first statement PDF that ever failed to
+download opened a gap that no subsequent run could close, however many times it
+succeeded afterwards. Three of the four were never even re-attempted.
+
+> **The general shape: any stream with a gap-OPENING path but no matching
+> gap-CLOSING path accumulates permanent false gaps.** It is silent — the stream
+> looks fully collected on every axis except the gap table — and it is
+> self-inflicted, needing no source misbehavior at all.
+
+**Worth auditing across every connector**: for each stream that can emit
+`DETAIL_GAP`, confirm a `DETAIL_GAP_RECOVERED` path exists and is reachable.
+The asymmetry within a single connector (USAA had closers for 3 streams and not
+the 4th) suggests these are added per-stream as each is built, so the omission
+is easy to repeat.
+
+**Fix:** `emitStatementCoverage` now emits recovery for each served gap whose
+statement is hydrated this run, driven by `coverage.hydratedKeys` — the same set
+that feeds the coverage numerator, so recovery and coverage cannot drift apart.
+
+## D3 — Gmail `message_bodies` has no denominator: PROHIBITED, not unimplemented
+
+Definitive answer to the owner's direct question ("are you sure it's not
+possible?").
+
+`manifests/gmail.json` declares `message_bodies` with `state_stream: "messages"`,
+making it a **static single-parent detail stream** whose checkpoint status is
+projected from the parent's own commit outcome. `runtime/index.ts:1514`
+(`validateDetailCoverageAgainstManifest`) **throws and fails the entire run** if
+such a stream emits `DETAIL_COVERAGE`. This is not a missing feature; emitting a
+denominator is an error.
+
+The owner's reasoning is sound but lands one level off. The message count *is*
+knowable — but it is the **parent's** number. Reporting it under
+`message_bodies` would assert `covered == considered` for bodies that were
+skipped or whose fetch failed. A real denominator would require a per-key
+hydration tally, which this stream does not produce.
+
+`attachments` is the instructive contrast: it has no `state_stream`, and it
+earns coverage from a genuine attempt-per-key tally — which is exactly why it
+is permitted to emit at all.
+
+Already corrected in-tree (`cfe738071`); live evidence shows `collected: 4,
+checkpoint: committed` with no fabricated coverage.
+
+## D4 — Gmail `attachments` "won't backfill": a forged impossibility proof
+
+The 32 terminal `too_large` gaps are **all collectible**. None is genuinely
+oversized.
+
+Gmail's hydrator briefly sized attachments from imapflow's `meta.expectedSize`,
+populated from the FETCH `RFC822.SIZE` item — the size of the **entire message**,
+identical for every part of a multipart message. Against a per-part cap this
+condemned every attachment of a message whenever their SUM crossed it.
+
+The durable signature, straight from the rows:
+
+| claimed "observed" | sum of that message's parts | largest single part |
+|---|---|---|
+| 32,229,094 | 32,218,046 | 7,711,218 |
+| 30,062,404 | 30,051,218 | 8,709,138 |
+| 29,830,196 | 29,800,496 | 10,947,340 |
+
+32 gaps, only **7 distinct claimed sizes** (real per-part sizes are never
+byte-for-byte identical across distinct attachments); every claim ≈ the sum of
+its message's parts; the smallest condemned item is **3,080 bytes** against a
+26,214,400-byte cap.
+
+The connector was fixed (`1bf3f6cfa`, per-part BODYSTRUCTURE size), but the rows
+already written stayed terminal, and `isProvenUnfillableGap` reads
+`observed > limit` as durable per-item proof — so the false verdict outlived the
+defect that produced it.
+
+> **Transferable lesson: a durable "proof" is only as good as the measurement
+> behind it.** `too_large` was treated as unfalsifiable because a size beats a
+> retry count as evidence. But the number itself came from a buggy source, and
+> nothing re-checked it against the item's own recorded size. Any terminal state
+> justified by a recorded measurement should be falsifiable by independent
+> evidence.
+
+**Fix:** `classifyTooLargeProof` adjudicates a row against the item's OWN size —
+requeue only on positive contradiction (`fabricated_proof`); keep terminal for
+`proof_holds`, `no_corroborating_record`, and `not_a_size_proof`. Absence of
+contradiction is not proof of fabrication.
+
+Repair tool: `scripts/repair/requeue-fabricated-too-large-detail-gaps.ts`
+(gmail/attachments-locked, dry-run by default).
+
+**APPLIED 2026-08-22** to `cin_12407c1afb78d56848fe0b20`, with backup table
+`gmail_gaps_backup_20260822034236` (32 rows):
+
+| | before | after |
+|---|---|---|
+| `recovered / temporary_unavailable` | 10,236 | 10,240 |
+| `terminal / too_large` | **32** | **0** |
+| `terminal / quarantined` | 3 | 0 |
+| `pending / temporary_unavailable` | 0 | 31 |
+
+All 32 adjudicated `fabricated_proof` (0 `proof_holds`, 0
+`no_corroborating_record`). The 3 `quarantined` rows went through the existing
+allowlisted tool in the same pass. **Zero terminal gaps remain.**
+
+Recovery is confirmed real, not hollow: within a minute, 4 of the 32 had already
+moved to `recovered` with `hydration_status: hydrated` and content-addressed
+blobs (2,248 / 13,162 / 220,273 / 1,970,646 bytes) — every one far under the cap
+it had been condemned against. So the requeued gaps ARE picked up by the ordinary
+recovery path; this is not the D2 defect class inverted.
+
+Each requeued row carries an audit trail
+(`class: "too_large_proof_contradicted"`) recording the claimed size, the cap,
+and the item's real size, so the repair is legible in the row's own history.
+
+
+## Operational: a repair CLI that silently misread its own arguments
+
+Found while applying the D4 repair. `--reason too_large` (space-separated)
+parsed as `reason = "true"`: the parser read a value only from `--key=value` and
+substituted the boolean `true` otherwise. The operator saw
+
+ --reason='true' is not requeueable (allowed: quarantined, ...)
+
+naming a value they never typed. The refusal was correct — `too_large` IS
+refused by that tool by design — but the message pointed at the wrong thing, so
+the tool looked broken in a different way than it was, and the real routing
+(use the adjudicating tool) stayed hidden.
+
+Both repair CLIs now accept `--flag value` and `--flag=value`, and REFUSE a
+value-taking flag given no value rather than defaulting. Defaulting is what
+produced the defect; failing closed is the only safe reading of an ambiguous
+argument list for a tool that writes to production.
+
+> **Worth checking in other operator tooling:** the `--key=value`-only parser
+> with a `true` fallback is a common hand-rolled shape, and it fails silently
+> and specifically on the argument form most operators type first.
diff --git a/design-notes/connector-sidecar-packaging-2026-08-17.md b/design-notes/connector-sidecar-packaging-2026-08-17.md
new file mode 100644
index 000000000..eee9b9de9
--- /dev/null
+++ b/design-notes/connector-sidecar-packaging-2026-08-17.md
@@ -0,0 +1,199 @@
+# Who owns a connector's native sidecar once connectors leave the server repo?
+
+**Status:** intake. No requirement proposed. Written from evidence produced while
+shipping the Signal connector on 2026-08-17.
+**Date:** 2026-08-17
+
+## The question
+
+Several connectors shell out to a native binary they do not own:
+
+| connector | sidecar | license | how it ships today |
+|---|---|---|---|
+| slack | `slackdump` v4.4.2 | AGPL-3.0 | pinned tarball, SHA256-verified, builder stage in the RI `Dockerfile` |
+| google_messages | `gmcli` | — | same arms-length-subprocess pattern |
+| signal | `sigtop` v0.24.0 | ISC | built from pinned source in a Go builder stage (added today) |
+
+All three live in the **reference implementation's** `Dockerfile`. If connectors move
+into their own distribution — the data-connectors reorg — that stops working: a
+connector shipped separately cannot edit the server's image build.
+
+So: **how does an independently-distributed connector declare and obtain a native
+dependency, and who verifies it works on the runtime that will actually execute it?**
+
+## Evidence from shipping sigtop today
+
+Seven build failures, in order. Every one was caught before shipping, but the pattern
+matters more than the count:
+
+1. `golang:1.23` too old — sigtop needs Go ≥ 1.25
+2. missing `libsecret-1-dev` at build time
+3. license file is `LICENSE.md`, not `LICENSE` or `COPYING`
+4. **`libsecret-1.so.0` missing at runtime** — binary compiled cleanly, could not load
+5. **GLIBC 2.38 vs 2.36** — `golang:latest` is Debian trixie, the runtime image is
+ bookworm; the binary ran in the builder and died in the final image
+6. `sigtop -v` is not a valid subcommand
+7. `sigtop version` is not either
+
+**4 and 5 are the load-bearing ones.** Both produced a binary that built successfully
+and would have failed on the owner's first real sync. Neither is discoverable from the
+connector's own source; both are properties of the *runtime image* the connector will
+be executed in.
+
+That is the crux. A connector author can pin a version and a checksum. A connector
+author cannot know the runtime's glibc, its installed shared libraries, or its
+architecture — and today's evidence says those are exactly what break.
+
+## What the current pattern gets right
+
+Worth preserving whatever the packaging answer is:
+
+- **Pinned version + SHA256** on the downloaded artifact (`slackdump`), or a pinned
+ source tag with a commit-exact `SOURCE_URL` recorded (`sigtop`).
+- **Isolated builder stage** — Go and build dependencies never reach the final image.
+- **License and corresponding-source URL copied into the image**, which AGPL §6(d)
+ requires for `slackdump` and is good practice for ISC.
+- **Build-time smoke test.** `slackdump version` and (now) an execute-and-check for
+ `sigtop`. This is what caught failures 4 and 5. A verification step written as
+ `... || true` would have shipped both.
+
+## Options, none yet chosen
+
+**A. Connector declares, runtime resolves.** The manifest names a sidecar (source, pinned
+version, checksum, license) and the runtime image build reads those declarations and
+produces the binaries. Keeps one place that knows the runtime's glibc and libraries.
+Cost: the runtime build must enumerate every connector, which partially re-couples what
+the reorg is trying to separate.
+
+**B. Connector ships prebuilt per platform.** Each connector distributes its own
+binaries for supported platform triples; the runtime verifies checksum and executability
+on load. Fully decoupled. Cost: connector authors take on cross-compilation and a
+platform matrix, and today's evidence says that is precisely where the failures live.
+
+**C. Sidecar declared as a runtime prerequisite.** The connector declares "requires
+`sigtop` ≥ 0.24 on PATH" and refuses to register when absent, with a clear message. Zero
+packaging burden, but it breaks the property that makes the current product good — a
+self-hoster following the docker/railway/fly.io steps gets working connectors with no
+extra install. Slack works today because `slackdump` is *in the image*.
+
+## The constraint any answer must satisfy
+
+**Whatever ships must be verified against the runtime it will execute on, at build time,
+by executing it.** Not "the artifact downloaded," not "the checksum matched" — those both
+passed today while the binary was unrunnable. The only check that caught it was running
+the thing.
+
+## Open questions
+
+- Does the reorg keep a single runtime image, or do connectors get their own containers?
+ Option B is much more attractive in the latter case.
+- Is there an existing prior-art answer here? Language package managers with native
+ extensions solve a similar problem (Python wheels' manylinux, Node prebuilds), and
+ manylinux exists specifically because of the glibc problem hit today. Worth a sweep
+ before designing.
+- How does a self-hoster on a non-Debian base fare today? Untested — the current
+ `slackdump`/`sigtop` stages both assume Debian.
+
+## Related
+
+Same shape as the collector/server contract gap
+(`upstream-disclosure-window-2026-08-17.md` and the collector-contract findings): a
+component whose correctness depends on a peer's version, with nothing verifying the pair
+is compatible. Here the failure is loud at build time if a smoke test exists, and silent
+until first use if it does not.
+
+---
+
+## Proposed requirement (appended 2026-08-17 after prior-art research)
+
+Research: `~/.tmp/reorg-0814/sidecar-abi-prior-art.md` (corpus entry filed). Key finding:
+manylinux, Node prebuilds, and N-API all declare compatibility as data and verify by static
+analysis or eliminate the variable by construction — **no surveyed ecosystem executes the
+artifact on the real target before accepting it**. The constraint this note demanded is a
+genuine gap in prior art; adopting it puts this registry ahead of, not behind, the state of
+the practice.
+
+Direction (option B, shaped by the registry design; proposed, not owner-ratified):
+
+1. **Static by default** — `CGO_ENABLED=0`/musl-static for any sidecar without a real
+ `dlopen` dependency; erases the glibc class by construction (verify per-tool, don't assume).
+2. **ABI tags where dynamic is unavoidable** — per-artifact `{os, arch, libc, libc_floor,
+ linkage}` (manylinux/prebuildify model), built inside a pinned deliberately-old shared
+ build image (the registry's manylinux-image equivalent), so the floor is infrastructure,
+ not per-author judgment.
+3. **`smoke_cmd` becomes a manifest/artifact field** — the trusted installer executes it on
+ the actual runtime at install time and refuses on failure; loader errors already
+ distinguish "missing library" from "symbol too new" with no parsing.
+4. **Graceful fallback** — on smoke failure, try the static/alternate build before failing
+ the connector.
+
+Why not options A/C: A cannot survive in-app connector install (no image rebuild available
+at user install time) — transitional-only by construction; C breaks the self-hoster
+works-out-of-the-box property this note already names.
+
+Transitional: today's Dockerfile builder stages are server-repo property, untouched by the
+connector-content move; recorded as a known coupling whose removal trigger is registry
+artifacts carrying ABI-tagged (or per-connector-container) sidecars. For server deployments
+the container sandbox tier ultimately makes the sidecar ABI self-contained inside the
+connector's own image; the tag machinery chiefly serves bare-metal desktop.
+
+---
+
+## The packaging rule (settled 2026-08-17, window 20 disposition)
+
+**Sidecar packaging keys off the connector's placement bindings, not one uniform
+mechanism.**
+
+- **Network-authenticated sidecars** (`slackdump`) belong in the server's runtime image.
+ The tool reaches the provider over the network, so the server is a legitimate place to
+ run it, and the builder-stage pattern above is the right answer.
+- **Session-bound sidecars** (`sigtop`) can only be acquired to the *user's* machine.
+ No server-side image stage can help, because the constraint is not where the file is —
+ it is where the key can be unwrapped.
+
+That sentence is what makes the rest of this note cohere, and it is why "put the binary in
+the image" was the wrong instinct for Signal.
+
+### Evidence: Signal cannot run server-side, by construction
+
+Tested against real data on this host, four successive configurations:
+
+| attempt | result |
+|---|---|
+| container, no mounts | `open /root/.config/Signal/sql/db.sqlite: no such file` |
+| + Signal data mounted read-only | `cannot decrypt database key: cannot connect to D-Bus session bus` |
+| + host `/run/user/1000/bus` mounted | `EOF` (uid mismatch) |
+| + `--user 1000:1000` | `An AppArmor policy prevents this sender from sending this message` |
+
+`~/.config/Signal/config.json` holds `encryptedKey` with `safeStorageBackend: kwallet6`
+and no plaintext key. Mounting the database is insufficient because **the key is not in
+the file** — it unwraps only through a session-bound keyring daemon.
+
+### Consequences adopted
+
+1. **Signal ships local-collector-only**, with a PATH/`SIGTOP_BIN` resolution and a clear
+ install error as the interim acquisition story. Connector code must not fetch
+ executables at runtime; a downloader in the npm package today would be the insecure
+ version of the signed, ABI-tagged registry artifacts already designed above.
+2. **The constraint is now declared, not discovered.** `desktop_session` is a
+ first-class binding in `runtime_requirements.bindings`, and
+ `sourceKindFromManifestBindings` resolves it to `local_device` — the same placement
+ mechanism that already keeps browser connectors off the collector profile. The engine
+ refuses server-side placement up front rather than failing four D-Bus layers deep.
+3. **The `sigtop` builder stage is removed from the Core image.** Shipping a binary that
+ cannot work there implies support that does not exist. The builder-stage pattern
+ remains proven via `slackdump`.
+
+### Edge case worth documenting, not shipping for
+
+Signal Desktop configured with `safeStorageBackend: basic_text` stores the key
+**unwrapped**, so a server-side path does exist for users who have disabled their keyring.
+That is a documentation note, not a reason to carry an image stage — and a connector that
+declares `desktop_session` should keep declaring it, since the common configuration is the
+session-bound one.
+
+### Carry-through
+
+The connector fleet was copied to `data-connectors` around this change. The manifest and
+engine edits above were made in pdpp's canonical copy and **must be carried through the
+cutover rather than silently diverging.**
diff --git a/design-notes/cursor-provenance-audit-2026-08-20.md b/design-notes/cursor-provenance-audit-2026-08-20.md
new file mode 100644
index 000000000..59678fca0
--- /dev/null
+++ b/design-notes/cursor-provenance-audit-2026-08-20.md
@@ -0,0 +1,114 @@
+# Detecting a cursor a connection never earned
+
+**Status:** deferred. Code removed from the branch; no requirement is proposed yet.
+**Date:** 2026-08-20
+
+## What was built, and why it is not in the tree
+
+`reference-implementation/runtime/cursor-provenance.ts` and its test were removed from
+`fix/sweep-fairness-and-transformer-bounds` before that PR landed. The module was never wired into
+anything: its only reference anywhere in the repo was its own test file, and it does not exist on
+`main`. Its own commit (`6af425709`, "detect the three ways a cursor silently excludes data") says
+so deliberately — it "stays unwired: it needs the full connection set, so it belongs in the
+maintenance sweep rather than the per-run commit."
+
+Unwired, it still failed the zero-connector-knowledge conformance gate, because its watermark table
+names eleven connectors in RI production source. That is a real gate finding, not a false positive.
+Shipping the module unwired bought nothing and blocked a PR carrying real production fixes, so the
+code came out and the analysis stays here.
+
+This note exists so the next person does not rediscover the defect, the ruling, or the blocker.
+
+## The defect: `duplicate_of_sibling`
+
+A watermark cursor ("fetch everything newer than T") is only sound when T was reached by *this*
+connection walking its own history. Seed a new connection with another connection's high-water mark
+and every record older than T becomes permanently unreachable — the connection will only ever ask
+for newer items — while `covered == considered` keeps reporting complete coverage, because the run
+genuinely did process everything it fetched.
+
+This is live, not hypothetical. ChatGPT connection `cin_484604984db7c091bd08b259` (created
+2026-08-17) held a `conversations` cursor of `2026-06-19T20:30:04.127Z`, byte-identical to the
+millisecond to the cursor of a different, paused connection (`cin_e4ab231c7d49b8f59e4c80ed`) that
+reached that value on its own final run. Two separate accounts do not independently walk to the same
+millisecond. The value was copied, and everything older than it is unreachable for the newer
+connection.
+
+The signal is exact equality across sibling connections for the same connector and stream, never
+proximity and never staleness. Staleness is explicitly not evidence: an account nobody has posted to
+in six months has a legitimately frozen watermark, and firing on it would cry wolf on exactly the
+quiet connections an owner is least able to check by hand.
+
+A finding is `suspected`, never a quantified loss. The evidence proves the cursor's *provenance* is
+unsound, which makes any completeness claim built on it unfounded; it does not measure how much data
+is unreachable. The right outcome is to withhold the healthy claim and tell the owner to re-seed.
+
+One rule was tried and disproved by the live fleet: flagging a cursor that predates its own
+connection's `created_at` produced seven false positives across reddit, github, and notion. Those
+watermarks store the newest *content item's* timestamp, not an observation time, so a connection
+created today that fully walks an account whose newest post is from 2024 correctly stores 2024. The
+rule was removed rather than tuned.
+
+## The ruling: this audit stays RI-owned
+
+The architecture owner's decision, which this note records so it is not relitigated: the watermark
+specs must NOT move into connector manifests, unlike the sibling `cursor-band-contiguity` check
+whose `cursor_shape` enum did move.
+
+Letting a connector declare which of its fields are checkable lets the audited party define the
+audit's scope. A connector could omit or mis-path its watermark and silently exempt itself, or
+mis-declare `valueKind` (`iso8601` as `epoch_seconds`), making comparisons meaningless with no
+error raised. This is an integrity check *about* the connector evaluated *against* the connector,
+so the connector cannot be the one who says what counts.
+
+The contrast with `cursor-band-contiguity` is the useful part. There, the manifest declares only a
+closed enum (`cursor_shape: "imap_uid_band"`) and the RI owns every semantic — the paths, the
+`UIDVALIDITY` epoch guard, the arithmetic. Declaring can only opt a stream *in*; omission yields
+silence, never a healthy verdict. That asymmetry is what makes manifest declaration safe there and
+unsafe here.
+
+## The blocker: the sibling-JSON route fails both ways
+
+The obvious fix — move `WATERMARK_SPECS` into an RI-owned JSON data file (e.g.
+`reference-implementation/data/watermark-specs.json`) and load it at runtime, keeping the data
+RI-owned rather than connector-declared — does not work. It was tried and probed empirically against
+the real scanner, not just reasoned about. Both halves of rule 5 reject it:
+
+1. **The path.** `SANCTIONED_POLICY_RESOURCES` in
+ `reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts` is
+ deliberately an empty `Map`. Its own doc records that the two production files which used to load
+ a sibling RI-owned registry through it (`compact-record-history.ts`, `version-disposition.ts`)
+ were migrated *away* on purpose. A probe file loading such a JSON returns
+ `unsanctioned-policy-resource-path`.
+
+2. **The content.** Even with an allowlist entry, `classifySanctionedPolicyResource` parses the
+ file and walks it for any string — object key, value, or array element, at any depth — equal to a
+ manifest-derived connector key, reporting `hardcoded-connector-literal-in-ri-owned-json`. A
+ watermark-spec file is nothing but connector keys, so it fails here even if the path were
+ sanctioned.
+
+The scanner states the principle directly, and it is correct: moving a connector-identity fact out
+of `.ts` source into a sibling RI-owned JSON file "is exactly as much self-attested connector
+knowledge as the literal it replaced, reached via a different seam."
+
+Adding an allowlist entry to get past this would be defeating a gate that is working as designed,
+not satisfying it.
+
+## The viable path
+
+Wire the check into the **maintenance sweep**, with the watermark specs supplied by the call site
+rather than held as a module-level table in RI production source.
+
+This fits the shape of the check anyway. `evaluateCursorProvenance` needs the full set of
+connections for a connector/stream to compare siblings, which is a sweep-time input, not a per-run
+one — the reason the original commit left it unwired. Making the specs a parameter means the pure
+comparison logic stays RI-owned and connector-agnostic, and the identity table lives wherever the
+sweep legitimately assembles per-connector context.
+
+That is a real design task with its own review surface: where the sweep gets the specs, whether a
+finding withholds a healthy verdict or only annotates, and how `suspected` is surfaced to an owner
+without implying a measured loss. It deserves its own PR rather than being rushed to unblock CI.
+
+Recovering the code: `git show 6af425709 -- reference-implementation/runtime/cursor-provenance.ts`
+(and the sibling test path). The module and its 9 tests were passing when removed; only the
+conformance gate objected, and only to the identity table.
diff --git a/design-notes/failure-diagnosability-2026-08-18.md b/design-notes/failure-diagnosability-2026-08-18.md
new file mode 100644
index 000000000..169cfa001
--- /dev/null
+++ b/design-notes/failure-diagnosability-2026-08-18.md
@@ -0,0 +1,635 @@
+# A failure must not destroy its own cause
+
+**Status:** intake. No requirement proposed. Written from five production
+failures observed on 2026-08-18, two of them fixed the same day.
+**Date:** 2026-08-18
+
+## Why this note exists
+
+Five failures in about one day. All five have the same shape: **something went
+wrong, and the evidence needed to act on it was destroyed by the code that
+handled it.** In four cases the failure itself may well have been transient and
+harmless. What made them cost a day was that nothing downstream could tell.
+
+1. **`[object Object]`.** `packages/polyfill-connectors/src/reference-blob-uploader.ts`
+ built its error text with `String(body.error ?? statusText)`. The RI host
+ always shapes `error` as an object — `pdppError` in
+ `reference-implementation/server/request-helpers.ts:92` writes
+ `{code, message, type}` — so `String()` produced the literal
+ `"[object Object]"` for every host-side failure. That discarded the cause of
+ 24 quarantined Gmail attachment gaps, each recorded as
+ `blob upload failed (503): [object Object]`. Fixed in `457e23e93`.
+2. **Silent statement_timeout.** `observeConnectorSummaryEvidence`'s outer catch
+ treated a typed `PostgresStatementTimeoutError` like any other error and
+ routed it to `markAllConnectorSummaryEvidenceDiscoveryFailed`, durably
+ writing `record_snapshot_state='failed'` across every row in scope. Neither
+ that path nor `repairCandidatePostgres` logged anything. 25 of 29 evidence
+ rows degraded in production with **zero** log output. Fixed in `1d8995b0f`.
+3. **Empty `failure_reason`.** A failed ChatGPT run (`run_1787075769450`) wrote
+ zero log lines matching its own `run_id`, and `run_history.failure_reason` is
+ empty on every failed row. Only `terminal_reason` and a
+ `connector_error_json` blob survived. Not fixed.
+4. **`[REDACTED]`.** HEB connection `cin_c875ca3ec8b6ce2c283a4288` failed with
+ `connector_error_json = {"code": null, "message": "heb_session_failed: [REDACTED]", "retryable": false}`.
+ The cause is literally the string `[REDACTED]`. Partly fixed same-day in
+ `46887c2e8`, which populates the `code` channel; the `message` channel is
+ the subject of the proof-of-concept below.
+5. **Unpublished dependency.** Every published `@pdpp/local-collector`
+ (1.5.1–1.5.4) has `import ... from "@pdpp/reference-contract/common"` as line
+ 1 of `dist/polyfill-connectors/src/local-device-client.js`, but that package
+ is not in `dependencies` and does not exist on npm. Every install crashes
+ with `ERR_MODULE_NOT_FOUND` on any invocation, including `--version`. It
+ works in the monorepo because pnpm resolves it through the workspace link.
+
+Numbers 1–4 are error handling. Number 5 is packaging, and it belongs to a
+different family; it is addressed separately at the end.
+
+## First, the scale numbers are wrong
+
+The intake for this note claimed ~246 bare catches and ~282 stringified-error
+coercions. Both were re-measured. The raw catch count is **higher** than
+claimed and the story is **much smaller**.
+
+| metric | intake claim | measured |
+|---|---|---|
+| bare `catch {` (non-test) | ~246 | **439** |
+| bare `catch {` (tests, separate) | — | 207 |
+| `catch (e) {}` empty body (non-test) | — | **0** |
+| `String(err)`-style on error-named values | ~282 | **173** |
+| `REDACTED` (non-test, in scope) | ~58 | 35 |
+
+The original grep almost certainly used `catch\s*\{`, which also matches
+`catch (e) {`. It conflated two populations while undercounting the one it
+named.
+
+Then 55 of the 439 bare catches were read and classified, one per file, across
+55 distinct files:
+
+| class | count | share |
+|---|---|---|
+| benign cleanup | 8 | 15% |
+| benign optional parse | 24 | 44% |
+| benign probe | 21 | 38% |
+| **fault-swallowing** | **0** | **0%** |
+| **degrades durably** | **2** | **3.6%** |
+
+**About 97% of bare catches are benign, and the fault-swallowing class is
+empty.** The dominant patterns are `JSON.parse` → `return null` on stored
+manifests and cursors, `new URL(x)` → `return false` in SSRF validators, and
+telemetry taps explicitly commented "must never break the streaming path."
+Several carry a comment justifying the swallow. This is a deliberate house
+style, not neglect. **There is no catch-block crisis here, and a campaign to
+migrate 439 call sites would be almost entirely waste.**
+
+What the sample did find is three real defects, verified end-to-end to their
+persistence points:
+
+- `packages/polyfill-connectors/src/statement-content-fingerprint.ts:142` —
+ any PDF text-extraction failure returns the all-null fingerprint, which is
+ then persisted as statement `content`. The record durably says "no extractable
+ content" whether the PDF is genuinely empty or the extractor crashed. USAA's
+ sibling path at `statement-pdfs.ts:502` shows the fix: it emits `onSkip` with
+ `structuralErrorDiagnostic(err)`.
+- `packages/polyfill-connectors/connectors/gmail/index.ts:1418` — an IMAP body
+ fetch failure returns null bodies and the message is emitted anyway, with no
+ `DETAIL_GAP` and no coverage marker. Given the measured ~4.65 KB/s IMAP
+ throttle on this connector, transient failures are expected, so bodyless
+ messages land durably with nothing distinguishing them from empty mail.
+- `reference-implementation/server/routes/as-grant-revoke.ts:157` —
+ `String(e?.message ?? hookErr)`, where the catch neither rethrows nor changes
+ the response. A failed grant-revoke side effect vanishes into one unreadable
+ log line while the caller gets a success envelope.
+
+Both durable-degradation sites are the same shape: **a fail-closed default that
+is indistinguishable from a legitimate empty result.** Neither needs a `try`
+restructured; each needs one extra field saying why the value is null.
+
+The `457e23e93` archetype has **no surviving siblings** — a targeted search for
+`String(...)` over parsed-JSON HTTP error bodies returns zero hits. And all 35
+`REDACTED` occurrences are redaction *mechanism* (regex constants, scrub tables,
+sanitizers), not a bare `message: "[REDACTED]"` standing in for a lost cause.
+
+So the honest headline is **three fixes, not 246.** The rest of this note is
+about why those three happened and what makes the next one impossible.
+
+## What this codebase already gets right
+
+There is a good, half-built convention here, and it is worth naming precisely
+because the answer is to finish it rather than import something foreign.
+
+**Two channels with opposite disciplines.** Stated outright in
+`packages/polyfill-connectors/src/terminal-error.ts:44-57` and implemented once,
+correctly, in `buildTerminalConnectorFields`
+(`reference-implementation/runtime/index.ts:2846`):
+
+- `code` is a **typed** channel. It is *validated, never redacted*, and fails
+ closed to null. `boundConnectorErrorCode` tests it against
+ `/^[a-z][a-z0-9_]{1,63}$/` and drops anything malformed.
+- `message` is a **prose** channel. It is *redacted and truncated*, never
+ trusted — `boundConnectorErrorMessage` runs `redactStderrTail` then caps at
+ 500 characters.
+
+The security argument for the asymmetry is explicit in the source: `code` is
+exempt from redaction *only because* the charset makes it incapable of carrying
+a credential, a URL, or a stack trace. That is a genuinely good design, and it
+is the thing to generalize.
+
+Three more pieces are already right:
+
+- **`recovery_hint` is a closed vocabulary** of 8 actions
+ (`runtime/connector-gap-bounding.ts:117`), with the design intent stated
+ well: *"a connector requests an ACTION this way; it never gets to pick one by
+ shaping its `code` or free-form `message` text."*
+- **Structured failure evidence on the row.** `search_index_dirty` records
+ `last_error`, `attempts`, and `next_attempt_at` atomically
+ (`queries/search/index-dirty/record-failure.sql`), with the comment
+ *"observable evidence, not just a console.warn line."* This is the right
+ instinct and the right place to put it.
+- **The typed-error → reason-code mapping** the `1d8995b0f` fix introduced:
+ `err instanceof PostgresStatementTimeoutError ? REASON_CODES.STATEMENT_TIMEOUT : default`.
+ That is exactly the shape the terminal design needs, written once.
+
+### Where the convention is only half-built
+
+The pattern exists. Its *closure* does not.
+
+- **Closure is upheld by hand-copied `Set`s and tests, not by types.**
+ `RECOVERY_ACTIONS` is a `Set`. Both `REASON_CODES` objects are
+ unexported `as const` with no derived type. `codeToStatus`
+ (`routes/ref-error-status.ts:89`) is `Record`, so an
+ unregistered error code compiles fine and silently becomes a 500. Only
+ `SharedConnectionConditionReason` is a real derived union — and it is the one
+ vocabulary with a single producer and a single consumer.
+- **It has already drifted.** `scripts/stream-health-audit/authority.ts`
+ maintains duplicate `Set`s that omit `repair_statement_timeout` and contain
+ `"summary_evidence_unavailable"`, which exists in no const.
+ `"update_connector"` is emitted at `runtime/connection-health.ts:1982,2747`
+ and is not in `RECOVERY_ACTIONS`.
+- **~170 error subclasses, no shared base.** The de-facto common field is
+ `code: string`, but the HTTP-status carrier is variously `statusCode`,
+ `httpStatus`, and `status`. Discrimination splits three ways: 134 `instanceof`
+ sites, 74 `.code ===` sites, and 3 fragile `.name === "..."` string
+ comparisons.
+- **The typed channel degrades into prose-sniffing.** When a recovery hint is
+ absent or invalid, `inferRecoveryAction`
+ (`runtime/connector-gap-bounding.ts:739`) regex-matches the free-form message
+ to guess an action. `connector-coverage-policy.ts:167` matches connector
+ reason strings by substring. These are the seams where a structured design
+ silently becomes a guess.
+- **There is no logger.** No module exports a log API. Pino exists but is wired
+ to exactly one call site (`transport.ts:335`) and is never exported, so no
+ library module can reach it. Everything else is `console.*` with a
+ `[module-tag]` prefix — **19 calls in the entire server, none carrying
+ `run_id`.** That is the mechanical reason incident 3 produced zero lines
+ matching its own run id: there is no facility that would have written one.
+ The real correlation channel is the spine (`lib/spine.ts:483`), which carries
+ `run_id`, `trace_id`, `request_id`, and `grant_id`, and hard-rejects a
+ malformed event. `packages/polyfill-connectors/src/` has no spine access at
+ all, which makes it the least observable surface in the system.
+
+## The invariant
+
+The candidate invariant from intake was:
+
+> A failure must never lose the information needed to act on it. Every failure
+> that crosses a durability boundary carries a machine-readable code, a
+> human-readable cause that is not a stringified object, and a PII-safe
+> diagnostic detail. A catch that discards a fault is a bug, not a style choice.
+
+The last sentence should go. The measurement says the fault-swallowing class is
+empty and 97% of bare catches are correct, so "a catch that discards a fault is
+a bug" indicts a house style that is not what broke. It would also push toward
+a 439-site migration that buys nothing.
+
+The rest is close but describes a payload rather than a property. What actually
+failed in all four cases is narrower and more testable:
+
+> **A failure that crosses a durability boundary must carry a cause that is
+> reconstructable from what is written down.** Every failure persisted to a row,
+> returned to a caller, or shown to the owner carries (a) a machine-readable
+> code from a closed vocabulary, and (b) a human-readable cause. Any transform
+> applied on the way out — coercion, redaction, truncation, classification —
+> must be **category-preserving**: it may drop detail, but it may never leave a
+> value whose failure class can no longer be told apart from a different one.
+
+The operative word is *category-preserving*. `[object Object]`, `[REDACTED]`,
+`''`, and `record_snapshot_state='failed'` with no reason are all the same
+defect under this rule: each is a value that survived the boundary while
+becoming indistinguishable from every other failure that produced the same
+placeholder.
+
+### Testing it against the five incidents
+
+| # | prevented? | why |
+|---|---|---|
+| 1 `[object Object]` | **yes** | The coercion is not category-preserving: every distinct host error maps to one string. Caught by the rule directly. |
+| 2 silent timeout | **yes** | A cancelled read and a genuinely bad row both wrote `failed`. Distinguishing them is exactly what `1d8995b0f` added, and it is what the rule requires. |
+| 3 empty `failure_reason` | **partly** | The rule forces the field to be populated. It does **not** by itself produce a correlated log line — that needs a logger this codebase does not have. |
+| 4 `[REDACTED]` | **yes** | Redaction that collapses distinct reasons to one token is not category-preserving. |
+| 5 unpublished dep | **no** | Nothing was mishandled. No failure crossed a boundary; the artifact never ran. Different family. |
+
+**Where it would not have helped, honestly:**
+
+- It would not have prevented incident 5 at all.
+- For incident 3 it fixes the durable row but not the missing log. Someone
+ debugging by `grep run_id` still finds nothing until a logger exists.
+- It says nothing about failures that are never caught in the first place, or
+ about a correct code attached to a wrong diagnosis.
+- It does not address the two durable-degradation sites found by measurement
+ (`statement-content-fingerprint.ts:142`, `gmail/index.ts:1418`). Those write a
+ *successful-looking record* with a silently-null field; no failure crosses a
+ boundary, so the invariant never engages. They need a different rule — a
+ fail-closed default must be distinguishable from a real empty result — and
+ that rule is worth stating separately rather than stretching this one.
+
+## The mechanism
+
+One small thing, not a framework. The two-channel design already exists and is
+already correct; what is missing is that its vocabularies are open and its
+transforms are not category-preserving. Three changes, in order of leverage.
+
+**1. Close the vocabularies with types, not `Set`s.** Every reason vocabulary
+becomes an exported `as const` with a derived union:
+
+```ts
+export const RECOVERY_ACTIONS = {
+ RETRY_BY_RUNTIME: "retry_by_runtime",
+ // ...
+} as const;
+export type RecoveryAction = (typeof RECOVERY_ACTIONS)[keyof typeof RECOVERY_ACTIONS];
+```
+
+This is a mechanical change with immediate payoff: `"update_connector"` and the
+drifted audit `Set`s become type errors rather than silent divergence, and
+`codeToStatus` stops turning an unregistered code into a 500 by default. It
+requires no call-site migration — only the declarations move.
+
+**2. Make redaction category-preserving.** This is the subject of the
+proof-of-concept below, and the finding there is the most interesting result in
+this note.
+
+**3. Populate `failure_reason` from the terminal event.** The empty field in
+incident 3 is not a bug in the ordinary sense. It is a hardcoded literal, in
+both backends, with a comment explaining why:
+
+```ts
+// reference-implementation/server/stores/run-history-writer.ts:275 and :366
+const terminalReason = typeof event.data.reason === "string" ? event.data.reason : null;
+const failureReason: string | null = null;
+```
+
+The comment says `failure_reason` is a scheduler-only classification and is
+"left null rather than fabricated, since no Slice A reader depends on it for
+non-scheduled runs." That reasoning was sound when written and is now false —
+`ref-spine-correlations-list` and the console both read it. **This is worth
+dwelling on: the information was dropped deliberately, on a reader-side
+assumption that later stopped holding.** No lint rule catches that. It is an
+argument for making the *durable schema* carry the classification unconditionally,
+so a writer cannot decide on a reader's behalf that a cause is not worth keeping.
+
+### The gate that makes regression impossible
+
+This repo already has the right mechanism, used twice, and it is better than a
+linter rule for this purpose. `lefthook.yml` runs
+`check-direct-prepare-conformance.ts`, which pins grandfathered sites at exact
+`(path, line)` in a checked-in allowlist and fails on three divergences: a
+**new** hit not in the allowlist, a **stale** row whose site moved or was
+migrated, and a **duplicate** row. The polyfill-connectors `noAwaitInLoops` gate
+has the same shape. The comment states the property that matters:
+
+> The rule still fires on any NEW direct-prepare anywhere, INCLUDING a new one
+> inside an already-allowlisted file, and additionally fails on a STALE
+> allowlist row so exceptions cannot be carried silently.
+
+That is precisely the migration mechanism this design needs, and it already
+exists. A new `check-error-envelope-conformance.ts` in the same shape would
+enforce the one defect worth banning outright: **`String(...)` applied to a
+caught value or a parsed HTTP error body.** That is a real, narrow, mechanically
+detectable pattern with exactly one known instance left
+(`as-grant-revoke.ts:157`), so the allowlist starts at ~1 entry rather than 439.
+
+Biome cannot express this. It is version 2.5.6 extending `ultracite`, and the
+repo already works around missing rules with grep gates — the `no-double-cast`
+job exists because "Biome/Ultracite has no equivalent to typescript-eslint's
+`consistent-type-assertions` yet." A `String()`-on-caught-value rule needs type
+information about the argument, which is why the grep gate is the honest answer
+here rather than a stopgap.
+
+**What should not be banned:** empty `catch {}`. The measurement says 97% are
+correct and the fault-swallowing class is empty. A lint rule there would
+generate 439 suppression comments and teach people to ignore the gate.
+
+## Proof of concept: category-preserving redaction
+
+Incident 4 was chosen over incident 3 because another agent is already working
+in `run-history-writer.ts`, and because it turned out to have the more
+interesting answer.
+
+While this note was being written, `46887c2e8` landed a **complementary** fix
+for the same incident from the other side: it populates `TerminalError.code` on
+the session-establishment path, so the *typed* channel carries the reason even
+when the prose channel is destroyed. That is the right first move and it
+confirms the two-channel design is the one to generalize. It does not restore
+the `message`, which is what an owner reads and what `inferRecoveryAction`
+consumes, so the two fixes stack rather than compete. The proof-of-concept below
+touches `runtime/stderr-redact.ts`, which `46887c2e8` does not; both were
+verified green together.
+
+**The defect, reproduced exactly.** `LONG_OPAQUE_RE`
+(`runtime/stderr-redact.ts:43`) is `/\b[A-Za-z0-9_-]{24,}\b/g` — an **entropy**
+heuristic, aimed at unlabelled API keys in stack traces. Categorical reason
+tokens match it too:
+
+```
+"heb_session_failed: login_form_never_appeared" -> "heb_session_failed: [REDACTED]"
+"usaa_session_failed: source_unavailable" -> "usaa_session_failed: source_unavailable"
+```
+
+`login_form_never_appeared` is 25 characters and carries no PII whatsoever.
+`source_unavailable` is 18 and survives. **Whether a failure stayed diagnosable
+was decided by the length of its reason token.** Run through the real production
+boundary, `boundConnectorErrorMessage` reproduces the exact production string
+`heb_session_failed: [REDACTED]`.
+
+There is a second half to this. Once the message is destroyed,
+`inferRecoveryAction` regex-matches the *redacted* text to choose a recovery
+action, and returns `"unknown"`. So the redaction did not just cost a human
+reader the cause — it silently degraded the machine-actionable output too.
+
+**The finding: shape cannot fix this.** The obvious fix is a smarter pattern —
+preserve alphabetic `snake_case`, redact anything with entropy. Tested against
+real secret shapes, it separates cleanly:
+
+```
+login_form_never_appeared kept sk_live_ redacted
+heb_verification_code_not_provided kept eyJhbGciOiJIUzI1NiIsInR5cCI6... redacted
+two_factor_challenge_unrecognized kept a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6 redacted
+```
+
+And then it fails, in the way that matters:
+
+```
+tim_nunamaker_gmail_com kept <- a personal name
+```
+
+**A name is alphabetic snake_case too.** No regex distinguishes a declared
+reason token from a person's name, because the difference is not spelling — it
+is *provenance*. That is the load-bearing conclusion, and it is why the
+mechanism is an allowlist rather than a better pattern.
+
+So: a token survives only if the connector **declared** it in advance. The
+declaration is reviewable in the connector's source, where a human reading
+`login_form_never_appeared` can see it is a constant, and would see a name for
+what it is.
+
+```ts
+// reference-implementation/runtime/stderr-redact.ts
+next = next.replace(LONG_OPAQUE_RE, (match) => (declared?.has(match) ? match : "[REDACTED]"));
+```
+
+The full change is 36 lines, most of it the comment explaining why shape does
+not work. Six tests in
+`reference-implementation/test/stderr-redact-declared-reasons.test.ts` pin the
+four properties the fix must have, including the two that say what it
+deliberately does not do:
+
+```
+✔ regression: the production defect — a declared reason token is destroyed by length alone
+✔ a declared reason token survives redaction, so the owner sees the real cause
+✔ secrets are still redacted even when a declaration set is supplied
+✔ an UNdeclared reason token still redacts — declaration is the safety property, not spelling
+✔ callers that do not opt in are byte-identical to the previous behaviour
+✔ disclosed pre-existing gap: this redactor is not a PII control
+```
+
+**Verification.** 18/18 tests pass across the new file and the existing
+`stderr-redact` suite; 20/20 across `connector-gap-bounding` and
+`device-exporter-sanitize` consumers; 16/16 on the systemic-failure redaction
+and stderr-tail oracles. `tsc --noEmit` clean, `biome check` clean. The
+declaration set is optional and defaults to empty, so every existing call site
+is byte-identical — pinned by its own test, since a redaction change that
+silently widened what escapes would be much worse than the bug it fixes.
+
+**Disclosed while building it.** Today's redactor already passes
+`tim.nunamaker@example.com` and `tim_nunamaker_example` through untouched, with
+no options involved — both are under the 24-character threshold. This is
+pre-existing and independent of the change, but it should be recorded plainly:
+`LONG_OPAQUE_RE` is an entropy heuristic and **is not a PII control**, though
+its name and position invite reading it as one. The declared-token change can
+only ever reduce what escapes, never widen it. Whether a real PII boundary is
+needed here is a separate question this note does not answer.
+
+## Incident 5 is a different family
+
+Nothing was mishandled. No failure crossed a boundary. The artifact simply never
+ran anywhere except where it was built — the same shape already recorded in
+`connector-sidecar-packaging-2026-08-17.md`, which put it well:
+
+> Whatever ships must be verified against the runtime it will execute on, at
+> build time, by executing it.
+
+The pleasant surprise is that **the smoke test already exists and is already
+correct.** `packages/local-collector/scripts/pack-install-run.ts` packs the
+tarball, installs it into a clean temp npm project outside the workspace with an
+isolated `HOME` and cache, and then *executes the installed binary* — `advertise`,
+`enroll`, and `run --connector codex` against an in-process reference server. It
+asserts forbidden packages are absent and the bin is executable. An `npm install`
+of a package whose dependency does not exist on npm fails at the install step.
+**It would have caught this.**
+
+It is wired to `pnpm --filter @pdpp/local-collector run verify`, and `verify` is
+`pnpm test && pnpm validate:package`. `pack-install-run` is not in it. CI calls
+`verify` at `.github/workflows/semantic-release.yml:130`.
+
+So the fix is one line — add `pack-install-run` to `verify` — and the rule is:
+
+> **A package's release gate must install the packed artifact in a clean
+> environment and execute it.** Not "the build passed," not "the types check" —
+> those both passed while every published version was unstartable.
+
+The right place is the existing `verify` script, so it runs in
+`semantic-release.yml` before publish and stays available locally. The same gap
+should be checked for `@pdpp/cli` and `@pdpp/mcp-server`, which have sibling
+`verify` scripts. Worth noting separately: `pnpm` workspace linking is what
+*hid* this, so any check that resolves through the workspace is structurally
+incapable of finding it. That is the same lesson as the sidecar note, one layer
+up the stack.
+
+## Cost, and what stays broken
+
+Hundreds of call sites cannot and should not be migrated. The measurement is
+what makes this affordable — the real work is small:
+
+1. **Close the vocabularies** (types only, no call-site changes). Highest
+ leverage, lowest risk. Turns existing drift into compile errors.
+2. **Wire `pack-install-run` into `verify`** for all three published packages.
+ One line each; closes incident 5 permanently.
+3. **Land the redaction change** and give connectors a place to declare reason
+ tokens. Already proven; needs the declaration plumbed from the manifest.
+4. **Populate `failure_reason`** in `run-history-writer.ts` for non-scheduler
+ runs. Needs coordination — another agent is in that file.
+5. **Add the conformance gate** for `String()` on caught values, starting from a
+ ~1-entry allowlist.
+6. **Fix the three measured defects** — `statement-content-fingerprint.ts:142`,
+ `gmail/index.ts:1418`, `as-grant-revoke.ts:157`.
+
+New code is forced onto the new path by step 5, which is the only step that has
+to be right the first time; the rest are additive.
+
+**What stays broken in the meantime:**
+
+- There is still no logger, so incident 3's "zero lines matching the run_id"
+ stays true even after `failure_reason` is populated. A structured logger with
+ correlation fields is a real piece of work and is not scoped here. The spine
+ is the closest existing thing and connectors cannot reach it.
+- `inferRecoveryAction`'s prose-sniffing fallback and
+ `connector-coverage-policy.ts`'s substring matching both stay. They are the
+ seams where the typed design degrades into a guess, and closing them means
+ deciding what happens when a connector supplies no hint at all.
+- The ~170 error classes keep their three discrimination styles and their
+ `statusCode`/`httpStatus`/`status` drift. The duck-typed `.code` convergence at
+ the HTTP boundary works well enough that a base-class migration is hard to
+ justify on today's evidence.
+- The two fail-closed-null sites are real but need their own rule; the invariant
+ in this note does not reach them.
+
+## Open questions
+
+- Where do connectors declare their reason vocabulary? The manifest is the
+ obvious home, and `reason-display-messages.ts` already keys on
+ `(connector_key, reason_code)` with an anti-parrot rule — but its
+ exhaustiveness is enforced only by an AST-scanning test, which is the same
+ open-vocabulary weakness described above.
+- Should `failure_reason` be a closed union rather than free text? There is
+ already an unexported closed union in `runtime/classify-runtime-failure.ts:14`
+ that never returns empty. Exporting it may be most of the answer.
+- Is a real PII boundary needed where `redactStderrTail` currently sits, given
+ it passes plain email addresses through today?
+- Do `@pdpp/cli` and `@pdpp/mcp-server` have the same unpublished-dependency
+ defect? Only `local-collector` was checked.
+
+## Provenance
+
+Written from five production failures on 2026-08-18. Incidents 1 and 2 fixed
+same-day in `457e23e93` and `1d8995b0f`. Scale numbers re-measured against
+`deploy/prod-plus-fixes-0817`; the catch classification is a 55-site sample
+across 55 files (12.5% of 439), so the two durable-degradation sites are
+verified end-to-end but any extrapolation from them should be treated as an
+order-of-magnitude hint, not a census. The redaction proof-of-concept and its
+six tests are working code, not a sketch.
+
+Related: `connector-sidecar-packaging-2026-08-17.md` (incident 5 is the same
+"verified where it was built, not where it runs" shape) and
+`summary-evidence-projection-controller-2026-08-18.md`, whose subject is the
+sweep that produced incident 2.
+
+---
+
+## Addendum: the wider class is failures nothing reports
+
+Added 2026-08-18 after the note was applied to code outside this repo.
+
+An engineer migrating this instance's Postgres read the draft and found the
+same defect in their own verification gate within five minutes:
+
+```bash
+tgt_rows=$(psql -Atqc "SELECT count(*) FROM ${tbl};" 2>/dev/null || echo "ERROR")
+```
+
+That is `[object Object]` in bash. The gate fails correctly and exits non-zero
+while collapsing permission denied, a missing table, a dropped connection, and
+genuine data loss into one string. On migration night that is the difference
+between a five-minute fix and a torn-down restore. It appeared twice, and the
+fix was the same: capture stderr and log it before failing.
+
+Worth recording because it says the rule is not TypeScript-specific and not
+about error objects. It is about any transform on the way out of a failure
+path that keeps the fact of failure and discards which failure.
+
+### The count that reframes it
+
+That engineer's session produced four significant findings, and only one was a
+thing breaking loudly:
+
+- Postgres crash-recovering behind `RestartCount: 0` and green healthchecks
+- meilisearch crash-looping behind a healthcheck that only probed its own port
+- promtail dying on a full disk without telling anyone
+- `pg_dump` exiting 0 while producing an unrestorable dump
+
+Four of five were *nothing reported it*, not *it broke*. Our five incidents
+were the narrower shape — something reported a failure and destroyed its cause.
+Both belong to one family, and the wider one is the more dangerous half,
+because the narrow shape at least leaves a row to investigate.
+
+So the invariant needs a second clause. The first is already stated above: a
+failure that crosses a durability boundary must carry a code, a human cause,
+and a PII-safe detail. The second: **a component whose failure is survivable
+must still be observable — a supervisor that restarts, a probe that recovers,
+or a process that degrades silently is not thereby healthy, and something must
+say so.** `RestartCount: 0` is the canonical false negative: the container
+never died, so every container-level signal stays green while the process
+inside it crashed and recovered.
+
+This is why the missing logger matters more than its size suggests. Nineteen
+`console.*` calls in the server and none carrying `run_id` is not merely
+inconvenient; it means the only detection layer for an in-process crash is the
+database's own log, which on this host was reaching no aggregator at all —
+promtail scrapes the systemd journal and has neither docker discovery nor
+socket permission. The blind spot was total, not partial, and nothing reported
+that either.
+
+### What this does not change
+
+The remediation scope stays three sites. The measurement in this note stands:
+439 catches, 97% benign, the `[object Object]` archetype with no surviving
+siblings. Adding a second clause to the invariant does not widen the migration
+— it widens what counts as evidence when deciding whether a component is
+healthy, which is a monitoring question, not a refactor.
+
+### A third variant: reported correctly, in a field nobody read
+
+The migration engineer, having just added the second variant, produced the
+third by making the mistake himself an hour later. He checked whether the
+maintenance sweep's backlog had drained:
+
+```sql
+SELECT record_snapshot_state, count(*) FROM connector_summary_evidence GROUP BY 1;
+-- current | 29
+```
+
+Twenty-nine of twenty-nine current, so he reported the backlog clean and
+warned that a post-deploy measurement could not distinguish the fix from the
+status quo. The real backlog at that moment was nine rows, and the sweep had
+logged seventy consecutive no-progress passes. `dirty` is a different column.
+
+Nothing was hidden, destroyed, or masked. The failure was fully and correctly
+recorded, in a field he did not read.
+
+This is the same error made earlier the same day from the other direction —
+reading `connector_summary_evidence` columns and reporting "15 of 21 green"
+while the Sources page showed otherwise. Both readers were competent, both
+queried real data, and both were confidently wrong.
+
+The structural cause is that this table carries seven signals — `dirty`,
+`state`, `record_snapshot_state`, `terminal_facts_state`,
+`manifest_declaration_state`, `retained_bytes_state`,
+`list_summary_projection_state` — none of which is the verdict, while the
+actual verdict lives in `isHealthyConditionSet`
+(`reference-implementation/runtime/connection-health.ts:1739`), ten conditions
+evaluated together. Any reader who samples one column gets a plausible answer
+that is not the answer.
+
+**A system with N independent health signals and no single authoritative one
+invites every reader to pick a different signal and be confidently wrong.**
+
+The remedy is not better logging — the logging was perfect. It is that a
+verdict must have exactly one source, and the raw signals must be hard to
+mistake for it. That is a naming and API problem: `record_snapshot_state` reads
+like the state of the record snapshot, which it is, and like the state of the
+row, which it is not.
+
+Worth noting what this shares with the other two variants and what it does
+not. All three end with an operator holding a wrong conclusion. Only the first
+involves anything being destroyed, and only the second involves anything being
+unreported. The third needs neither — it is sufficient to offer several true
+answers to slightly different questions and let the reader choose.
diff --git a/design-notes/fused-source-status-2026-08-22.md b/design-notes/fused-source-status-2026-08-22.md
new file mode 100644
index 000000000..4ea9dfdea
--- /dev/null
+++ b/design-notes/fused-source-status-2026-08-22.md
@@ -0,0 +1,133 @@
+# The fused source status line — why it wasn't there, and what it cost
+
+2026-08-22
+
+The owner asked on 2026-08-19 why the console has no product-standard fused
+status string — the "Last updated 3 hours ago / Syncing now" line every mature
+sync product shows — and asked for the underlying design problem explained
+rather than patched over. This is that explanation, and the design that shipped.
+
+## What the owner saw
+
+On `/sources`, each row showed one colored glyph (`●◐⊘○◌⏸`) and nothing else
+about state. The status *text* existed but was `sr-only` — announced to screen
+readers, invisible to eyes. Freshness never reached the row at all. Whether a
+source was syncing right now was used only to disable buttons and set a polling
+interval; it was never displayed.
+
+So the row could not answer any of the three questions an owner actually has:
+what is this source's state, when did it last update, and is it working right
+now?
+
+## Why this is not a one-line fix
+
+The tempting fix is to concatenate three fields. That fails because **these are
+three independent axes that routinely disagree**, and the disagreements are
+exactly the cases that matter:
+
+| Situation | Freshness | Activity | Verdict | What the owner needs to hear |
+|---|---|---|---|---|
+| Sync running against a broken connector | stale | syncing | blocked | **Blocked** — the run will not save it |
+| Healthy source mid-refresh | fresh | syncing | healthy | Working, syncing now |
+| Stale because every run fails | stale | idle | blocked | Blocked, and here is how long |
+| Paused with a stale in-flight run flag | stale | "syncing" | n/a | **Paused** — it is not syncing |
+| Never connected | none | idle | unknown | Never updated |
+
+A source can be fresh and failing. It can be stale and syncing. It can look busy
+and be dead. Any fused string has to *resolve* those conflicts, and the
+resolution rule is the entire design — not the concatenation.
+
+## What the code actually did: last-writer-wins
+
+`deriveRenderedSourceStatus` in `apps/console/src/app/(console)/lib/source-actionability.ts`
+resolves the conflict by returning early:
+
+```ts
+if (running) {
+ return { dot: "◌", freshnessNote: null, kind: "pending", label: "Syncing", tone: "muted" };
+}
+```
+
+An in-flight run **erased both the freshness note and the health verdict**. A
+source that was blocked, stale, and had a doomed retry in flight rendered as
+`"Syncing"` in a muted tone — the calmest thing on the page.
+
+That is the fabricated-green defect class this program exists to kill, in
+miniature. Nobody wrote "pretend it is fine"; the most reassuring axis just
+happened to be evaluated last and win the single available slot. **Fabricated
+green is usually an architecture accident, not a lie somebody typed.** One slot
+plus three axes forces a silent choice about which truth to discard, and the
+convenient one wins by default.
+
+## The rule that shipped
+
+> **Activity is additive, never substitutive.**
+
+"Syncing" is something a source is *doing*, not something it *is*. So:
+
+- The **state slot** always holds the worst honest verdict.
+- **Freshness** gets its own slot and is never erased by activity.
+- **Syncing** is appended as a separate clause and owns no color of its own.
+
+The line therefore reads `Blocked · Last refreshed 6 days ago · Syncing now` —
+which is uncomfortable, and correct. The old rendering of that same source was
+the single word `Syncing`.
+
+Implementation: `apps/console/src/app/(console)/lib/fused-source-status.ts`.
+Axes are ranked worst-to-best (`SEVERITY_BY_KIND`) and the worst wins the slot
+by comparison, not by branch order — so adding a state later cannot silently
+reintroduce last-writer-wins.
+
+### Recovering the discarded verdict
+
+Because `deriveRenderedSourceStatus` throws the verdict away on `running`, the
+fused line needs it back. `deriveSourceVerdictStatus` re-derives the
+verdict-only status, and `projectSourceActionability` passes it as
+`verdictFallback` — but **only for the `running` collapse**. `revoked`,
+`paused`, and `pending` are lifecycle facts that outrank any verdict: a revoked
+source is revoked no matter how its last verdict read. Passing the fallback for
+those would let a stale "Blocked" overwrite "Revoked".
+
+### Honest absence
+
+A missing freshness note is not "fine". When a source has never had a
+successful run, the line says `Never updated` rather than omitting the slot —
+omission reads as "not applicable", which is a stronger claim than the evidence
+supports. When a source *has* succeeded but carries no annotation, the slot is
+genuinely absent rather than guessed.
+
+The server already folds activity into its own freshness annotation
+(`rendered-verdict.ts` emits `"Refreshing now."` when `badges.syncing`). That
+phrasing is dropped here and re-added from the real activity flag, so the
+activity slot has exactly one owner and cannot double up.
+
+## What is guarded
+
+`fused-source-status.test.ts` pins the disagreement cases, and the rule is
+mutation-proven three ways: restoring last-writer-wins, inverting the severity
+comparison, and letting paused/revoked show as syncing each fail tests that name
+the specific dishonesty.
+
+CSS state rules change **color only** — the geometry-bearing rules are
+state-independent — so the list never reflows as sources change health. This is
+enforced by the pre-existing "status state does not create separate row
+geometry" invariant.
+
+## Coordination note
+
+This work deliberately does **not** restructure
+`apps/console/src/app/(console)/lib/connection-evidence.ts`, which the
+`labels-naming-0822` lane is changing. The fused line *composes* the label
+strings that module and the server verdict produce; it defines no owner-facing
+label vocabulary of its own beyond the two connective strings `"Syncing now"`
+and `"Never updated"`. If that lane renames a pill label, the fused line picks
+it up with no change here.
+
+## What is not addressed
+
+- Freshness granularity is still the server's day-level prose
+ (`"3 days ago"`), not minute-level. Mixing the client's `formatRelative`
+ (minutes/hours) with the server's day buckets would read inconsistently, so
+ one source of truth was kept. A finer server annotation would improve this.
+- The detail page still renders the axes separately. Only the `/sources` list
+ row is fused so far.
diff --git a/design-notes/graceful-drain-verdict-2026-08-22.md b/design-notes/graceful-drain-verdict-2026-08-22.md
new file mode 100644
index 000000000..8cd08e924
--- /dev/null
+++ b/design-notes/graceful-drain-verdict-2026-08-22.md
@@ -0,0 +1,86 @@
+# Graceful drain on SIGTERM: do not re-attempt
+
+**Verdict: rejected, with production evidence. `fix/graceful-drain-0821` should
+not be landed.**
+
+This note exists because the idea is genuinely appealing and has already been
+built twice. Without a durable record it will be rediscovered — the reasoning
+below is what stops the third attempt.
+
+## What was tried
+
+`fix/graceful-drain-0821` (uncommitted work in
+`~/.tmp/graceful-drain-0821`) makes `drainActiveRuns` CANCEL every in-flight
+run on SIGTERM and then await its terminalization within a 5s budget, so the
+dying process writes each run's own terminal event instead of leaving it to be
+adjudicated at the next boot. It also closes run admission during shutdown and
+adds a `shutdown_drained` terminal reason.
+
+The engineering is careful and the diagnosis of the read-side problem is right
+(a restart-ended run should not be reported as `run.failed`). The mechanism is
+what does not work.
+
+## Why it does not work
+
+**The budget cannot cover the work.** Production sets no `--stop-timeout`, so
+Docker's 10s default governs, and `--stop-timeout` is fixed at container
+creation — Docker has no equivalent of systemd's runtime `EXTEND_TIMEOUT_USEC=`.
+Real runs take minutes: production medians span 9s (github) to 1221s (ynab). The
+gap is two orders of magnitude and is not closable by tuning.
+
+**It was measured failing exactly that way.** A production shutdown logged:
+
+```
+drained:0, elapsedMs:5000, timedOut:1
+```
+
+The drain spent its entire budget and abandoned the run anyway, having consumed
+half the window before SIGKILL while making the failure look handled. That is a
+negative win — worse than not trying, because it delays the storage close that
+still has to happen.
+
+**Correctness never depended on it, and cannot.** A `kill -9`, an OOM kill, or a
+power loss gets no drain at all. Any design that needs the dying process to
+write its own terminal state has an unhandled case by construction. The
+successor must adjudicate regardless, so the drain is a redundant second
+mechanism guarding a case the first already covers.
+
+This is the standard layering, not a local shortcut: Temporal's
+`WorkerStopTimeout` defaults to 0s and the service writes the terminal state on
+a timer; Kafka recovers a dead producer's transaction through the successor's
+`InitProducerId` epoch bump.
+
+## What replaced it
+
+`2ddcca1b8` ("perf(shutdown): drop the connector drain from the SIGTERM path")
+removed the shutdown call site and left `drainActiveRuns` on the controller,
+where it means "await in-flight runs" for the watchdog, `awaitRun`, and the test
+suite. Correctness lives in `reconcileOrphanedRunsAtBoot`
+(`lib/controller-boot.ts`), which writes `run.abandoned` for any run whose owner
+epoch is not the current one.
+
+That commit is an ancestor of the current base. **The drain branch predates it
+and does not contain it**, so landing the branch would re-introduce a mechanism
+this repo already removed on evidence.
+
+## The valid insight, and where it went instead
+
+The drain branch was right that a restart-ended run must not be reported as a
+connector failure. That belongs on the read side, where it works for every
+restart shape including `kill -9`:
+
+- `8609f37a8` — restart-abandoned runs no longer classify connection health;
+ they defer to the last run that actually observed something.
+- `3cf21fd04` — restart-ended runs no longer reset the schedule anchor, so a
+ restart stops delaying the next run by a full interval.
+
+Both apply regardless of how the process died, which is precisely the property
+the drain could not offer.
+
+## If someone still wants a drain
+
+The only version worth discussing would need all of: a stop timeout raised well
+beyond 10s at container creation, evidence that the raised timeout does not get
+the process SIGKILLed mid-storage-close, and a reason why successor
+adjudication is insufficient despite handling the `kill -9` case the drain
+cannot. Absent all three, this is settled.
diff --git a/design-notes/local-collector-coverage-architecture-2026-08-22.md b/design-notes/local-collector-coverage-architecture-2026-08-22.md
new file mode 100644
index 000000000..b0003fcb4
--- /dev/null
+++ b/design-notes/local-collector-coverage-architecture-2026-08-22.md
@@ -0,0 +1,171 @@
+# Local-collector coverage: two parallel systems, not one dropped field
+
+Investigation of PR #166 owner-feedback item C1: four local-collector sources
+(peregrine Claude Code, peregrine Codex, Simon VM Claude Code, vivid fish Claude
+Code) emit coverage facts with blank `covered`/`considered` while server-run
+connectors emit real numbers.
+
+**Read this before "fixing" the blank denominators.** The obvious fix is wrong
+and would fabricate a false coverage claim. The disproof is in §3.
+
+## Verdict
+
+The owner's reasoning — "local collector runs a connector the same way the
+server does, so denominators should depend on the CONNECTOR, not the runner" —
+is sound in principle but does not describe how this is built. There is no
+single runner with a dropped field. There are **two parallel coverage systems**
+with different units, vocabularies, and transports.
+
+This is a **per-connector gap plus a transport that cannot carry counts**, not a
+runtime silently discarding numbers the connector emitted. `claude_code` and
+`codex` emit no `DETAIL_COVERAGE` message at all — zero occurrences in either
+`connectors/*/index.ts`.
+
+| | Server path | Local path |
+|---|---|---|
+| Signal | `DETAIL_COVERAGE` message | `coverage_diagnostics` records |
+| Unit | per-record counts | per-**store** status enum |
+| Vocabulary | `covered` / `considered` | `collected`, `inventory_only`, `missing`, `deferred`, `excluded`, `unsupported`, `unaccounted` |
+| Verdict source | `covered` vs `considered` | `localCoverageConditionForStatus` |
+
+## 1. Three independent barriers
+
+Any one of these alone is sufficient to produce blank denominators. All three
+are present, so a partial fix changes nothing.
+
+**Barrier 1 — the producer has no numeric field.**
+`packages/polyfill-connectors/src/local-source-inventory.ts:168-181`.
+`CoverageRecord` carries `status`, `store`, `stream`, `reason` — no count.
+`coverageStatus()` (:541) answers "does this store exist and what is its
+policy", one status per store. Counts that *are* measured get stringified into
+prose at :245:
+
+```ts
+return `enumeration complete, ${input.examined} examined (${input.emitted} emitted)`;
+```
+
+**Barrier 2 — the wire contract forbids counts.**
+`packages/reference-contract/src/reference/index.ts:1839-1848`:
+
+```ts
+const DeviceTerminalRunFactSchema = {
+ additionalProperties: false,
+ properties: {
+ coverage_statuses: { items: { minLength: 1, type: "string" }, minItems: 1, type: "array" },
+ scoped: { type: "boolean" },
+ stream: { minLength: 1, type: "string" },
+ },
+ required: ["coverage_statuses", "stream"],
+```
+
+`additionalProperties: false` — numeric coverage is actively rejected, not
+merely absent. `canonicalTerminalRunCommitEnvelope`
+(`packages/reference-contract/src/common/terminal-run-commit.ts:32-45`)
+reconstructs each fact from exactly these three fields, so anything else is
+dropped before signing.
+
+**Barrier 3 — the server hard-codes null.**
+`reference-implementation/operations/local-device-terminal-collection.ts:249-258`:
+
+```ts
+const fact = readRuntimeCollectionFact({
+ checkpoint: "committed",
+ collected: 0,
+ considered: null,
+ coverage_statuses: coverageStatuses,
+ covered: null,
+ ...
+```
+
+This exactly reproduces the observed rows: `covered: null`, `considered: null`,
+`collected: 0`, `checkpoint: "committed"`.
+
+## 2. The local path is NOT verdict-less
+
+Worth stating because "coverage unknown" was over-scoped in the ledger. The
+server has a working local coverage authority:
+`reference-implementation/server/ref-control.ts:3829-3848` maps
+`collected -> "complete"`, `inventory_only -> "inventory_only"`,
+`unaccounted -> "gaps"`. `coverageTone` (`runtime/rendered-verdict.ts:560`)
+renders `complete`, `deferred`, and `inventory_only` all **green**.
+
+`connector-coverage-policy.ts:246-249` treats `observed_collected` (derived from
+`coverage_statuses`) as legitimate non-numeric coverage proof, and
+`verdict.proven` can return `"complete"` with no denominator at all.
+
+So blank denominators do not automatically mean a red or unknown stream. Check
+the rendered verdict per stream before scoping work.
+
+## 3. Why `examined` is NOT the denominator (the trap)
+
+The prose already contains real numbers, so piping them into `considered` looks
+like a one-line fix. It is a fabricated denominator. Production disproof
+(peregrine Codex, `cin_ece4bfe5096b8bf67a1468c2`):
+
+```
+coverage_diagnostics reason: "enumeration complete, 566589 examined (0 emitted)"
+actual stored records: 758,345
+```
+
+**Examined (566,589) is LOWER than stored (758,345).** `scanLocalJsonl`
+(`packages/polyfill-connectors/src/local-jsonl-cursor.ts:139-172`) is
+incremental: on a `fast_skip` or `append` decision `onLine` fires only for new
+bytes. `examined` is therefore a **per-run delta**, not a corpus size.
+
+Rendering that as `considered` would show "566589 of 566589 — complete" for a
+run that skipped most of the corpus. That is precisely the fabricated-watermark
+defect class this program exists to eliminate.
+
+It is also not uniform. `sessions` reports `"declared rollout source"` — **no
+count exists at all**. Any fix must be per-stream.
+
+## 4. What an honest denominator would be
+
+For these inventory-style local collectors: **corpus discovered vs corpus
+parsed**, measured at the enumeration site independently of the incremental
+cursor — e.g. sessions/transcript files discovered on disk vs successfully
+parsed. That satisfies the standard `DetailCoverageParams` already documents
+(`connector-runtime.ts:494-509`): measured at the enumeration site, never
+aliased to the collected/emitted count.
+
+Today's counters cannot supply it. `messagesExamined`
+(`connectors/claude_code/index.ts:1447-1470`) is honestly measured per-line and
+classified independently of emission — it is a *good* number, just a
+per-run-delta one. Getting a real denominator means teaching the connectors to
+measure corpus size on a full pass, separate from the cursor.
+
+Streams that genuinely cannot have one (a store whose whole content is
+`inventory_only` by policy) should keep saying so precisely via
+`coverage_statuses` rather than reporting a blank number.
+
+## 5. Cross-repo scope
+
+The end-to-end fix spans three places, one of which is not this repo:
+
+- **data-connect** (`PDP-Connect/data-connect @ 9155e57`) — the collector
+ runtime and `@pdpp/connector-protocol` that build `terminal_facts`. Vendored
+ here as `vendor/pdpp-collector-runtime-0.0.1.tgz` and
+ `vendor/pdpp-connector-protocol-0.0.1.tgz`; not editable in this repo.
+- **this repo, reference-contract** — the wire schema AND
+ `canonicalTerminalRunCommitEnvelope`, which is the **commit-id / replay hash
+ authority**. Widening the fact shape changes envelope hashes, so it is a
+ compatibility break requiring a versioned migration, not an additive field.
+- **this repo, reference-implementation** — `normalizeTerminalFacts` and the
+ coverage policy.
+
+Recorded as a known architectural gap. Not attempted here: a hash-contract break
+across repos needs deliberate scoping.
+
+## 6. Related finding: freshness is a separate defect
+
+Ledger C3 suspects local sources write no run history. **They do** — 537
+succeeded runs for peregrine Claude Code, 380 for peregrine Codex, all recent.
+C3 is right only for Signal.
+
+Local-device runs are deliberately discarded for health
+(`ref-control.ts:5215-5221`, `localDeviceBacked ? null : ...`), so freshness
+rests entirely on the heartbeat gate. A single dead-lettered row out of 10,001
+forced `blocked` -> `stalled` -> no freshness proof -> RED "can't collect" on a
+2.5-million-record source whose collector was healthy. Fixed separately in
+`localDeviceFreshnessHeartbeatAt`; see
+`test/local-device-terminal-backlog-freshness.test.ts`.
diff --git a/design-notes/manual-import-coverage-receipt-2026-08-19.md b/design-notes/manual-import-coverage-receipt-2026-08-19.md
new file mode 100644
index 000000000..feeea8d1f
--- /dev/null
+++ b/design-notes/manual-import-coverage-receipt-2026-08-19.md
@@ -0,0 +1,164 @@
+# A finished import has no way to prove what it ingested
+
+**Status:** superseded 2026-08-20. The premise below was wrong on the facts, and
+the proposed build is unnecessary. See "What was actually true" at the end.
+Written 2026-08-19 after a related fix was deliberately scoped to exclude this
+case.
+
+## The state that has no honest verdict
+
+Two sources on this instance hold real data and can never be green:
+
+- Google Maps Timeline Import — 299,248 records
+- WhatsApp — 120,042 records
+
+Both are `source_kind='manual'`, paused, with zero runs ever and zero rows in
+`connector_detail_gaps`, `run_history`, or `acquisition_batches`. They are
+finished one-time file imports. Nothing will run again.
+
+`isHealthyConditionSet` requires `SourceCoverageComplete` to be true. Coverage
+evidence is produced by a collection run. No run, no evidence, no green — on a
+source holding 419k records the owner can read today. The page says "Not
+measured", which he reads as broken.
+
+## Why this is not the same as the case next to it
+
+An adjacent problem looks identical from the pill and is not. Gmail has 32
+terminal gaps, each carrying `observed_size_bytes > configured_limit_bytes` —
+a specific attachment, a recorded size, a recorded cap. Gmail measured its
+shortfall and can enumerate it. That case is being handled by
+`unfillableAccounted`, a flag whose contract requires **every** outstanding gap
+to carry durable per-item evidence.
+
+These two imports fail that contract on the facts. There is no `terminal_gap`
+axis to account for, because there are no gaps. Their coverage axis is
+`unknown` — never measured — not `terminal_gap` — measured and partly
+impossible.
+
+**"Measured, and this part is provably impossible" and "never measured, and
+never will be" are different states and must not share a signal.** Setting the
+Gmail flag for an import would assert that every outstanding gap is proven
+unfillable over an empty set, inferred from the total absence of evidence.
+That is the exact false-green the anti-green tests exist to catch.
+
+An earlier framing in this work treated both as one design. That was wrong, and
+the evidence above is what corrected it.
+
+## The line that already exists, and holds
+
+`Fresh` can be satisfied by `not_applicable` (`conditionIsSettledSatisfied`).
+`SourceCoverageComplete` cannot — it is gated by `conditionIsTrue`
+(`connection-health.ts:1830`). That asymmetry is deliberate, and
+`source-state-truth-2026-08-18.md` states why: a completed import buys
+exemption from a *freshness* proof, never from proving it ingested what it
+claimed. Relaxing coverage would let any source with no evidence read as
+complete.
+
+So the fix is not to exempt coverage. It is to let an import **prove** its
+coverage.
+
+## The shape a proof would take
+
+`connector-coverage-policy.ts` already declares a `snapshot_import_receipt`
+coverage strategy alongside `checkpoint_window` and `full_inventory`. No
+connector emits evidence for it. The placeholder is the design, unbuilt.
+
+What it needs, roughly:
+
+1. The manual-upload route writes an `acquisition_batches` receipt at import
+ time recording what the file claimed to contain and what was ingested. The
+ table exists and is empty for both connections.
+2. The stream's manifest declares `coverage_strategy: snapshot_import_receipt`.
+3. The coverage projection grows a branch that reads that receipt as proof, the
+ way it reads a run's coverage report today.
+
+Then a finished import satisfies coverage the same way a collecting source
+does — by evidence, not by exemption — and the terminal label already built for
+it (`Fresh: not_applicable`, "Import complete") becomes reachable.
+
+## Cost and scope
+
+This touches the manifest schema, the upload route, and the coverage read path.
+It is a separate change from the Gmail work deliberately: bundling them would be
+two problems in one story, and the Gmail fix is narrow and provable on its own.
+
+Worth stating plainly: until this exists, these two sources stay honestly red.
+That is the correct outcome. The alternative — green by exemption — would mean
+the page can no longer distinguish a source that proved its completeness from
+one that never tried.
+
+## Open questions
+
+- Does the receipt record the file's own claim (a manifest inside the export,
+ a row count) or only what PDPP ingested? A receipt that only records what was
+ ingested proves nothing about what was missed.
+- Are existing imports retrofittable, or is this only correct for imports made
+ after it ships? Both sources here predate any receipt, so they may need a
+ one-time backfill with explicitly weaker provenance — and that weaker
+ provenance should be visible, not silently equal to a real receipt.
+- Does a partial import (an interrupted upload) produce a receipt that honestly
+ reports incompleteness, or none at all?
+
+---
+
+## What was actually true (2026-08-20)
+
+Three of the four load-bearing claims above are false. Steps 1-3 should not be
+built.
+
+**"No connector emits evidence for `snapshot_import_receipt`" — false.** Both
+connectors already emit an artifact-grounded coverage declaration, and both
+manifests already declare the strategy (`google_maps.json:166,246`,
+`whatsapp.json:140,208`). `google_maps` counts every point/segment element the
+parser produced, at the parse site, and reports `covered < considered` when an
+element had no usable id or timestamp (`finishPoints`/`finishSegments`,
+hardened by `e1b92b36c` the same week this note was written). `whatsapp` counts
+the archive's own file and attachment listing and subtracts media the bounded-
+read policy dropped. Neither denominator is recomputed from the survivors.
+
+**"Nothing will run again" — false, and this is the load-bearing one.** The
+uploaded artifact is durable and is re-read on **every** run, not just at setup:
+`ManualUploadDurableSourceBinding` carries `import_dir`/`import_dir_env_var`
+specifically so they survive promotion, and the run orchestrator injects the
+directory as the connector-declared env var. These connections are re-runnable
+against the file already on disk.
+
+**"Coverage evidence is produced by a collection run" — true, and sufficient.**
+`connector_summary_evidence` is keyed by `connector_instance_id`, but is only
+ever populated by folding a terminal run event's `collection_facts`. Zero runs
+therefore means `stream_latest_facts_json IS NULL` and axis `unknown` — which is
+the correct and honest reading of "never measured", not a defect.
+
+Derivation over the exact shapes these two connectors emit, through the real
+`deriveStreamCoverageCondition`:
+
+| fact | axis |
+| --- | --- |
+| `considered=299248, covered=299248` (fully reconciled) | `complete` |
+| `considered=299248, covered=299243` (5 elements unaccounted) | `partial` |
+| whatsapp attachments, media dropped by read policy | `partial` |
+| no run ever — the live state today | `unknown` |
+
+So the proof path is built and works end-to-end, including the honest partial.
+**The remedy is operational — re-run each connection once against its stored
+artifact — not a receipt, a manifest change, or a read-side branch.** A backfill
+would be strictly worse: it would synthesize a denominator from records already
+ingested, which is the fabricated-denominator anti-pattern (`covered ==
+considered` recomputed from survivors) that the coherence contract exists to
+reject, and it could never produce the `partial` verdict a real re-run can.
+
+One genuine defect did come out of this review, fixed separately: `CredentialsValid`
+had no branch for a connector that authenticates to nothing, so both sources sat
+at `credentials_not_probed`/`unknown` forever. That is now `not_applicable`,
+derived from the manifest declaration.
+
+What remains unprovable either way, and is worth not overclaiming: a reconciled
+artifact proves the run ingested everything the *file* contained. It says
+nothing about whether the file is a complete export of the owner's history —
+the provider decides what goes into it, and no check on this side can see past
+that.
+
+## Related
+
+`source-state-truth-2026-08-18.md` — the `not_applicable` design and the
+deliberate decision not to extend it to coverage.
diff --git a/design-notes/semantically-bounded-consent-2026-08-07.md b/design-notes/semantically-bounded-consent-2026-08-07.md
new file mode 100644
index 000000000..41d63a444
--- /dev/null
+++ b/design-notes/semantically-bounded-consent-2026-08-07.md
@@ -0,0 +1,78 @@
+# Semantically Bounded Consent (derived streams vs dynamic selectors)
+
+Status: captured
+Owner: Tim
+Created: 2026-08-07
+Related: spec-core Grant semantics; derived subset streams aside (non-normative);
+spec-deferred predicate scoping; openspec change harden-pdpp-authorization-and-0-1-migration
+(critical-extension and seam-spike gates); inbox/8-7-26-chatgpt-convo.txt
+
+## Question
+
+A user wants consent bounded by a subjective rule ("my accountant may read financial
+documents, excluding items my agent flags as private"). Can PDPP express this without
+changing the grant model or sync semantics, and what minimal seams should exist so a
+future extension can carry it?
+
+## Context
+
+Two designs were compared, independently by two analyses (this repo, 2026-08-05; an
+external ChatGPT session with its own red-team, 2026-08-07), converging on the same
+answer.
+
+Dynamic selectors: grants carry typed, monotonically narrowing constraints evaluated
+per request, possibly by a model. Rejected for Core: it converts the immutable grant
+from the complete authorization into a maximum bound, leaks excluded records through
+side surfaces (counts, aggregations, search, expansion), risks per-grant membership
+state at platform scale, and produces interoperability in name only when evaluator
+contracts differ.
+
+Derived streams: an evaluator materializes a subset stream upstream; the recipient
+receives an ordinary deterministic grant to that stream. Core is untouched, side
+surfaces are contained because excluded records are absent from the granted stream,
+and existing mutable-stream sync carries membership changes.
+
+Honest limit of the derived-stream design: the grant fully describes authorization
+only syntactically. Stream membership changes at the evaluator's discretion, so the
+indeterminism moves behind the stream name rather than disappearing. The real
+arguments are Core stability, side-channel containment, and reuse of existing sync.
+
+Evaluator placement is a deployment property, and the spec stays deployment-agnostic.
+Where the evaluator is co-located with the data (a personal server, or the provider
+itself), no second disclosure occurs. A remote evaluator is a second grantee and needs
+its own grant. An extension should state this trust consequence explicitly.
+
+## Stakes
+
+Low until an implementer wants it. The protocol-design payoff is flexibility: the same
+seams cover role changes, household membership, classification, and jurisdiction, well
+beyond AI evaluators.
+
+## Current Leaning
+
+1. Prototype subjective consent as a materialized derived stream. No Core change.
+2. One near-term semantic clarification worth owner review before or after the v0.1.0
+ freeze: on subset streams, a tombstone signals membership removal and does not
+ assert source deletion. This ambiguity exists today without any evaluator, and the
+ two claims carry different recipient obligations. One sentence, optionally a reason
+ field later.
+3. Reserve a namespaced critical-extension mechanism (an enforceable constraint an RS
+ must reject when unrecognized, distinct from ignorable capabilities). PR #77's
+ accepted proposal already moves in this direction; keep the reservation, publish no
+ selector grammar.
+4. Revisit a Dynamic Disclosure Profile only after derived streams fail against
+ several real use cases, and require: hard Core boundary, evaluator identity,
+ decisions tied to record versions, fail-closed behavior, and authoritative
+ resynchronization rules.
+
+## Promotion Trigger
+
+A second implementer asks for subjective or externally evaluated consent, or derived
+streams demonstrably fail a real deployment (per-recipient stream explosion, consent
+legibility complaints, or re-consent churn on stream redefinition).
+
+## Decision Log
+
+- 2026-08-07 — Captured from convergent internal (2026-08-05) and external analyses.
+ Owner direction: keep the protocol flexible and cohesive; no build planned; the
+ tombstone clarification is the only near-term action candidate.
diff --git a/design-notes/source-state-truth-2026-08-18.md b/design-notes/source-state-truth-2026-08-18.md
new file mode 100644
index 000000000..9165ba3d2
--- /dev/null
+++ b/design-notes/source-state-truth-2026-08-18.md
@@ -0,0 +1,328 @@
+# What a source's state should tell its owner
+
+**Status:** intake. Terminal model proposed, not owner-ratified. One case
+implemented as proof; the rest is unbuilt.
+**Date:** 2026-08-18
+
+## The evidence
+
+The owner's goal is every source green, honestly — green only if genuinely
+collecting, never by loosening a condition. Today `/sources` shows 23 sources in
+five display states, and four of the five lie in a way the owner can catch:
+
+| shown | reality |
+|---|---|
+| `○ Not measured · Fresh today` | claude-code holds 2,408,082 records and is collecting right now |
+| `○ Not measured · Freshness has not been measured yet` | Google Maps Timeline Import holds 299,248 records; the import finished and will never run again |
+| `◐ Needs refresh · Review: Resume schedule` | Chase is fine; an operator disabled its schedule to stop an OTP loop |
+| `⊘ Can't collect` + `Last successful refresh today` + 2,129 records | USAA, all three simultaneously true |
+
+Verified in production today:
+
+```
+claude-code local_device 2408082 stale (3 instances: 2.4M, 38k, 20k)
+codex local_device 1299535 fresh
+google-maps manual 299248 fresh
+whatsapp manual 120042 fresh
+usaa account 2129 fresh
+```
+
+Both `manual` sources have **zero rows in `run_history` and zero schedules**.
+There is nothing to run, and nothing that will ever run.
+
+The four failures have one shape. `isHealthyConditionSet`
+(`reference-implementation/runtime/connection-health.ts:1755`) collapses ten
+conditions to one boolean, and it requires three of them to be affirmatively
+`true`:
+
+```
+CollectionSucceeded === true
+SourceCoverageComplete === true
+Fresh === true
+```
+
+Three of those ten are also required *not* to be `false`, and `BacklogClear`
+must not be `error`. The predicate has exactly one caller, `classifyHealthy`,
+the last of fourteen ordered classification steps.
+
+The collapse is not the bug by itself. The bug is that the predicate cannot
+distinguish **"we don't know"** from **"the question doesn't apply here"**, so
+it treats both as not-green. A finished import can never produce a freshness
+proof, so it can never be green, no matter what the owner does.
+
+## What the code already knows
+
+This codebase already diagnosed this problem and solved it one layer too high.
+`ConnectionConditionStatus` (`connection-health.ts:107`) has four values, and
+the doc comment on the fourth is worth quoting:
+
+> `not_applicable` : the condition cannot apply to this connection at all,
+> because the evidence source it reads does not exist here. This is a *settled*
+> answer, not a pending one.
+>
+> `not_applicable` exists so the projection stops encoding certainty as doubt.
+
+And then, three lines later:
+
+> Classification treats `not_applicable` exactly as it treated the `unknown` it
+> replaces: it is never `true` and never `false`, so no headline state, axis, or
+> healthy-set predicate changes. **Only presentation changes.**
+
+That last sentence is the decision to revisit. The concept is right and already
+shipped; it was deliberately confined to cosmetics. Making it load-bearing in
+the healthy predicate is a smaller change than inventing anything new.
+
+Two more pieces already exist:
+
+- **`source_kind`** is a real column with a CHECK constraint over `account`,
+ `local_device`, `browser_collector`, `manual`
+ (`server/postgres-storage.ts:885`). The two "never measured" sources are
+ exactly the two rows with `source_kind = 'manual'`. The manifest already
+ carries enough to decide this — nothing new needs to be declared.
+- **`COVERAGE_UNKNOWN_STALE_COLLECTOR`** (`connection-health.ts:2716`) already
+ says *"This local collector build predates coverage evidence the server now
+ requires. Update the collector."* That is the honest sentence for the 2.4M-record
+ case. It exists, it is correct, and the sources list does not show it.
+
+## The dimensions, derived from the incidents
+
+Not a taxonomy invented for symmetry — each of these is a distinct axis because
+a real source varies on it independently of the others.
+
+1. **Is data arriving?** claude-code: yes, 2.4M records. Independent of whether
+ we can prove anything about it.
+2. **Is coverage provable?** Separate from (1). The stale collector emits data
+ but not the stores that prove coverage. Data flowing and proof complete are
+ genuinely orthogonal — that pair is the whole "Not measured · Fresh today"
+ contradiction.
+3. **Is currency meaningful, and if so, is it current?** Two questions, and the
+ model only asks the second. For a finished import the first answer is *no*,
+ which makes the second a category error.
+4. **Who can resolve the blocker?** Connector maintainer, owner, operator, or
+ external provider. USAA's detail page says "Connector code needs a fix"; the
+ list says "Can't collect". The useful sentence is the one not shown.
+5. **Is this source finished by design?** No state expresses it. There is no
+ terminal state at all.
+
+## Is a single green/not-green verdict the right shape?
+
+**Yes — keep the boolean, and fix which conditions are required versus
+inapplicable per source.** I considered the alternatives seriously.
+
+**Two-axis (data-flowing × proof-complete)** describes the claude-code case
+exactly, and it is the model I most wanted to adopt. I rejected it because it
+does not generalize: it has nothing to say about the operator-paused case or
+the provider-down case, so those would need a third and fourth axis, and the
+owner would be reading a vector. The owner's stated goal is *every source
+green*. A goal phrased as a scalar needs a scalar answer.
+
+**A state machine with a terminal Archived/Complete state** is the wrong
+primitive because completeness is not a state a source *transitions* into
+through the health pipeline — it is a property of the source's kind, known at
+creation. Google Maps Timeline Import was complete the moment its import
+finished. Modeling it as a reachable state implies a transition that never
+fires, and this codebase has already been bitten by exactly that: the
+`terminal_facts_historical` exclusion in
+`summary-evidence-projection-controller-2026-08-18.md` stranded three
+production rows behind an exit condition that was unreachable by construction.
+
+**Keeping the boolean, fixing the required set** wins because the boolean was
+never actually the problem. The problem is that "required" is currently a fixed
+list of ten conditions applied uniformly to every source, when some conditions
+are unanswerable for some source kinds. Green should mean *every condition that
+applies to this source is satisfied* — which is what the owner already thinks it
+means.
+
+**What it costs.** The predicate stops being a fixed list, so reading it no
+longer tells you the whole rule; you must also know which conditions the source
+kind marks inapplicable. That is real complexity and I am not going to pretend
+otherwise. The mitigation is that inapplicability is derived from `source_kind`
+and the manifest — both durable, both already there — rather than from
+per-source configuration an operator can get wrong. The failure mode to guard
+is a condition marked inapplicable when it is merely unproven, which would
+manufacture exactly the false green the owner refuses to accept. Hence the rule
+below.
+
+### The rule
+
+> A source is green when every condition that **applies** to it is satisfied.
+> `not_applicable` is satisfaction. `unknown` is not.
+>
+> A condition may be marked `not_applicable` only from durable evidence that the
+> question is meaningless for this source — never from the absence of an answer.
+
+The second sentence is the entire safety property. "We couldn't measure it" and
+"there is nothing to measure" must never collapse, or this design becomes the
+loosening the owner rejected.
+
+## The manual-import case
+
+**Settled: `Fresh` is `not_applicable`, not `false` and not `unknown`, for a
+source whose acquisition is complete.**
+
+Not `true`. Claiming a finished 2023 WhatsApp export is "fresh" replaces one lie
+with another. The honest statement is that freshness does not apply.
+
+Coverage is deliberately **not** relaxed. A completed import must still prove it
+ingested what it claimed. Unknown or gapped coverage keeps it out of green — the
+completeness declaration buys exemption from a freshness proof only.
+
+`source_kind = 'manual'` already carries this and is written at exactly one
+place (`server/routes/ref-manual-upload-draft-connection.ts:687`). The health
+input takes a new `acquisition: { complete: true }` evidence field rather than
+reading `source_kind` directly, matching how every other signal reaches
+`computeConnectionHealth` — the projection trusts caller-supplied evidence and
+never reads storage itself.
+
+## The operator-paused case
+
+**Settled: this is already correct in the health model and wrong only in the
+rendering. Do not touch the health model.**
+
+`classifyOwnerPaused` (`connection-health.ts:1356`) runs third of fourteen
+steps, before every failure classifier, and routes a disabled schedule to
+`idle` — not `degraded`, not `blocked`. `isDegradingCondition` explicitly
+excludes `ScheduleEligible`. The model already says an operator pause is not a
+source defect.
+
+The damage is done downstream: `idle` + disposition `owner_refresh_due` renders
+the amber pill `"Needs refresh"` (`runtime/rendered-verdict.ts:432`), and the
+console prefixes the CTA with a hardcoded `"Review: "`
+(`apps/console/.../sources/sources-view.tsx:338`). Amber plus "Review" reads as a
+defect for a source that has none.
+
+The fix belongs in the pill vocabulary — an operator-paused source is not amber
+— and I am explicitly not making it here, because `rendered-verdict.ts` is the
+same file the actor vocabulary below would rewrite, and both should land
+together.
+
+## The actor vocabulary
+
+"Can't collect" names no actor, so it cannot be acted on. Every state must name
+who resolves it. All five derive from evidence that already exists:
+
+| state | meaning | derived from |
+|---|---|---|
+| **Collecting** | green | the healthy predicate above |
+| **Complete** | finished by design, final | `acquisition.complete` (from `source_kind = 'manual'`) |
+| **Needs your login** | owner action | `CredentialsValid` false, `CREDENTIAL_REQUIRED` / `CREDENTIAL_REJECTED` |
+| **Needs a collector upgrade** | owner action, distinct from the above | `COVERAGE_UNKNOWN_STALE_COLLECTOR` — exists today, unshown |
+| **Needs a connector fix** | maintainer action, not the owner's | `terminalCoverageCta`, `audience: "maintainer"` |
+| **Paused by operator** | operator action, not a defect | `SCHEDULE_PAUSED` |
+| **Provider is down** | nobody's action; wait | `REMOTE_SURFACE_FAILED`, `EXTERNAL_TOOL_UNAVAILABLE` |
+
+Every row maps to a reason code already in `CONNECTION_CONDITION_REASONS`. This
+is a presentation vocabulary over existing evidence, not new derivation — which
+is why it is cheap, and why it is worth doing before anything more ambitious.
+
+Note "Needs a collector upgrade" is the sentence the owner most needs today: it
+covers 2.4M + 1.3M + 38k + 20k records currently labelled "Not measured", and
+the string already exists in the codebase.
+
+## Proof of concept
+
+The manual-import case, implemented end to end in
+`reference-implementation/runtime/connection-health.ts`. It is the cleanest test
+of the model because it is the case with no possible workaround — no owner
+action can ever make a finished import fresh.
+
+New test: `reference-implementation/test/connection-health-completed-import.test.ts`.
+
+**Fail before** (against unmodified `connection-health.ts`) — this reproduces
+the production symptom exactly:
+
+```
+✖ a completed one-time import reports Fresh as not_applicable, not unknown
+ + actual 'unknown' - expected 'not_applicable'
+✖ a completed one-time import is healthy without a Fresh=true proof
+ 'idle' !== 'healthy'
+```
+
+**Pass after** — 6/6:
+
+```
+✔ a completed one-time import reports Fresh as not_applicable, not unknown
+✔ a completed one-time import is healthy without a Fresh=true proof
+✔ a completed import still needs complete coverage to be healthy
+✔ a completed import with a terminal coverage gap is not healthy
+✔ acquisition completeness does not leak into recurring sources
+✔ a recurring source that is genuinely stale is never rescued by this path
+```
+
+The last three tests are the ones that matter. They prove the change cannot
+manufacture a false green: coverage is still required, and a recurring source
+without the completeness declaration behaves exactly as before.
+
+The change is 90 lines, of which the load-bearing edit is **one**:
+
+```
+- conditionIsTrue(conditions, "Fresh") &&
++ conditionIsSettledSatisfied(conditions, "Fresh") &&
+```
+
+where `conditionIsSettledSatisfied` accepts `true` or `not_applicable`, and
+pointedly not `unknown`. The other 89 lines are the new
+`ConnectionAcquisitionEvidence` type, one branch in `freshCondition`, and one
+branch in `collectionSucceededCondition` that mirrors the existing
+`localDeviceCollection.verdict` precedent for sources that legitimately write no
+spine run.
+
+**Regression evidence:** 305 existing tests pass unchanged —
+`connection-health.test.ts` 151/151, `connection-health-acceptance.test.ts`
+70/70, `rendered-verdict.test.ts` 84/84 — and `tsc --noEmit` is clean.
+
+Not wired to the read path. `projectConnectorSummaryConnectionHealth` in
+`server/ref-control.ts` would need to pass `acquisition` from the instance's
+`source_kind`, and that file is being actively edited by another agent. The
+runtime model is proven; the wiring is one line in a file I did not touch.
+
+## What I deliberately left alone
+
+- **The other nine conditions stay required.** Only `Fresh` gained a
+ not-applicable path, and only for one source kind. Extending this to coverage
+ is where a false green would come from, so it needs its own evidence and its
+ own argument.
+- **`classifyOwnerPaused` and the classification order.** Already correct. The
+ paused-source damage is in the pill vocabulary, not the model.
+- **`rendered-verdict.ts`.** The actor vocabulary rewrites it; the paused-pill
+ fix rewrites it; doing either piecemeal now means doing it twice.
+- **`hasAffirmativePassiveRecoveryEvidence`** (`connection-health.ts:1751`) —
+ the scheduler's passive-recovery authority. It independently requires
+ `axes.freshness === "fresh"` and `Fresh === "true"`. I did not touch it: a
+ completed import has no schedule and no next attempt, so it can never reach
+ that path, and relaxing a scheduler predicate to fix a display problem would
+ be scope I cannot justify. It is, however, the second place the same
+ fixed-required-list assumption lives, and it will need the same treatment if
+ this model is adopted.
+- **The `dirty`/projection layer.** Orthogonal, and owned by
+ `summary-evidence-projection-controller-2026-08-18.md`.
+- **Production data.** Nothing deployed, nothing committed, no database written.
+
+## Cost and risk
+
+**What breaks if the predicate changes.** Less than feared — `isHealthyConditionSet`
+is private with exactly one caller. The blast radius is `classifyHealthy`, and
+from there whatever reads `state === "healthy"`. The real risk is not
+mechanical; it is that every future `not_applicable` is a potential false green.
+That is why the rule above forbids deriving inapplicability from a missing
+answer, and why the proof-of-concept spends half its tests on that boundary.
+
+**The honest residual risk.** `not_applicable` is now load-bearing, so a bug
+that marks a condition inapplicable is a bug that turns a source green. Before
+this change such a bug was cosmetic. That is a genuine increase in the cost of
+being wrong, accepted because the alternative is a permanently dishonest display
+on 420k records that are complete and correct.
+
+**Confidence.** That the manual-import fix is right: high — it is proven by
+test, and the case admits no other honest answer. That the same shape extends
+cleanly to the stale-collector and paused cases: moderate — the evidence exists
+and the vocabulary maps, but neither is implemented, and the paused case needs a
+pill-vocabulary decision I did not make.
+
+## Related
+
+`upstream-disclosure-window-2026-08-17.md` names the same failure from the other
+side — "Imports have no upstream. They need to be first-class *not applicable*,
+not zero — the same failure this codebase already has with 'Not measured'." That
+note wanted this primitive and could not assume it. This note builds it for
+freshness; the boundary case will want it too.
diff --git a/design-notes/summary-evidence-projection-controller-2026-08-18.md b/design-notes/summary-evidence-projection-controller-2026-08-18.md
new file mode 100644
index 000000000..1305ebf24
--- /dev/null
+++ b/design-notes/summary-evidence-projection-controller-2026-08-18.md
@@ -0,0 +1,207 @@
+# One generation row per connection, not a generic job queue
+
+**Status:** intake. Terminal design proposed by an independent reviewer, not yet
+owner-ratified. Deliberately NOT scoped into the corrective branch that prompted it.
+**Date:** 2026-08-18
+
+## Why this note exists
+
+Five starvation bugs were found in the bounded maintenance sweep inside about
+twenty-four hours. All five have the same shape: **work that cannot progress
+consumes a shared budget, and work that can progress never runs.**
+
+1. **Checkpoint floor.** The fold read from `min(checkpoint)` across participants.
+ Three rows sat at checkpoint 0, so the floor was 0 against a 1.44M-event log.
+ Every 2s pass restarted at 0, read zero qualifying events, and repeated.
+2. **Zero-vs-null.** The first fix guarded `null`. The rows stored a literal `0`.
+3. **Phase starvation.** "Missing" discovery consumed the whole budget before
+ "generic" — the only path that classifies a row as dirty — could run.
+4. **Post-deadline skip.** Discovery expired the deadline, so the repair loop
+ skipped every candidate. 16 classified, 16 skipped, 0 repaired, on an idle
+ database.
+5. **Permanent exclusion.** The fix for #2 excluded `terminal_facts_historical`
+ rows at checkpoint 0 from the fold. Its stated exit condition was unreachable:
+ nothing marks such a row dirty, and the checkpoint advances only via the fold
+ the row is excluded from. Three production rows stranded, one an active
+ connection that could not recover.
+
+Number 5 is the one that matters most for design purposes. It was introduced *by*
+a starvation fix, written immediately after fixing the previous one, and it
+converted a livelock into a permanent exclusion. That is not an attention failure.
+It is what happens when fairness is an emergent consequence of phase order, cursor
+position, and exception paths rather than an explicit durable invariant.
+
+## The reviewer's verdict
+
+> Replace the scheduling model; keep the shipped fix only as incident mitigation.
+
+Confidence that the current model produces more bugs of this family: **0.96**.
+Confidence that a small durable reconcile queue is the right terminal shape: **0.90**.
+
+Crucially, the preferred design is **not** a generic durable job framework with
+fold, missing-repair, generic-repair, and audit job types. That would preserve the
+task-kind zoo that produced the bugs. It is:
+
+> A level-triggered, generation-based projection controller keyed by connection.
+
+## The design
+
+One durable projection-state row per connection:
+
+```
+connector_instance_id
+desired_generation
+applied_generation
+target_event_seq
+folded_event_seq
+applied_contract_version
+dirty_since
+next_attempt_at
+last_attempt_at
+attempt_count
+last_outcome
+```
+
+Every canonical change that could affect a connection's summary increments
+`desired_generation`, **preferably in the same transaction as the change**.
+Multiple changes coalesce into the same row — the row *is* the durable
+deduplicating queue entry. There is no separate queue table to keep in sync.
+
+One bounded, idempotent operation reconciles a connection:
+
+```
+reconcile(connection_id):
+ snapshot desired_generation
+ fold at most N indexed events for this connection
+ persist fold progress and yield if more remain
+ read bounded canonical facts
+ compute and write the complete desired summary
+ set applied_generation to the generation that was reconciled
+```
+
+If the connection changes mid-reconciliation, `desired_generation` advances past
+`applied_generation`, so it stays eligible automatically. A deferred or failing
+connection gets `next_attempt_at`/backoff and cannot permanently hold first
+position.
+
+## What this deletes
+
+These concepts stop existing, and with them the bugs they produced:
+
+- "missing" versus "generic" discovery phases
+- the shared minimum participant checkpoint
+- the rotating page cursor
+- process-local phase alternation
+- the first-candidate deadline exemption
+
+Missing, dirty, and code-version-stale collapse into two conditions:
+
+```
+applied_generation < desired_generation
+applied_contract_version != CURRENT_VERSION
+```
+
+**Both of those conditions would have prevented a bug this codebase actually
+shipped.** `applied_generation < desired_generation` is derived, not remembered,
+so bug #5 is unrepresentable — a row cannot be stranded by a predicate that
+forgot to let it back in. And `applied_contract_version != CURRENT_VERSION` is
+exactly the check that was missing when production ran fold logic version 5 while
+every committed branch was at 4: a clean build shipped a binary older than its own
+data, the version guard failed closed, and 26 of 28 evidence rows went unreadable
+with no signal beyond a fleet of grey pills.
+
+## Bounds are still required
+
+The controller shape does not remove the need for hard bounds:
+
+- bounded indexed event pages
+- PostgreSQL `statement_timeout` and `lock_timeout`
+- bounded SQLite query shapes, or interruption where the driver allows it
+- a soft pass admission deadline
+- a maximum number of units per wake
+
+The reviewer's P1-2 stands independently of the redesign: the current 2000ms
+`maxDurationMs` is a cooperative admission hint, not a wall-clock or database
+occupancy bound. Measured on production *after* removing an unrelated CPU
+contention problem, a pass still reported `repair_duration_ms: 5322` with skipped
+candidates. If a unit cannot be hard-cancelled, it does not belong inside a
+claimed 2-second maintenance loop.
+
+## The three invariants
+
+The reviewer rejected the single-invariant framing ("a pass that finds candidates
+must repair at least one") as insufficient — it conflates repair success with
+scheduling and does not bound a pathological unit. Three independent, separately
+testable invariants are required:
+
+**A. Bounded yield.** No operation may run between durable yield points unless its
+worst-case work is bounded or it has enforceable cancellation.
+
+**B. Monotonic outcome.** Every attempted work item must durably advance a cursor,
+complete, defer with a future eligibility time, back off, or terminate. An
+identical no-op retry cannot repeat forever.
+
+**C. Bounded fairness.** Every continuously eligible item and nonempty task class
+must receive an attempt within a defined number of scheduler turns, **across
+process restarts**.
+
+Invariant C is the one the current implementation cannot satisfy: fairness lives
+in module-local variables (`nextDirtyAfterId`, `nextFirstObservationPhase`) whose
+convergence bound vanishes on restart.
+
+## The audit becomes a backstop
+
+The hot path should not recompute fleet-wide aggregates. Expensive facts such as
+record counts should be maintained incrementally where practical and verified by a
+slower paged audit. The periodic sweep stops being the primary repair engine and
+becomes what it should have been: a detector of missed invalidations that marks
+connections behind. Orphan detection belongs there too, not in the latency-
+sensitive loop.
+
+## Scope discipline
+
+The reviewer was explicit that this redesign should **not** be added to the
+corrective branch. That branch finishes a bounded list:
+
+- advance fairness from the last *attempted* candidate, not the last fetched
+ page member — **done** (`ab28764f2`)
+- repair `terminal_facts_historical` re-entry and boundary stamping — **partially
+ done** (`078b72e3a` prevents new stranding; already-stranded rows still need a
+ one-time re-entry path, in progress)
+- the four adversarial tests, plus below-page-limit, above-page-limit, and
+ restart cases — **partially done**, restart case outstanding
+- hard per-query/per-unit bounds, and an honest name for the pass deadline
+- no-progress telemetry and alerting
+- durable fairness, or fairness derived from durable per-item attempt state —
+ **deliberately deferred to this note's design**, since it needs a schema change
+ and a different discovery query shape
+
+Durable fairness is the item that most clearly belongs here rather than there:
+implementing it in the current model means adding per-item attempt columns and
+reshaping the discovery query, which is most of the projection-state row anyway.
+Doing it twice would be waste.
+
+## Open questions
+
+- Does `desired_generation` increment in the same transaction as every canonical
+ change, or is a trigger acceptable? Same-transaction is stated as preferred;
+ the cost is touching every writer.
+- What is the migration path for the existing `connector_summary_evidence` rows,
+ including the three currently stranded at checkpoint 0?
+- Does the audit backstop need its own fairness guarantee, or is a slow full
+ rotation sufficient given it is no longer the primary repair path?
+- SQLite parity: `statement_timeout` has no direct equivalent. Is a bounded query
+ shape provably sufficient, or is driver-level interruption required?
+
+## Provenance
+
+Independent design review of the bounded maintenance sweep, 2026-08-18, conducted
+against `sweep-design-review-20260818.zip` (the four starvation bugs, the shipped
+minimum-one fix, and the supporting patches). The reviewer retracted one
+production measurement from that packet after it was shown to be confounded by an
+uncapped embedding transformer competing with PostgreSQL — the code-level
+counterexamples and the structural conclusion were unaffected.
+
+Related: `connector-sidecar-packaging-2026-08-17.md` and
+`upstream-disclosure-window-2026-08-17.md` share the shape of a component whose
+correctness depends on a peer's version with nothing verifying the pair.
diff --git a/design-notes/upstream-disclosure-window-2026-08-17.md b/design-notes/upstream-disclosure-window-2026-08-17.md
new file mode 100644
index 000000000..1a62e931b
--- /dev/null
+++ b/design-notes/upstream-disclosure-window-2026-08-17.md
@@ -0,0 +1,129 @@
+# Surfacing a shrinking upstream disclosure window
+
+**Status:** intake. Not an OpenSpec change; no requirement is proposed yet.
+**Date:** 2026-08-17
+
+## The observation that prompted this
+
+PDPP holds two H-E-B orders for the owner, both captured in the same scan on 2026-07-15:
+
+| id | date | status | total |
+|---|---|---|---|
+| `HEB20169324473` | 2023-08-09 | Order canceled (`SHORTED`) | $293.98 |
+| `HEB20607368035` | 2023-08-19 | Delivered | $382.67 |
+
+H-E-B now displays only the second one to that account.
+
+The owner's inference is the load-bearing one: **PDPP scrapes what the account UI shows, so if it
+captured the first order, H-E-B was showing it then.** The record did not move. The provider's
+disclosure did.
+
+That is the product working exactly as intended — PDPP holds data the provider no longer surfaces.
+But the app cannot say so. Its dashboard shows "2 orders" today and would show "2 orders" after a
+fresh run that finds nothing. Something important happened and the product is silent about it.
+
+For a tool whose purpose is outrunning deletion, "the source is disclosing less than it used to"
+is not noise. It is the alarm.
+
+## What the system already has
+
+- Per-source checkpoint: `{"checkpoint": "2023-08-19", "fingerprints": {...}}`
+- `fetched_at` on every record
+- Scan-termination reasons in the H-E-B connector distinguishing `pagination_exhausted` from
+ `selector_drift`, `pagination_metadata_absent`, `source_auth_or_challenge`
+ (`packages/polyfill-connectors/connectors/heb/index.ts:320-335`)
+
+What is missing is durable evidence of *why a scan stopped* and *how far back it reached*. For
+`cin_c875ca3ec8b6ce2c283a4288` no such evidence was stored, and there is no run history at all —
+so today we cannot distinguish "H-E-B showed us everything" from "we hit a wall."
+
+## The proposed primitive: an observed boundary, not availability
+
+Per successful run, one value: **the oldest item the provider displayed**, recorded only when the
+scan proves it reached the end of its range. Comparing across runs yields a moving frontier.
+
+The product could then say something entirely factual:
+
+> H-E-B showed orders back to 2023-08-09 in July 2026; today it goes back to 2023-08-19.
+> 1 stored order is no longer displayed.
+
+Every clause is an observation. The conclusion — *their retention window is closing* — is the
+owner's to draw, and they can draw it, because they know whether they shopped in 2024.
+
+## Hard constraints
+
+**1. A moving boundary is information about the provider, never an annotation on a record.**
+This is the one absolute. Nothing in this design may mark a stored record deleted, stale, or
+suspect. Deliberately: no connector in the fleet emits deletions today — verified, zero occurrences
+of a delete/tombstone emission across `packages/polyfill-connectors/connectors/` — so provider
+erasure structurally cannot propagate into the owner's copy. Introducing a path that annotates
+records based on absence would give that up for a signal that is frequently wrong.
+
+**2. Absence is evidence only when the scan proves it covered the range.**
+A run ending `pagination_exhausted` makes absence meaningful. A run ending `selector_drift` or
+`source_auth_or_challenge` makes it meaningless. Without a stored termination reason there is no
+signal, and the correct output is "unknown."
+
+**3. Do not encode provider retention policies.**
+"H-E-B keeps 18 months" is undocumented, changes silently, and varies by account and region. A
+wrong constant produces confident lies. The empirically observed window is strictly better: it is
+measured, not asserted, and it survives the provider changing policy without telling anyone.
+
+## Edge cases that make a *general* solution hard
+
+- **Silent auth degradation.** A session expiring into a logged-out-but-200 view returns no items,
+ reports pagination exhausted, and yields a boundary of nothing — indistinguishable from a total
+ purge. The nastiest false positive, and the reason constraint 1 is absolute.
+- **Not every source has an ordering.** Contacts have no time axis. Notion pages are edited, so
+ recency is not age. Gmail's "oldest visible" depends on the query. A single scalar frontier fits
+ perhaps half the fleet and produces meaningless numbers for the rest.
+- **Scope change mimics retention.** Leaving a Slack channel removes its history from view; a plan
+ downgrade hides older data; a provider splitting history by store looks like shrinkage. No
+ boundary comparison can tell these from deletion.
+- **Retention is rarely uniform.** Amazon keeps orders but drops invoice PDFs; Slack's free tier
+ hides messages but keeps files; Gmail retains mail but purges trash at 30 days. One per-source
+ boundary cannot express "text kept, attachments gone," and per-stream boundaries multiply the
+ connector burden.
+- **Imports have no upstream.** Google Maps and the WhatsApp exports are one-shot files. They need
+ to be first-class *not applicable*, not zero — the same failure this codebase already has with
+ "Not measured" (see `add-honest-uncollected-source-states`).
+- **Contiguity assumptions are false.** "Orders are sequential, so a gap means deletion" breaks on
+ a month with no shopping. The owner's own H-E-B data is two orders ten days apart and then
+ nothing — a gap that is a fact about their life, not their provider.
+
+## Recommended shape
+
+**Opt-in per connector, not a universal contract.** Connectors with a genuine monotonic frontier
+and a provable pagination stop — orders, transactions, messages — report a boundary. Everything
+else reports nothing, and nothing is fine. A signal present on eight connectors and honest beats
+one present on twenty-four and wrong.
+
+**Runtime owns the bookkeeping.** The author declares the scanned range and the termination reason;
+the runtime derives the boundary and its movement. A connector must never assert that something is
+gone.
+
+**Fail closed.** No termination reason means no boundary claim. A lazy or broken connector produces
+"unknown," never a false purge.
+
+**Machine reports, human interprets.** State the observation; leave the conclusion to the owner.
+That is not a cop-out — it puts the inference where the context actually lives. The owner knew
+instantly that PDPP could not have collected an invisible order; no rule authored here would have
+encoded that.
+
+## Honest caveat on feasibility
+
+This is reasoned from one connector and a day of code reading. The fleet has not been surveyed for
+how many connectors could actually satisfy constraint 2. Today's evidence argues for pessimism:
+Slack emitted duplicate coverage for every multi-archive run since inception, and a Codex source
+was rendered unmeasurable by a single stale store name. If the existing, simpler coverage contract
+is unevenly met, a boundary contract will be too. Expect "unknown" from a meaningful fraction of
+the fleet for a long while — and prefer that to false confidence.
+
+## Relationship to existing work
+
+Same failure shape as several open items: a derived value that goes quietly stale because nothing
+watches whether its source changed. Compare `add-projection-contract-versioning` (input checkpoints
+cannot see a formula change), `make-local-coverage-tolerate-unexpected-stores` (a stale store name
+discards a valid proof), and `add-durable-connection-account-identity` (identity derived from a
+provisional binding key). The recurring principle is worth stating once, somewhere durable:
+**derive nothing durable from a value that may be provisional, and watch anything you do derive.**
diff --git a/docker-compose.yml b/docker-compose.yml
index ca9a3ebdb..85aa387d7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -26,6 +26,17 @@ services:
# below — the env guard exists precisely to refuse boot if this policy
# (or an equivalent one) is missing.
restart: unless-stopped
+ # An unconstrained container reports the WHOLE host to
+ # effectiveCpuCount()/effectiveMemoryBudgetBytes(), so
+ # resolveEmbeddingConcurrency() sizes the transformer pool against every
+ # core on the machine. On a 24-core host that derives workLimit=8 x
+ # intraOpNumThreads=3 = 24 native ONNX threads, which then contend with
+ # postgres and the web service in this same stack. The sizing math is
+ # correct; it was being handed a budget this service does not actually own.
+ # Declare the share explicitly and let operators raise it on dedicated
+ # hardware.
+ cpus: ${PDPP_REFERENCE_CPUS:-4}
+ mem_limit: ${PDPP_REFERENCE_MEM_LIMIT:-4g}
environment:
AS_PORT: "7662"
RS_PORT: "7663"
@@ -179,6 +190,14 @@ services:
# the storage backend. To re-run the migration tool, do it on the
# host with the SQLite file at `./packages/polyfill-connectors/
# .pdpp-data/pdpp.sqlite` and the host-published Postgres port.
+ # Browser profiles live under PDPP_BROWSER_PROFILE_ROOT
+ # (/var/lib/pdpp/browser-profiles, baked into the image). Without this
+ # mount they sit in the container's ephemeral layer and are destroyed on
+ # every restart, so every browser-backed connector -- ChatGPT, Reddit,
+ # Amazon, Chase -- demands a fresh interactive login after each deploy.
+ # deploy/docker/docker-compose.yml already mounts this; the owner stack
+ # did not, and twelve deploys in one day cost the owner that many logins.
+ - pdpp-data:/var/lib/pdpp
- pdpp-transformers:/var/cache/pdpp/transformers
- pdpp-home:/root/.pdpp
# File connector imports. Override the host-side paths with
@@ -225,6 +244,33 @@ services:
# `PDPP_POSTGRES_BIND_HOST` AND set non-default credentials.
ports:
- "${PDPP_POSTGRES_BIND_HOST:-127.0.0.1}:${PDPP_POSTGRES_PORT:-55432}:5432"
+ # Defaults are sized for a small demo database, not for the ingest volume a
+ # real instance reaches. Two symptoms this addresses, both observed live:
+ # thousands of "checkpoints are occurring too frequently" warnings during
+ # bulk ingest (max_wal_size too small), and autovacuum taking 19+ minutes on
+ # a 4.2M-row search index while starving the maintenance sweep
+ # (maintenance_work_mem at the 64MB default). Override per deployment.
+ # Docker gives a container 64MB of /dev/shm by default. Postgres uses
+ # shared memory for parallel query workers, and on a multi-million-row
+ # table that is not enough -- a parallel VACUUM or aggregate fails with
+ # "could not resize shared memory segment ... No space left on device",
+ # which reads as a disk problem and is not one. Observed on a 5.5M-row
+ # records table.
+ shm_size: ${PDPP_POSTGRES_SHM_SIZE:-1g}
+ command:
+ - postgres
+ - -c
+ - max_wal_size=${PDPP_POSTGRES_MAX_WAL_SIZE:-8GB}
+ - -c
+ - min_wal_size=${PDPP_POSTGRES_MIN_WAL_SIZE:-1GB}
+ - -c
+ - checkpoint_timeout=${PDPP_POSTGRES_CHECKPOINT_TIMEOUT:-15min}
+ - -c
+ - shared_buffers=${PDPP_POSTGRES_SHARED_BUFFERS:-512MB}
+ - -c
+ - effective_cache_size=${PDPP_POSTGRES_EFFECTIVE_CACHE_SIZE:-1536MB}
+ - -c
+ - maintenance_work_mem=${PDPP_POSTGRES_MAINTENANCE_WORK_MEM:-512MB}
volumes:
- pdpp-postgres-data:/var/lib/postgresql/data
healthcheck:
@@ -267,6 +313,7 @@ services:
- "${PDPP_WEB_PORT:-3002}:3000"
volumes:
+ pdpp-data:
pdpp-transformers:
pdpp-home:
pdpp-postgres-data:
diff --git a/docs/design-system/ink-carbon/project/explorer/data.js b/docs/design-system/ink-carbon/project/explorer/data.js
index 2a21d68f2..97ed0f0b1 100644
--- a/docs/design-system/ink-carbon/project/explorer/data.js
+++ b/docs/design-system/ink-carbon/project/explorer/data.js
@@ -1379,7 +1379,7 @@ const deepGmail = [
{
id: rid("gm"),
thread_id: "th8",
- from: "Stanford Alumni ",
+ from: "Northcrest Alumni ",
to: ["the owner@example.com"],
subject: "Class of 2014 — 5-year reunion",
snippet: "Save the date: October 19, 2019. Memorial Auditorium.",
diff --git a/docs/design-system/ink-carbon/project/explorer/data.ts b/docs/design-system/ink-carbon/project/explorer/data.ts
index 75e9c5270..65b525bcb 100644
--- a/docs/design-system/ink-carbon/project/explorer/data.ts
+++ b/docs/design-system/ink-carbon/project/explorer/data.ts
@@ -1413,7 +1413,7 @@ const deepGmail = [
{
id: rid("gm"),
thread_id: "th8",
- from: "Stanford Alumni ",
+ from: "Northcrest Alumni ",
to: ["the owner@example.com"],
subject: "Class of 2014 — 5-year reunion",
snippet: "Save the date: October 19, 2019. Memorial Auditorium.",
diff --git a/docs/design/media-acquisition.md b/docs/design/media-acquisition.md
index b030f929a..e326adb6a 100644
--- a/docs/design/media-acquisition.md
+++ b/docs/design/media-acquisition.md
@@ -157,5 +157,5 @@ written, and Immich is the gate.
---
-Signed-off-by: Tim Nunamaker
+Signed-off-by: PDPP Maintainers
Assisted-by: AI
diff --git a/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md b/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md
index df0a8d4ac..1cb090855 100644
--- a/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md
+++ b/docs/handoffs/pdpp-onto-remote-surface-minimal-migration-plan-2026-07-17.md
@@ -94,7 +94,7 @@ neko-specific input seam). Default: **defer** to stay inside the 1–2h box.
is a behavior change — STOP, flag it, do not proceed).
3. `pnpm --filter types:check` green.
4. Full console test suite green.
-5. Committed as `tnunamak@gmail.com`, one lane per commit, message states which local code
+5. Committed as `maintainers@example.com`, one lane per commit, message states which local code
was deleted and which RS primitive replaced it.
### Hard stops (escalate, do not improvise)
diff --git a/docs/operator/local-collector-runbook.md b/docs/operator/local-collector-runbook.md
index 2ab6646bc..9d5cc1144 100644
--- a/docs/operator/local-collector-runbook.md
+++ b/docs/operator/local-collector-runbook.md
@@ -1,8 +1,10 @@
-# Local Collector Runbook (Claude Code / Codex)
+# Local Collector Runbook (Claude Code / Codex / Signal)
Status: reference-experimental operator surface. Not PDPP Core or Collection Profile protocol.
-This is the single-page operator runbook for running Claude Code and Codex local collectors against a PDPP Docker reference deployment, with resumable connector state. It supersedes the bare `bin/local-device-exporter.ts` flow in `reference-implementation/docs/local-device-exporter.md` — that script remains as a compatibility shim but does not participate in STATE sync.
+This is the single-page operator runbook for running Claude Code, Codex, and Signal Desktop local collectors against a PDPP Docker reference deployment, with resumable connector state. It supersedes the bare `bin/local-device-exporter.ts` flow in `reference-implementation/docs/local-device-exporter.md` — that script remains as a compatibility shim but does not participate in STATE sync.
+
+Steps 1–5 below apply to every local-collector connector, including Signal — swap `--connector claude_code` for `--connector signal` in Steps 2 and 4. Signal carries one additional prerequisite (the `sigtop` sidecar binary) and one structural constraint (it only runs on the owner's own logged-in desktop session) that Claude Code/Codex do not have; see "Signal Desktop prerequisites" below before Step 1.
## What you are setting up
@@ -40,6 +42,95 @@ State is authoritative on the server. Before each connector pass the runner fetc
current published release. See
`docs/reference/local-collector.md`§"Deployment Posture: Published vs Dev".
+## Signal Desktop prerequisites
+
+Signal is a **local-collector-only** connector: unlike Slack (`slackdump`) or
+Google Messages (`gmcli`), its sidecar tool cannot ship inside the Core
+server image. The manifest declares a `desktop_session` runtime binding, and
+the engine resolves that to `local_device` placement and refuses server-side
+placement outright. This is by design, not a temporary gap — see
+`design-notes/connector-sidecar-packaging-2026-08-17.md` for the full
+evidence trail (a container hits four successive failures: missing file,
+no D-Bus session, uid mismatch, AppArmor denial).
+
+**Why:** Signal Desktop's SQLCipher database key is stored encrypted in
+`~/.config/Signal/config.json` (`encryptedKey`) and unwraps only through a
+session-bound OS keyring — KWallet/GNOME Keyring (`libsecret`) on
+Linux, Keychain on macOS, DPAPI on Windows. Check which backend your
+Signal Desktop uses:
+
+```bash
+cat ~/.config/Signal/config.json
+```
+
+- `"safeStorageBackend": "kwallet6"` (or `gnome_libsecret`, etc.) — the
+ key only unwraps inside the owner's own logged-in desktop session. Run the
+ collector directly on that machine, logged in as that user, outside any
+ container.
+- `"safeStorageBackend": "basic_text"` — the key is stored unwrapped.
+ A server-side path is technically possible in this configuration, but it
+ is not what this connector is built or tested for; treat it as a
+ documentation note, not a supported deployment target.
+
+**Install `sigtop`** (github.com/tbvdm/sigtop, ISC license) — the CLI
+this connector spawns as an arms-length subprocess to decrypt and read
+Signal Desktop's database, the same "sidecar" pattern the `slack` connector
+uses for `slackdump`:
+
+```bash
+# Linux: pkg-config needs the libsecret headers to build the safeStorage
+# unwrap. If you cannot install system-wide (no root), download the .deb
+# with apt-get download (works without sudo) and extract it locally, then
+# point PKG_CONFIG_PATH/CGO_LDFLAGS at the extracted tree instead of
+# installing system-wide.
+sudo apt install libsecret-1-dev pkg-config # Debian/Ubuntu
+# or: dnf install libsecret-devel pkgconf-pkg-config # Fedora
+
+GOBIN=~/.local/bin go install github.com/tbvdm/sigtop/cmd/sigtop@latest
+```
+
+Note the real import path is `github.com/tbvdm/sigtop/cmd/sigtop` —
+`go install github.com/tbvdm/sigtop@latest` (without `/cmd/sigtop`) fails
+with "module ... found, but does not contain package ...".
+
+**Verify the binary actually works** — `sigtop -v` and `sigtop version`
+are NOT valid subcommands (there is no version flag at all); use a real
+subcommand instead:
+
+```bash
+sigtop check-database # fast SQLCipher integrity check against the local DB;
+ # exits 0 with no output on success
+```
+
+If Signal Desktop is running, close it first — sigtop needs unlocked
+read access to `db.sqlite`/`db.sqlite-wal`/`db.sqlite-shm`.
+
+**Point the collector at `sigtop`** if it is not on `PATH` (a custom
+`GOBIN`, a non-standard install location, etc.):
+
+```bash
+export SIGTOP_BIN=/absolute/path/to/sigtop # default: "sigtop" on PATH
+```
+
+`resolveSigtopBin`/`runSigtop` in `packages/polyfill-connectors/connectors/signal/index.ts`
+implement this resolution; a missing binary fails fast with a message
+naming both the install command and the `SIGTOP_BIN` override, rather than
+an opaque `ENOENT`.
+
+**Building `@pdpp/local-collector` from source for Signal support**: if your
+installed `@pdpp/local-collector` predates Signal (`advertise` does not list
+`signal` under `bundled_connectors`), rebuild from a checkout that has the
+connector. The collector's `tsconfig.build.json` `include` list is the
+package's actual shipping manifest — a connector must be listed there
+(and its test-only fixture files that pull in `better-sqlite3` must be
+listed under `exclude`, the same way `imessage/fixtures.ts` and
+`signal/fixtures.ts` are) or its compiled `.js` never reaches the tarball,
+even if the connector is registered in `collector-registry.ts` and shows up
+under `advertise`. A collector shipping only `collector-definition.js` for a
+connector (no `index.js`) will advertise it but fail at spawn time with
+`spawn tsx ENOENT` (falling back to running uncompiled `.ts` source, which
+needs a `tsx` binary this package deliberately does not depend on).
+
## Step 1 — Confirm collector runtime capabilities
On the host with Claude/Codex data:
@@ -55,10 +146,14 @@ Expected output (capabilities may grow):
"runtime": "collector",
"bindings": ["network", "filesystem", "local_device"],
"collector_protocol_version": "1",
- "bundled_connectors": ["claude_code", "codex"]
+ "bundled_connectors": ["claude_code", "codex", "google_takeout", "imessage", "apple_photos", "google_messages", "signal"]
}
```
+If `signal` is missing from `bundled_connectors`, see "Signal Desktop
+prerequisites" above — your installed build predates Signal support
+and needs rebuilding from a checkout that has it.
+
Both `claude_code` and `codex` require the `filesystem` binding, which the collector advertises by default. The published package intentionally does not bundle the `browser` binding; browser-bound connectors stay in the monorepo until each has its own publishability review. A connector that requires a binding the collector does not advertise will fail before spawn with `runtime_capability_mismatch` — you do not need to discover that empirically.
## Step 2 — Mint an enrollment code
@@ -67,7 +162,7 @@ In a browser, open `/device-exporters` on the reference deployment, signed in as
Use the "Create enrollment code" form:
-- Connector id: `claude_code` (or `codex`).
+- Connector id: `claude_code` (or `codex`, `signal`, ...).
- Local binding: a stable name like `personal-laptop` or `ci-runner-eu-1`. Used by the server to namespace the connection id. Existing server responses still expose this compatibility field as `source_instance_id`.
- Display name: optional, propagates as the device label.
@@ -129,7 +224,7 @@ PDPP_CONNECTION_ID=si_... \
--connector claude_code
```
-Swap `--connector claude_code` for `codex` to ingest Codex CLI history/skills/etc.
+Swap `--connector claude_code` for `codex` to ingest Codex CLI history/skills/etc., or for `signal` to ingest Signal Desktop messages/conversations/reactions/attachments (see "Signal Desktop prerequisites" above first — `sigtop` must be installed and Signal Desktop must not be running).
Live progress prints to stderr as the connector finds records (phase, running
counts, and a final summary), so a large local archive no longer looks stuck
diff --git a/docs/reference/stream-evidence-inventory.md b/docs/reference/stream-evidence-inventory.md
index 61a891a78..60d218723 100644
--- a/docs/reference/stream-evidence-inventory.md
+++ b/docs/reference/stream-evidence-inventory.md
@@ -142,8 +142,8 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
-| timeline_points | checkpoint_window | manual_as_of | — | true | — | — |
-| timeline_segments | checkpoint_window | manual_as_of | — | false | — | — |
+| timeline_points | snapshot_import_receipt | manual_as_of | — | false | — | — |
+| timeline_segments | snapshot_import_receipt | manual_as_of | — | false | — | — |
## polyfill/google-maps-data-portability
@@ -273,6 +273,15 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| --- | --- | --- | --- | --- | --- | --- |
| orders | checkpoint_window | manual_as_of | — | true | — | — |
+## polyfill/signal
+
+| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
+| --- | --- | --- | --- | --- | --- | --- |
+| messages | snapshot_import_receipt | manual_as_of | — | true | — | — |
+| conversations | snapshot_import_receipt | manual_as_of | — | false | — | — |
+| reactions | parent_detail_accounting | manual_as_of | — | false | — | — |
+| attachments | parent_detail_accounting | manual_as_of | — | false | — | — |
+
## polyfill/slack
| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
diff --git a/openspec/changes/add-connection-pause-resume/design.md b/openspec/changes/add-connection-pause-resume/design.md
new file mode 100644
index 000000000..35323e284
--- /dev/null
+++ b/openspec/changes/add-connection-pause-resume/design.md
@@ -0,0 +1,54 @@
+## Context
+
+`connector_instances.status` admits `active`, `paused`, `revoked`, `draft`. Before this change, `paused` was reachable only by data transplant or direct SQL, and escapable only through `resumePausedConnectionAfterCredentialCapture` — a side effect of the static-secret credential-capture route.
+
+The owner asked for pause and resume directly, and two stranded production connections proved the state needed a real exit, not just an entrance.
+
+## Goals
+
+- One coherent state machine: `active <-> paused`, `revoked -> active` via reactivate.
+- Wrong-state refusals are typed and distinguishable, never a generic inactive error.
+- A paused connection is visible in the console and carries its own way back.
+
+## Decisions
+
+### Pause ships as an owner action
+
+The alternative was to ship resume alone (enough to rescue the stranded rows) and defer pause until an owner asked twice. Rejected: a state that can be left but not entered is exactly the incoherence that produced this defect, and the state, the storage constraint, and the owner-facing spec language for it already existed. Shipping resume alone would have left `paused` reachable only by transplant — still a trapdoor, just a monitored one.
+
+Pause is genuinely distinct from revoke and not redundant with it. Revoke means "stop collecting and treat the authorization as withdrawn". Pause means "stop collecting, keep the credential, resume when I say". A source you can pause without re-authorizing is the feature the owner named.
+
+### Pause and resume are zero-cascade
+
+Both are pure status flips on the connector-instance row, matching revoke/reactivate. Neither touches records, credentials, grants, schedules, or the spine. Pause deliberately does NOT revoke the credential — retaining it is the entire difference from revoke, and it is what makes resume a single click rather than a re-authorization.
+
+### Wrong-state refusals get their own codes
+
+The namespace resolver reports any status mismatch as `connector_instance_inactive` (400). Both routes re-label it: resume against a non-paused target returns `connector_instance_not_paused` (409); pause against a non-active target returns `connector_instance_not_active` (409). This mirrors reactivate's existing `connector_instance_not_revoked` and lets a caller distinguish "wrong state" from "no such connection" without parsing prose.
+
+### The explicit owner route is not restricted by binding kind; the automatic hooks still are
+
+The shared `applyResume` primitive takes an optional `requireSourceBindingKind`. The automatic resume hooks — credential capture and run admission — pass `historical_archive`, keeping the implicit, non-owner-initiated path as narrow as it was. The explicit owner routes pass nothing: when the owner clicks Resume, the binding kind is not the server's business, and restricting it would have re-created the trap for every other paused connection.
+
+### Credential freshness is delegated to the next run
+
+Resume does not validate or supply a credential, matching reactivate. A resumed connection whose credential has expired surfaces a typed credential error on its next run through the existing health projection. Validating at resume time would duplicate that machinery and could refuse a resume the owner legitimately wants (for example, a file-import connection that authenticates to nothing).
+
+### A missing import directory is an infrastructure fault, not owner data loss
+
+`isManualUploadBinding` returning null for an unusable binding meant the run silently received no import-dir env var and reported `source_incomplete` — which reads as "your archive was incomplete" when the truth is "this server was told to read a path that is not on this disk". The resolver now stats the directory and throws `manual_upload_import_dir_missing` naming the path, the env var, and the connection. This is the silence that let two intact archives sit unnoticed.
+
+## Risks / Tradeoffs
+
+- **Pause could be mistaken for revoke.** Mitigated by console copy stating that data and the credential are retained, and by keeping the two actions visually and textually distinct.
+- **A paused connection stops collecting silently.** Mitigated by rendering `paused` as a first-class status and surfacing the connection with a Resume action rather than hiding it as revoked connections are hidden.
+- **Statting the import directory adds a filesystem call per manual-upload run.** Negligible against a run that then reads the archive, and it converts a silent misattribution into an actionable error.
+
+## Acceptance Checks
+
+- Resuming a paused connection returns 200 and the row becomes `active`; a second resume returns `connector_instance_not_paused` (409).
+- Pausing an active connection returns 200 and the row becomes `paused`; a second pause returns `connector_instance_not_active` (409).
+- Pause and resume leave record counts, credentials, grants, and schedules unchanged.
+- A paused connection renders a `paused` status and a Resume action in the console.
+- A manual-upload run whose `import_dir` is absent fails with `manual_upload_import_dir_missing` naming the missing path and env var.
+- A transplanted `historical_archive` binding is not claimed by the manual-upload resolver.
diff --git a/openspec/changes/add-connection-pause-resume/proposal.md b/openspec/changes/add-connection-pause-resume/proposal.md
new file mode 100644
index 000000000..dec6b2006
--- /dev/null
+++ b/openspec/changes/add-connection-pause-resume/proposal.md
@@ -0,0 +1,24 @@
+## Why
+
+`paused` has been a valid `connector_instances.status` since the schema's first CHECK constraint, and the owner-facing specs already promise pause and resume as per-instance actions. Neither existed. No production code path ever wrote `paused`, and only one path ever cleared it — a side effect of storing a static-secret credential.
+
+That left `paused` a state with no entrance and, for any connection that reached it another way, no exit. Two of the owner's connections (419,290 records) arrived pre-paused from an archive transplant and could not be recovered: `run` refused them as inactive, `reactivate` refused them as not-revoked, the console had no control because `SourceStatusKind` had no `paused` member, and file-import connections authenticate to nothing so the credential-capture side effect never fired.
+
+## What Changes
+
+- Add owner-initiated `pause` (active -> paused) and `resume` (paused -> active) as first-class connection actions, on both the owner-agent bearer surface and the owner-session reference surface.
+- Add `connector_instance_not_paused` (409) and `connector_instance_not_active` (409) so each action refuses a wrong-state target with a typed, distinguishable code.
+- Make `paused` a first-class `SourceStatusKind` the console renders and explains, with a Resume control on a paused connection and a Pause control on an active one.
+- Keep the narrow automatic resume for recovered `historical_archive` connections (credential capture and run admission) unchanged, so repairing a credential still resumes that row without a second owner step.
+- Fail loudly when a manual-upload binding's `import_dir` is absent on the host, naming the missing path and env var instead of reporting a bare `source_incomplete`.
+
+## Capabilities
+
+Modified:
+- `reference-connector-instances`
+
+## Impact
+
+- Pause and resume are zero-cascade status flips. Records, credentials, grants, schedules, and the audit spine are untouched by both.
+- Pause is deliberately NOT a substitute for revoke: it stops collection while keeping the credential, so it carries no credential-revocation or grant-narrowing semantics.
+- A manual-upload run whose artifact directory is missing now fails with a typed, actionable error rather than an owner-blaming coverage verdict.
diff --git a/openspec/changes/add-connection-pause-resume/specs/reference-connector-instances/spec.md b/openspec/changes/add-connection-pause-resume/specs/reference-connector-instances/spec.md
new file mode 100644
index 000000000..c9b4d5b5d
--- /dev/null
+++ b/openspec/changes/add-connection-pause-resume/specs/reference-connector-instances/spec.md
@@ -0,0 +1,74 @@
+## ADDED Requirements
+
+### Requirement: Owner-Initiated Connection Pause And Resume
+
+The reference implementation SHALL expose connection pause and connection resume as owner-initiated, connection-scoped actions keyed on exactly one `connector_instance_id`.
+
+Pause SHALL transition a connection from `active` to `paused`. Resume SHALL transition a connection from `paused` to `active`. Both SHALL be zero-cascade status flips: they SHALL NOT delete, rewrite, or hide already-collected records; SHALL NOT revoke, rotate, or erase stored credentials; and SHALL NOT alter disclosure grants, schedules, or the audit spine.
+
+Pause SHALL remain distinct from revoke. Pause stops future collection while retaining the connection's stored credential so that resume requires no re-authorization. Revoke SHALL continue to express a withdrawn authorization.
+
+Both actions SHALL be reachable on the owner-agent bearer control plane and on the owner-session reference control plane. Neither SHALL be reachable over `/mcp` or by a client grant token.
+
+A paused connection SHALL be refused by run admission until it is resumed. Resume SHALL NOT itself validate or supply a credential; a resumed connection whose credential is missing or expired SHALL surface a typed credential error on its next collection run.
+
+#### Scenario: Owner pauses an active connection
+
+- **WHEN** the owner pauses a connection whose status is `active`
+- **THEN** the connection's status SHALL become `paused`
+- **AND** its collected records, stored credential, disclosure grants, schedule, and audit spine SHALL be unchanged
+- **AND** subsequent run admission for that connection SHALL be refused while it remains paused
+
+#### Scenario: Owner resumes a paused connection
+
+- **WHEN** the owner resumes a connection whose status is `paused`
+- **THEN** the connection's status SHALL become `active`
+- **AND** its collected records SHALL be unchanged
+- **AND** the connection SHALL become eligible for run admission
+
+#### Scenario: Wrong-state pause and resume are typed and distinguishable
+
+- **WHEN** the owner resumes a connection whose status is not `paused`
+- **THEN** the reference SHALL refuse with `connector_instance_not_paused`
+- **AND** **WHEN** the owner pauses a connection whose status is not `active`
+- **THEN** the reference SHALL refuse with `connector_instance_not_active`
+- **AND** neither refusal SHALL mutate the connection
+
+#### Scenario: Unknown or foreign connection
+
+- **WHEN** a pause or resume names a connection that does not exist or belongs to another owner
+- **THEN** the reference SHALL refuse with `connector_instance_not_found`
+- **AND** SHALL NOT disclose whether the connection exists
+
+#### Scenario: Paused is an owner-visible status with a way back
+
+- **WHEN** the owner views a connection whose status is `paused`
+- **THEN** the console SHALL render `paused` as a first-class status distinct from revoked, syncing, and setup-in-progress
+- **AND** SHALL present a resume action for that connection
+- **AND** SHALL state that collected data is retained
+
+### Requirement: Manual Upload Import Directory Absence Is A Typed Failure
+
+The reference implementation SHALL verify that a manual-upload connection's `import_dir` exists as a directory on the host before a run is given the binding's import-directory environment variable.
+
+When the directory is absent or is not a directory, the reference SHALL fail the run environment resolution with a typed `manual_upload_import_dir_missing` error naming the missing path, the binding's `import_dir_env_var`, and the `connector_instance_id`. The reference SHALL NOT report this condition as a bare coverage or completeness verdict, because the fault is host or binding configuration rather than an incomplete owner-supplied archive.
+
+A source binding that is not a manual-upload binding SHALL continue to resolve to no manual-upload environment fragment without raising this error.
+
+#### Scenario: Missing import directory names what is missing
+
+- **WHEN** a manual-upload connection's `import_dir` does not exist on the host
+- **THEN** run environment resolution SHALL fail with `manual_upload_import_dir_missing`
+- **AND** the error SHALL name the missing path, the import-directory environment variable, and the connection
+- **AND** the run SHALL NOT report the condition as an incomplete owner-supplied source
+
+#### Scenario: Import path that is not a directory
+
+- **WHEN** a manual-upload connection's `import_dir` exists but is not a directory
+- **THEN** run environment resolution SHALL fail with `manual_upload_import_dir_missing`
+
+#### Scenario: Non-manual-upload binding is unaffected
+
+- **WHEN** a connection's source binding is not a manual-upload binding
+- **THEN** manual-upload run environment resolution SHALL yield no environment fragment
+- **AND** SHALL NOT raise `manual_upload_import_dir_missing`
diff --git a/openspec/changes/add-connection-pause-resume/tasks.md b/openspec/changes/add-connection-pause-resume/tasks.md
new file mode 100644
index 000000000..2ac63eefa
--- /dev/null
+++ b/openspec/changes/add-connection-pause-resume/tasks.md
@@ -0,0 +1,43 @@
+## 1. Resume action
+
+- [x] 1.1 Add the shared `applyResume` primitive with an optional `requireSourceBindingKind` guard.
+- [x] 1.2 Add owner-agent bearer resume routes addressed by `connection_id` and by `connector_id`.
+- [x] 1.3 Add the owner-session reference resume route, unrestricted by binding kind.
+- [x] 1.4 Keep the automatic `historical_archive` resume hooks (credential capture, run admission) narrow.
+- [x] 1.5 Pin `connector_instance_not_paused` (409) in the error-status table.
+- [x] 1.6 Publish the resume contract manifests and regenerate the OpenAPI/route docs.
+
+## 2. Pause action
+
+- [x] 2.1 Add the shared `applyPause` primitive (active -> paused).
+- [x] 2.2 Add owner-agent bearer pause routes addressed by `connection_id` and by `connector_id`.
+- [x] 2.3 Add the owner-session reference pause route.
+- [x] 2.4 Pin `connector_instance_not_active` (409) in the error-status table.
+- [x] 2.5 Publish the pause contract manifests and regenerate the OpenAPI/route docs.
+
+## 3. Console
+
+- [x] 3.1 Add `paused` to `SourceStatusKind` and render it distinctly from revoked/syncing/pending.
+- [x] 3.2 Surface a paused connection with a Resume action instead of hiding it.
+- [x] 3.3 Add a Pause action to an active connection's detail page.
+- [x] 3.4 Keep the recovered-archive reconnect journey routed to credential repair.
+
+## 4. Manual-upload import directory
+
+- [x] 4.1 Verify `import_dir` exists before returning the import-dir env fragment.
+- [x] 4.2 Fail with `manual_upload_import_dir_missing` naming path, env var, and connection.
+- [x] 4.3 Confirm a non-manual-upload binding still resolves to null without raising.
+
+## 5. Stranded transplanted bindings
+
+- [x] 5.1 Add a dry-run-default repair tool that lifts `original_source_binding` to the top level, restores `kind`, and rewrites `import_dir` to a discovered on-disk path.
+- [x] 5.2 Refuse on zero and on ambiguous discovery candidates rather than guessing.
+- [x] 5.3 Back up the pre-image binding in the same transaction as the write, guarded on the pre-image still being current.
+
+## 6. Validation
+
+- [x] 6.1 Unit-test the repair tool's envelope recognition, lift, discovery, and refusals.
+- [x] 6.2 Unit-test the import-directory guard, including mutation-checking that removing it turns the tests red.
+- [x] 6.3 Test the resume and pause routes, including wrong-state and foreign-target refusals.
+- [x] 6.4 Typecheck polyfill-connectors, reference-implementation, console/operator-ui, and packages/mcp-server.
+- [x] 6.5 Run `npx biome check` on every touched file.
diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/.openspec.yaml b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/.openspec.yaml
new file mode 100644
index 000000000..d160e09cf
--- /dev/null
+++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/.openspec.yaml
@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-08-21
diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/design.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/design.md
new file mode 100644
index 000000000..965d30553
--- /dev/null
+++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/design.md
@@ -0,0 +1,202 @@
+## Context
+
+PDPP already had the right mechanism and the right terminal vocabulary.
+`run.abandoned` is a first-class terminal event in `check-run-terminal.sql`,
+`run-history-writer.ts`, and the `terminal_status` contract in
+`openspec/specs/reference-implementation-architecture/spec.md`. A boot
+reconciler already wrote it. The design was disabled by one identity defect,
+and the surrounding machinery — a drain, a wall clock, a second reconciler —
+existed to compensate for the resulting silence.
+
+This change is therefore mostly subtraction. The one substantive addition is
+durable identity; the deletions are its consequence.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- Every run reaches exactly one durable terminal state, written by exactly one
+ writer.
+- An interrupted run is named `abandoned`, never `failed`.
+- A successor container can adjudicate its predecessor's orphans.
+- No adjudication decision depends on a tuned time threshold.
+
+**Non-Goals:**
+
+- Finishing in-flight work during shutdown.
+- Committing staged cursors under interruption.
+- A generic cross-table "interrupted work engine". Six boot reconcilers should
+ share a *predicate*, not an executor; a unified engine would need per-table
+ SQL, projections, and terminal vocabularies injected into it, which is
+ relocation rather than decomplecting.
+
+## Decisions
+
+### The successor adjudicates; the dying owner writes nothing
+
+The dying process is the wrong writer because it cannot be relied on to run at
+all. A `kill -9` gets no shutdown path, so any design that needs the owner to
+write its own terminal state has an unhandled case by construction.
+
+This is the layering mature systems ship. Temporal's server has no
+crash-detection channel and converts worker silence into a recorded outcome
+with a timer alone; its `WorkerStopTimeout` — "the time delay before hard
+terminate worker" — defaults to **0s**, so it does not wait for in-flight work
+by default. Kafka's successor performs the recovery: `InitProducerId` "Bumps up
+the epoch of the PID, so that the any previous zombie instance of the producer
+is fenced off" and "Recovers (rolls forward or rolls back) any transaction left
+incomplete by the previous instance." Primary sources for both, with quotes and
+access dates, are in the research entry
+`ai/research/pdpp/interrupted-work-needs-an-owner-fenced-terminal-state-not-a-graceful-shutdown-because-dockers-10s-stop-is-shorter-than-the-work.md`
+(claims A, sources `sdk-go-worker-base`, `temporal-activity-failures`,
+`kip-98`).
+
+Alternative: retarget the controller path to write `run.abandoned` instead of
+deleting it. Rejected because two writers racing for one run's terminal event
+is worse than one. The boot reconciler is the better writer: it reads the
+append-only spine rather than the `controller_active_runs` flight table, it is
+idempotent on `caused_by_event_id` through the
+`spine_run_abandoned_cause_unique` partial index, and it aborts boot on error
+instead of swallowing it.
+
+### `abandoned` stays distinct from `failed`
+
+Interruption is Kubernetes' `Unknown`, not `False`. Kubernetes keeps the two
+distinct because they drive different remediation — node `Unknown` yields an
+`unreachable` taint, `False` yields `not-ready`. The same holds here: `failed`
+on a bank connector means ask the human; `abandoned` means nobody knows, and
+the normal schedule will pick it up. Collapsing them is what pages an owner for
+a deploy.
+
+The cost of the collapse is measured, not theoretical: of 134 production runs
+recorded as `run.failed`/`controller_restarted`, 55 had staged a cursor and 34
+had durably ingested a batch before being written down as plain failures.
+
+### Durable identity beats an env var
+
+Setting `PDPP_CONTROLLER_ID` in the run command would fix the filter. It is
+rejected as the *default* because its failure mode is silent. Production runs
+from a hand-rolled `docker run`; an identity that depends on an operator
+reproducing a flag on every container recreation is one omission away from
+reopening this exact defect, with no signal that it has reopened. That is how
+the defect hid for three months. Reading the value from the same database that
+holds the runs makes the correct answer the default. `PDPP_CONTROLLER_ID` is
+kept as an override so a genuine multi-controller deployment can still
+partition ownership.
+
+`os.hostname()` survives only as the seed for the first row, never as the live
+identity. The boot epoch still advances per boot; only the identity is stable,
+so adjudication still distinguishes "a prior incarnation owned this" from "I
+own this".
+
+### The drain is deleted, not lengthened
+
+Rejected on measurement. Production sets no `--stop-timeout`
+(`docker inspect pdpp-core-prod-drain` returns `StopTimeout=`), so
+Docker's 10s default governs, and `--stop-timeout` is fixed at container
+creation — Docker has no equivalent of systemd's runtime `EXTEND_TIMEOUT_USEC=`
+extension. Only 2 of 17 connectors finish inside 10s at p95; slack's p95 is
+4052.8s. The gap is two orders of magnitude and is not closable by tuning.
+
+The drain was observed in production failing exactly that way, logging
+`{"drained":0,"elapsedMs":5000,"timedOut":1,"msg":"connector run drain complete"}`.
+It is a negative, not a small win: it consumes half the SIGKILL budget doing
+nothing while making the failure look handled.
+
+Only the shutdown call site is removed. `drainActiveRuns` stays on the
+controller, where it means "await in-flight runs" — 137 references exist on the
+base branch and 136 remain, the single removal being the shutdown call. Sidekiq
+draws the same line between *quiet* and *drain*.
+
+### The epoch fence, not a wall clock, decides in-flight ownership
+
+A wall-clock reaper must guess a threshold and can be wrong in both directions.
+River documents the cost in its own config comment — "this can result in repeat
+or duplicate execution of a job that is not actually stuck but is still
+working" — and Oban's Lifeline carries the same caveat. An epoch fence needs no
+guess: a unit stamped with epoch E, observed by epoch E' ≠ E, is *provably*
+orphaned.
+
+A `NULL` owner epoch must be swept, since no live process claims it. On
+PostgreSQL this arm is spelled out explicitly rather than left to
+`IS DISTINCT FROM`, because `owner_epoch IS DISTINCT FROM NULL` reduces to
+`owner_epoch IS NOT NULL` and would spare exactly the legacy rows that most
+need reclaiming.
+
+## Corrections to the research entry
+
+The research entry is `settled` and its prior-art layer holds, but
+implementation measurement refuted four of its claims. The spec deltas reflect
+the measurements, not the entry.
+
+1. **"The leak is live and accruing" — false.** The entry inferred acceleration
+ from re-measuring 121 as 123 forty minutes later. Those two extra rows were
+ runs the live container had started minutes earlier — live work, not new
+ orphans. The newest true orphan is 2026-07-10. The leak stopped because the
+ *dishonest* path started catching what the honest one could not see; two
+ defects were masking each other.
+
+2. **Deleting `MANUAL_UPLOAD_IN_FLIGHT_STALE_MS` rests on an epoch that did not
+ exist.** The entry said the wall clock answered "a question the epoch
+ answers exactly", assuming an epoch was available on that table. It was not:
+ `manual_upload_artifacts` had no epoch column, confirmed absent on the live
+ database. Deleting the clock without adding the column would have been a
+ straight regression — the sweep would have had nothing left to distinguish
+ live work from dead. The column is the substantive change; the deletion is
+ the consequence.
+
+3. **"The two reconcilers race on the same runs" — false.** They are fully
+ disjoint: zero of the 134 `controller_restarted` runs ever also received a
+ `run.abandoned`. This strengthens the case for deleting the controller path
+ rather than weakening it — there was no overlap proving the boot path
+ already covered those runs, so the identity fix had to land first.
+
+4. **The §11 gate is answered NO.** The entry asked whether connectors emit
+ bounded `DETAIL_COVERAGE` with `covered == considered` and a non-null
+ boundary. Zero of 34,928 `run.detail_coverage_declared` events in production
+ carry `boundary`, `slice_start`, or `slice_end`. Committing staged cursors
+ under `INTERRUPTED` would therefore fabricate denominators. That stage is
+ correctly not shipped and is an explicit non-goal.
+
+## Risks / Trade-offs
+
+- [A backfill adjudicates live work] -> The repair tool excludes runs belonging
+ to the newest `controller.booted` epoch. This is load-bearing: the first dry
+ run against production reported 123 because it picked up two runs the live
+ container had started ninety seconds earlier. Adjudicating those would have
+ declared live work abandoned and freed the connection for a competing run,
+ reintroducing the exact duplicate-execution hazard the epoch fence exists to
+ avoid. Excluding the newest epoch fixes it with no threshold to tune; the
+ dry run then reported 121, matching the measured backlog.
+- [Deleting the controller path loses stale-claim cleanup] -> It does not. The
+ function is retained as `releaseAbandonedControllerRunClaims` and still
+ releases stale `controller_active_runs` rows, because
+ `reconcileBrowserSurfaceLeasesAfterBoot` reads that table to decide which
+ leases are still held. Releasing a claim and reporting on the work are
+ separate jobs.
+- [No drain means Chromium residue leaks] -> Chromium residue is still cleaned
+ at next boot by `profile-lock.ts`.
+- [A multi-controller deployment adjudicates a peer's live runs] -> The boot
+ reconciler still filters on `controller_id`, and `PDPP_CONTROLLER_ID`
+ partitions ownership. Only the repair tool ignores `controller_id`, and it is
+ documented as single-controller-only, with `--connector` scoping for anyone
+ else.
+- [Legacy rows without an epoch] -> Both new columns are NULL-tolerant. A NULL
+ owner epoch is treated as unclaimed and swept; a NULL `controller_id` is
+ treated as ours under the single-controller assumption, which is the
+ pre-existing behavior.
+
+## Migration Plan
+
+1. Land durable identity first. It is behavior-preserving on a host whose
+ hostname was already stable, and on a container host it only makes the
+ existing ownership filter match reality.
+2. Run the repair tool's dry run, confirm the scope, then `--apply` with
+ pre-image snapshots. Reversible: it only adds terminal events where none
+ exist, and never edits or deletes an existing event.
+3. Delete the controller failure path only after identity is durable, so the
+ boot reconciler demonstrably covers the cases it covered.
+4. Delete the drain and fence the manual-upload sweep. Both are independent of
+ the above.
+5. Roll back by restoring the controller path and the drain call site. Already
+ written `run.abandoned` events remain valid and correct.
diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/proposal.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/proposal.md
new file mode 100644
index 000000000..8d0edfb46
--- /dev/null
+++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/proposal.md
@@ -0,0 +1,107 @@
+## Why
+
+A run interrupted by container replacement never received an honest terminal
+state, and the mechanism built to give it one was silently disabled.
+
+`resolveControllerId` fell back to `os.hostname()`, which under Docker is the
+container id and is fresh on every `docker run`. `PDPP_CONTROLLER_ID` is unset
+in production, so the boot reconciler's ownership filter
+`COALESCE(data_json->>'controller_id', $2) = $2` excluded every prior
+container's orphans — a lost-and-found that only accepted items it had lost
+itself. Production holds 1,231 `controller.booted` events across 1,153 distinct
+controller ids; the one host with a stable id (`peregrine`, non-Docker, 18 boots
+under one id) is the only place the mechanism ever worked.
+
+The measured cost: 121 `run.started` events spanning 2026-05-15 to 2026-07-10,
+across 106 distinct controller ids, carried no terminal event of any kind. A
+second, cruder path picked up part of the slack with the wrong vocabulary —
+of 134 runs it recorded as `run.failed`/`controller_restarted`, 55 had staged a
+cursor and 34 had durably ingested a batch first. Those were interruptions
+recorded as failures, and the two states have different remedies: `failed` on a
+bank connector means ask the human, `abandoned` means nobody knows and the
+normal schedule will pick it up.
+
+The shipped 5s SIGTERM drain cannot close this. Production sets no
+`--stop-timeout`, so Docker's 10s default governs, and only 2 of 17 connectors
+finish inside 10s at p95. It was observed in production logging
+`{"drained":0,"elapsedMs":5000,"timedOut":1}` — burning its whole budget and
+abandoning the run anyway.
+
+## What Changes
+
+- Make controller identity durable in a `controller_identity` row, seeded from
+ the hostname on the first boot that finds the table empty and read back
+ unchanged thereafter, so a successor container inherits the ownership filter
+ that lets it adjudicate its predecessor's orphans. `PDPP_CONTROLLER_ID` still
+ wins when set. The boot epoch still advances per boot.
+- Stop recording interrupted runs as failures. The controller path no longer
+ emits `run.failed`/`controller_restarted`; it retains only its stale-claim
+ release, renamed to say what it does. The boot reconciler becomes the single
+ writer of an interrupted run's terminal state.
+- Delete the connector drain from the SIGTERM path. `drainActiveRuns` stays on
+ the controller, where it means "await in-flight runs" — 136 of its 137
+ references carry that meaning and keep working.
+- Fence in-flight manual-upload artifacts by owner epoch instead of the
+ 10-minute `MANUAL_UPLOAD_IN_FLIGHT_STALE_MS` wall clock. This requires adding
+ an `owner_epoch` column to `manual_upload_artifacts`, which did not exist.
+- Add an owner-operated repair tool that adjudicates the already-stranded
+ backlog, dry-run by default, with pre-image snapshots taken inside the write
+ transaction.
+
+## Capabilities
+
+### New Capabilities
+
+None.
+
+### Modified Capabilities
+
+- `reference-implementation-architecture`: Define durable controller identity,
+ the successor-adjudicates ownership rule and its newest-epoch exclusion, the
+ single-writer rule for an interrupted run's terminal state, epoch-fenced
+ in-flight artifact sweeps, and the absence of a shutdown-path drain.
+
+## Impact
+
+- `reference-implementation/lib/controller-boot.ts`,
+ `runtime/controller.ts`, `server/index.ts`,
+ `server/stores/manual-upload-artifact-store.ts`,
+ `server/routes/ref-manual-upload-draft-connection.ts`
+- New `controller_identity` table and additive `manual_upload_artifacts.owner_epoch`
+ column on SQLite and PostgreSQL. Both are NULL-tolerant, so existing databases
+ migrate without a backfill.
+- New owner-only repair script
+ `reference-implementation/scripts/repair/adjudicate-orphaned-runs.ts`.
+- Shutdown is faster and its failure mode is honest. Interrupted runs terminalize
+ as `abandoned` at the next boot rather than as `failed` or not at all.
+- Does not change record ingestion, checkpoint commit, or any connector.
+
+## Non-Goals
+
+- **Committing staged cursors under interruption.** The research entry gated
+ this on whether connectors emit bounded `DETAIL_COVERAGE` with
+ `covered == considered` and a non-null boundary. Measured on the live spine,
+ the answer is no: zero of 34,928 `run.detail_coverage_declared` events carry
+ `boundary`, `slice_start`, or `slice_end`. Committing staged cursors under an
+ interrupted terminal state would fabricate denominators, so it is out of scope
+ until that evidence exists.
+- **The checkpoint-contract / interval-claim design.** Out of scope here, and
+ now carried by `qualify-connectors-for-incremental-checkpoint-commit`. An
+ earlier draft of this non-goal described the prototype as not surviving
+ contact with 2 of 3 connectors and therefore "not ready for OpenSpec". That
+ reading is corrected: the contract is a **qualification standard**, and a
+ connector failing it is the qualifier working. `slack` is a genuine
+ disqualifier (no ordered scan at any granularity); `heb` fails at fine
+ granularity but qualifies at day granularity under a closed-day rule. See
+ that change for the verdicts and their evidence.
+- **Any Node version change.**
+- **Auto-resume or auto-retry of interrupted runs.** `chase`, `usaa`, `venmo`,
+ `heb`, `amazon`, and `reddit` need an interactive human sign-in; adjudication
+ is silent by design and lets the normal schedule pick the work up.
+
+## Residual risks
+
+- Owner-authorized live verification remains: after one deploy in a replaced
+ container, a `SIGKILL` mid-run should produce exactly one terminal event for
+ that run and it should be `run.abandoned`. This is live-environment
+ verification, not an implementation task.
diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/specs/reference-implementation-architecture/spec.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/specs/reference-implementation-architecture/spec.md
new file mode 100644
index 000000000..5c632e5dd
--- /dev/null
+++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/specs/reference-implementation-architecture/spec.md
@@ -0,0 +1,242 @@
+## ADDED Requirements
+
+### Requirement: Every started run SHALL reach exactly one durable terminal state
+
+For every `run.started` event in the spine, either a terminal event SHALL exist
+for that run, or the run's `boot_epoch` SHALL equal the current process's boot
+epoch. The canonical terminal set is `run.completed`, `run.failed`,
+`run.browser_surface_failed`, `run.cancelled`, and `run.abandoned`.
+
+Exactly one component SHALL write a given run's terminal state. A run
+interrupted by process death SHALL be adjudicated by the boot reconciler
+reading the append-only spine. No other reconciler SHALL emit a terminal event
+for an interrupted run, and no component SHALL emit a second terminal event for
+a run that already has one.
+
+Adjudication SHALL be idempotent: each emitted terminal event SHALL carry
+`caused_by_event_id`, and repeated adjudication passes SHALL produce exactly
+one terminal event per orphan.
+
+#### Scenario: A run interrupted by process death is terminalized at the next boot
+
+- **WHEN** a run has a `run.started` event, no terminal event, and a `boot_epoch`
+ that is not the current process's boot epoch
+- **THEN** the boot reconciler SHALL emit exactly one terminal event for that run
+- **AND** the run's `terminal_status` SHALL become non-null
+
+#### Scenario: Repeated adjudication does not duplicate terminal events
+
+- **WHEN** two successive reconcile passes observe the same orphaned run
+- **THEN** exactly one terminal event SHALL exist for that run
+
+#### Scenario: A second writer does not emit a competing terminal event
+
+- **WHEN** a reconciler other than the boot reconciler observes a stale
+ `controller_active_runs` claim for a run
+- **THEN** it SHALL release the stale claim
+- **AND** it SHALL NOT emit any terminal event for that run
+
+### Requirement: An interrupted run SHALL terminalize as abandoned, never as failed
+
+A run whose owner process died without reporting SHALL be recorded as
+`run.abandoned` with reason `controller_terminated_before_run_finished`. It
+SHALL NOT be recorded as `run.failed`, and SHALL NOT be assigned any
+failure reason that asserts an observed failure.
+
+`abandoned` and `failed` SHALL remain distinct terminal states because they
+carry different remedies: `failed` indicates an observed failure that MAY
+require owner attention, while `abandoned` indicates that no owner will ever
+report on the unit and the normal schedule will pick the work up.
+
+Adjudicating an interrupted run SHALL NOT request owner attention, SHALL NOT
+emit an owner notification, and SHALL NOT re-run, re-queue, or retry the work.
+
+Adjudication SHALL NOT delete or edit any existing event, SHALL NOT modify
+ingested records, and SHALL NOT revise `records_emitted`. Records durably
+ingested before the interruption SHALL stay committed.
+
+#### Scenario: An interrupted run with staged cursors is not called a failure
+
+- **WHEN** a run staged a cursor or durably ingested a batch and was then
+ interrupted by process death
+- **THEN** its terminal event SHALL be `run.abandoned`
+- **AND** no `run.failed` event SHALL exist for that run
+
+#### Scenario: Adjudication is silent
+
+- **WHEN** an interrupted run for a connector that requires an interactive human
+ sign-in is adjudicated
+- **THEN** zero owner-attention rows SHALL be created
+- **AND** zero owner notifications SHALL be sent
+- **AND** the run SHALL NOT be automatically retried
+
+#### Scenario: Durable records survive adjudication
+
+- **WHEN** a run is adjudicated as abandoned after durably ingesting records
+- **THEN** those records SHALL remain committed
+- **AND** the run's reported `records_emitted` SHALL NOT be revised
+
+### Requirement: Controller identity SHALL be durable across container replacement
+
+The reference implementation SHALL resolve controller identity from
+`PDPP_CONTROLLER_ID` when set, and otherwise from a durable single-row
+`controller_identity` record stored in the same database that holds the runs.
+The record SHALL be seeded from the host name on the first boot that finds it
+absent, and SHALL be read back unchanged on every later boot.
+
+The host name SHALL NOT be used as the live controller identity. Under a
+container runtime the host name is the container id and is fresh on every
+container creation, which makes an ownership filter keyed on it exclude every
+prior container's orphans.
+
+The boot epoch SHALL continue to advance on every boot. Only the identity is
+stable, so adjudication can still distinguish a prior incarnation's work from
+the current process's work.
+
+An identity that must be supplied by operator configuration SHALL NOT be the
+default, because omitting it fails silently and reopens the ownership defect
+with no signal.
+
+#### Scenario: A replacement container inherits its predecessor's identity
+
+- **WHEN** the reference implementation boots in a new container against a
+ database whose `controller_identity` row already exists
+- **THEN** it SHALL adopt the stored controller id rather than its host name
+- **AND** its ownership filter SHALL select orphans left by the prior container
+
+#### Scenario: First boot seeds the identity
+
+- **WHEN** the reference implementation boots against a database with no
+ `controller_identity` row and no `PDPP_CONTROLLER_ID`
+- **THEN** it SHALL write one row seeded from the host name
+- **AND** every later boot SHALL read that same value back
+
+#### Scenario: The operator override still partitions ownership
+
+- **WHEN** `PDPP_CONTROLLER_ID` is set
+- **THEN** it SHALL take precedence over the stored row
+- **AND** a multi-controller deployment SHALL remain isolated by controller id
+
+### Requirement: A successor SHALL adjudicate only units whose owner epoch is not its own
+
+A unit of work SHALL carry, in the same durable write that starts it, the
+identity of the owner epoch entitled to finish it. A successor epoch SHALL
+adjudicate a unit only when that unit's owner epoch is not the successor's own
+epoch.
+
+A successor SHALL NOT adjudicate any unit belonging to the newest boot epoch.
+A unit started by the process that is still running is live work, not an
+orphan: it lacks a terminal event for the ordinary reason that it has not
+finished. Adjudicating it would declare live work abandoned and free its
+resource for a competing run, reintroducing the duplicate-execution hazard the
+epoch fence exists to prevent.
+
+Eligibility SHALL be decided by epoch comparison, not by an age threshold. A
+unit with a `NULL` owner epoch SHALL be treated as unclaimed and SHALL be
+eligible, since no live process claims it.
+
+An owner-operated repair tool MAY ignore the controller-identity filter, since
+that field is the defect being healed, but SHALL NOT ignore the newest-epoch
+exclusion.
+
+#### Scenario: Live work in the newest epoch is never adjudicated
+
+- **WHEN** an adjudication pass observes a unit whose owner epoch equals the
+ newest `controller.booted` epoch
+- **THEN** it SHALL NOT adjudicate that unit
+- **AND** it SHALL NOT release or reassign that unit's resource
+
+#### Scenario: A prior epoch's unit is adjudicated without a time threshold
+
+- **WHEN** an adjudication pass observes a unit whose owner epoch is neither its
+ own nor the newest epoch
+- **THEN** it SHALL adjudicate that unit regardless of the unit's age
+
+#### Scenario: A unit with no recorded owner epoch is eligible
+
+- **WHEN** an adjudication pass observes an in-flight unit whose owner epoch is
+ `NULL`
+- **THEN** it SHALL treat that unit as unclaimed and eligible for adjudication
+
+### Requirement: In-flight manual-upload artifacts SHALL be swept by owner epoch
+
+The manual-upload artifact store SHALL record the owner epoch of the process
+that created an artifact, written in the same INSERT that creates it. An
+artifact left in an in-flight state SHALL be eligible for sweep when its owner
+epoch is not the current epoch, or when its owner epoch is `NULL`.
+
+Sweep eligibility SHALL NOT be decided by a wall-clock staleness threshold. A
+guessed threshold can be wrong in both directions: it can sweep a slow but live
+validation out from under itself, or leave a genuinely dead upload in place.
+
+An artifact whose owner epoch matches the current epoch SHALL NOT be swept.
+Claiming an artifact for sweep SHALL remain an atomic compare-and-swap and
+SHALL stamp the claiming epoch on a win, so a concurrent second claim loses.
+
+The owner-epoch column SHALL be additive and `NULL`-tolerant on both backends so
+existing databases migrate without a backfill.
+
+#### Scenario: A live in-flight artifact is never swept
+
+- **WHEN** the boot sweep observes an in-flight artifact whose owner epoch is the
+ current epoch
+- **THEN** it SHALL leave that artifact untouched regardless of its age
+
+#### Scenario: An orphaned in-flight artifact is swept immediately
+
+- **WHEN** the boot sweep observes an in-flight artifact whose owner epoch is not
+ the current epoch
+- **THEN** it SHALL claim and sweep that artifact without waiting for any
+ staleness interval
+
+#### Scenario: Legacy artifacts written before the column existed are swept
+
+- **WHEN** the boot sweep observes an in-flight artifact whose owner epoch is
+ `NULL`
+- **THEN** it SHALL treat it as unclaimed and sweep it
+- **AND** the backend predicate SHALL NOT reduce to one that spares `NULL` rows
+
+#### Scenario: Concurrent claims resolve to one winner
+
+- **WHEN** two processes attempt to claim the same in-flight artifact for sweep
+- **THEN** exactly one SHALL win the compare-and-swap and stamp its epoch
+- **AND** the loser SHALL NOT sweep that artifact
+
+### Requirement: Shutdown SHALL NOT attempt to drain in-flight connector runs
+
+The reference implementation SHALL NOT wait for in-flight connector runs on the
+`SIGTERM` path. Interrupted runs SHALL be adjudicated by the successor at the
+next boot instead.
+
+A shutdown drain SHALL NOT be reintroduced as the mechanism for terminalizing
+interrupted work. The container runtime's stop grace period is fixed at
+container creation and is shorter than a connector run, so a drain cannot
+complete the work, and any drain consumes the grace period that remains before
+forced termination. A forced termination receives no shutdown path at all, so a
+design that requires the dying process to write its own terminal state has an
+unhandled case by construction.
+
+Removing the drain SHALL NOT remove the controller's ability to await in-flight
+runs. That capability SHALL remain available to the run watchdog and to
+callers awaiting a specific run.
+
+Resources that outlive an abrupt termination, such as browser profile locks,
+SHALL continue to be reclaimed at the next boot.
+
+#### Scenario: Shutdown does not wait for a running connector
+
+- **WHEN** the reference implementation receives `SIGTERM` while a connector run
+ is in flight
+- **THEN** it SHALL NOT block shutdown awaiting that run
+- **AND** the run SHALL be adjudicated as abandoned at the next boot
+
+#### Scenario: Awaiting in-flight runs remains available to other callers
+
+- **WHEN** the run watchdog or a caller awaiting a specific run needs in-flight
+ run completion
+- **THEN** the controller SHALL still provide that capability
+
+#### Scenario: Browser profile locks are reclaimed after an abrupt termination
+
+- **WHEN** the process is terminated without running any shutdown path
+- **THEN** stale browser profile locks SHALL be reclaimed at the next boot
diff --git a/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/tasks.md b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/tasks.md
new file mode 100644
index 000000000..04e5f193e
--- /dev/null
+++ b/openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/tasks.md
@@ -0,0 +1,86 @@
+## 1. Durable controller identity
+
+- [x] 1.1 Add a `controller_identity` table on SQLite and PostgreSQL holding one
+ row, seeded from `os.hostname()` on the first boot that finds it empty.
+- [x] 1.2 Resolve controller identity as `PDPP_CONTROLLER_ID`, then the durable
+ row, so the hostname survives only as a first-boot seed and never as the
+ live identity.
+- [x] 1.3 Keep the boot epoch advancing per boot, so adjudication still separates
+ "a prior incarnation owned this" from "I own this".
+- [x] 1.4 Add `test/controller-identity-durability.test.ts` proving identity
+ survives a simulated container replacement and that a successor's
+ ownership filter then selects the predecessor's orphans.
+
+## 2. Single writer for an interrupted terminal state
+
+- [x] 2.1 Remove the `run.failed`/`controller_restarted` emission from the
+ controller reconciliation path.
+- [x] 2.2 Retain the stale-claim release as
+ `releaseAbandonedControllerRunClaims`, since
+ `reconcileBrowserSurfaceLeasesAfterBoot` depends on
+ `controller_active_runs` to decide which leases are still held.
+- [x] 2.3 Update `list-active-runs.sql` and `check-run-terminal.sql` and the
+ SQLite/collection-store scheduler drivers for the new terminal vocabulary.
+- [x] 2.4 Update tests to the new contract rather than relaxing them: the restart
+ test asserts `terminal_status === "abandoned"` and that no `run.failed`
+ exists for the run.
+
+## 3. Drop the shutdown drain
+
+- [x] 3.1 Remove the connector drain from the SIGTERM path in `server/index.ts`.
+- [x] 3.2 Verify `drainActiveRuns` itself is retained on the controller and that
+ exactly one reference is removed — 137 references on the base branch, 136
+ after — so the watchdog and `awaitRun` keep working.
+
+## 4. Epoch-fence in-flight manual uploads
+
+- [x] 4.1 Add a NULL-tolerant `owner_epoch` column to `manual_upload_artifacts`
+ on both backends, written in the same INSERT that creates the artifact.
+- [x] 4.2 Replace the `MANUAL_UPLOAD_IN_FLIGHT_STALE_MS` eligibility predicate
+ with the epoch predicate, spelling the NULL arm out explicitly on
+ PostgreSQL so `IS DISTINCT FROM` does not reduce to `IS NOT NULL` and
+ spare the legacy rows.
+- [x] 4.3 Keep the atomic compare-and-swap in `claimForSweep` unchanged in kind,
+ stamping `owner_epoch` on a win so a concurrent second claim loses.
+- [x] 4.4 Update crash-recovery and artifact-store tests on both backends.
+
+## 5. Repair the stranded backlog (operational)
+
+- [x] 5.1 Add `scripts/repair/adjudicate-orphaned-runs.ts` following the sibling
+ repair tools: dry-run by default, `--apply` required to write, full scope
+ printed on every invocation, payload-free output, `--limit` bounding.
+- [x] 5.2 Select orphans without filtering on `controller_id` — that field is
+ what broke — but exclude runs belonging to the newest
+ `controller.booted` epoch so live work is never adjudicated.
+- [x] 5.3 Snapshot the pre-image of every targeted `run.started` event and every
+ re-projected `run_history` row into an `aor_backup` table inside the same
+ transaction as the write.
+- [x] 5.4 Add `test/adjudicate-orphaned-runs.test.ts` covering idempotency,
+ newest-epoch exclusion, and backup completeness.
+- [x] 5.5 Run the dry run against production first and confirm the scope before
+ `--apply`. Verified: dry run reported 121 after the newest-epoch exclusion
+ (123 before it, the two extras being live runs started 90 seconds
+ earlier).
+- [x] 5.6 Apply. Verified on the live instance: backup table
+ `aor_backup_ad8166a2__all__20260821161222` holds 121 spine-event
+ pre-images spanning 2026-05-15 to 2026-07-10 across 106 distinct
+ controller ids, and the orphan predicate now returns 0.
+
+## 6. Validation
+
+- [x] 6.1 Confirm the corrections against the live database rather than the
+ research entry: newest true orphan is 2026-07-10 (the leak is not
+ accruing); `manual_upload_artifacts` had no epoch column; zero of the 134
+ `controller_restarted` runs also received `run.abandoned` (the reconcilers
+ never raced); zero of 34,928 `run.detail_coverage_declared` events carry
+ `boundary`, `slice_start`, or `slice_end`.
+- [ ] 6.2 Run focused controller, controller-boot, manual-upload, and repair-tool
+ suites on both backends, plus typecheck, formatting, and strict OpenSpec
+ validation.
+- [ ] 6.3 Owner-authorized live verification: after one deploy in a replaced
+ container, `SIGKILL` the server mid-run, restart, and assert the run has
+ exactly one terminal event and it is `run.abandoned`, with zero
+ `needs_human` attention rows and zero pushes. This must run the successor
+ in a *new container*, not just a new process — every pre-existing
+ boot-orphan test shared a `controller_id` with the orphan it created,
+ which is what let the defect survive.
diff --git a/openspec/changes/allow-resumable-checkpoints-after-restart/proposal.md b/openspec/changes/allow-resumable-checkpoints-after-restart/proposal.md
new file mode 100644
index 000000000..ef91c0536
--- /dev/null
+++ b/openspec/changes/allow-resumable-checkpoints-after-restart/proposal.md
@@ -0,0 +1,127 @@
+## Why
+
+A run interrupted by a server restart resumes from zero. The owner's standard
+for #166 is "if I restart the server, my data doesn't all go unhealthy"; this
+is the largest remaining gap against it, and it needs a spec change, so it is
+proposed rather than implemented.
+
+`commitState` has exactly two call sites, both inside `handleDoneClose`
+(`runtime/index.ts`). `handleStateMessage` only stages cursors in the
+in-process `newState` map, which dies with the process. **A run that never
+reaches DONE commits no cursor at all, however long it ran.**
+
+Concrete cost. A Slack archive walk takes ~54 minutes. Killed at minute 50, it
+re-fetches all 50 minutes of work on the next run. If deploys land closer
+together than one walk length, such a connection can never finish — it is not
+slow, it is non-converging. 45 runs were ended by restarts between 2026-08-15
+and 2026-08-22 (28 `controller_terminated_before_run_finished` + 17
+`controller_restarted`), concentrated in the longest walks: 9 Slack, 4 Gmail,
+3 YNAB, plus Amazon and Google Maps.
+
+**This is bounded, not catastrophic.** `records` carries a UNIQUE CONSTRAINT
+`records_connector_instance_stream_key (connector_instance_id, stream,
+record_key)`, so ingest is idempotent: a re-walk re-upserts rather than
+duplicating, and no already-collected record is lost. The cost of the current
+rule is **wasted work and staleness, never data loss.** Any change here must
+preserve that property — the current rule is conservative in the right
+direction, and a careless relaxation would trade a real-but-bounded cost for an
+unbounded one.
+
+## The rule today, and why it exists
+
+`spec-collection-profile.md` states it twice, and deliberately:
+
+- Line 151: "The runtime MUST NOT persist STATE checkpoints from a run that
+ terminates in the `failed` state, except for the certified stream-scoped
+ failure described under DONE. State is otherwise persisted only after a
+ successful DONE."
+- Line 497: "A missing or mismatched terminal code, a missing or untargeted
+ skip, an out-of-scope stream, a protocol violation, an invalid terminal count
+ or exit code, **a process exit without valid DONE**, or cancellation MUST
+ preserve the default fail-closed rule and persist no staged STATE."
+
+Line 497 covers a restart directly — a restart *is* a process exit without
+valid DONE. There is no `abandoned` loophole; the case was considered and
+closed. The rule is correct in its purpose: a cursor must not advance past
+records whose detail coverage was never proven. `DETAIL_COVERAGE` is evaluated
+at DONE, and a detail stream may emit coverage for a parent long after that
+parent's own STATE message. Committing a parent early could advance past
+unhydrated detail that no later run would revisit — converting today's honest
+re-fetch into silent data loss.
+
+An implementation attempt confirmed this is load-bearing, not incidental:
+committing at STATE time broke 19 existing tests in
+`test/collection-profile.test.ts`, including the explicit contract "STATE is
+only committed when DONE status is succeeded". It was reverted.
+
+## What Changes
+
+A narrow exception, for a restart only, limited to checkpoints whose coverage
+is already PROVEN at the moment the STATE message is handled.
+
+- Add a terminal disposition for a run ended by controller restart, distinct
+ from `failed`: the run's outcome was never observed, rather than observed to
+ be bad. (This mirrors the distinction already drawn on the read side, where
+ restart-abandoned runs no longer classify connection health.)
+- Permit a runtime to persist a staged checkpoint at STATE time **only when
+ every one of these holds**:
+ 1. The checkpoint stream is not named as a detail parent by any in-scope
+ stream's manifest `parent_streams`/`state_stream` declaration — so no
+ DONE-time `DETAIL_COVERAGE` verdict can ever apply to it. This is
+ decidable from the manifest before the run starts, and is exactly the
+ predicate `missingDetailCoverageReports` already uses.
+ 2. The records the cursor covers are already durably ingested.
+ `handleStateMessage` already awaits `flushBatch(stateStream)` before
+ staging, so this holds today at that point.
+ 3. The connector declared no gap for that stream this run.
+- Keep the fail-closed default for every other case: any stream that could
+ shortfall, any protocol violation, any connector-reported failure, and any
+ owner cancellation continue to persist nothing.
+
+Under this exception a restart-interrupted Slack walk resumes near minute 50
+instead of zero, while a list+detail connector like ChatGPT — whose
+`conversations` checkpoint gates `messages` detail — is unaffected and keeps
+committing only at DONE.
+
+## What this does NOT propose
+
+- No change to the `failed`-run rule. A connector that reports failure still
+ commits nothing beyond the existing certified stream-scoped exception.
+- No change to cancellation. An owner cancel still persists nothing.
+- No weakening of the coverage gate. A stream that could shortfall is excluded
+ by construction, not by a runtime judgement call.
+- No trust in connector self-declaration. Eligibility is derived from the
+ manifest and the runtime's own flush ordering, never from a connector flag —
+ a manifest claim of "safe to commit early" would be voluntary honesty, which
+ this program has been burned by before.
+
+## Alternatives considered
+
+- **Do nothing.** Defensible, because idempotency bounds the cost to wasted
+ work. Rejected as the default answer because a walk longer than the interval
+ between deploys never converges — the owner cannot get a complete Slack
+ archive, which is a data-completeness failure, not a performance one.
+- **Shorten the walks instead.** Real, and worth doing independently, but it
+ does not fix restart-during-walk; it only narrows the window.
+- **Graceful drain on SIGTERM.** Already built, deployed, measured, and
+ removed in `2ddcca1b8` — production logged `drained:0, elapsedMs:5000,
+ timedOut:1`. Docker's stop timeout is 10s and fixed at container creation
+ while real runs take minutes, and a `kill -9` gets no drain at all. Not a
+ viable path; see `design-notes/graceful-drain-verdict-2026-08-22.md`.
+
+## Impact
+
+- Affects `spec-collection-profile.md` (normative), and `runtime/index.ts`
+ `handleStateMessage` once the spec permits it.
+- The eligibility predicate is manifest-derived and computable before the run
+ starts, so conformance is testable offline.
+- Verification should follow D15: kill a run mid-walk, then prove the next run
+ re-fetches nothing before the last committed boundary AND that a
+ detail-parent checkpoint did not advance.
+
+## Owner decision required
+
+The owner gates spec changes. The question is narrow: **may a checkpoint whose
+coverage is already proven survive a restart, when the alternative is that
+long walks never converge?** The safety property that makes this askable is
+that eligibility is decided from the manifest, not from connector claims.
diff --git a/openspec/changes/allow-resumable-checkpoints-after-restart/specs/reference-implementation-architecture/spec.md b/openspec/changes/allow-resumable-checkpoints-after-restart/specs/reference-implementation-architecture/spec.md
new file mode 100644
index 000000000..37a0b436c
--- /dev/null
+++ b/openspec/changes/allow-resumable-checkpoints-after-restart/specs/reference-implementation-architecture/spec.md
@@ -0,0 +1,69 @@
+## MODIFIED Requirements
+
+### Requirement: Checkpoint Persistence Across a Controller Restart
+
+The runtime SHALL continue to fail closed by default: a run that ends in a
+protocol violation, a connector-reported failure, an owner cancellation, or any
+process exit without a valid DONE SHALL persist no staged STATE, except as
+provided below.
+
+As a narrow exception, the runtime MAY persist a staged checkpoint at the time
+its `STATE` message is handled, so that a run interrupted by a controller
+restart resumes from that checkpoint rather than from zero. The runtime SHALL
+apply the exception to a checkpoint stream ONLY when all of the following hold:
+
+1. No in-scope stream declares that checkpoint stream as a detail parent, via
+ the manifest's `state_stream` or `parent_streams` declaration. Eligibility
+ SHALL be derived from the manifest alone and SHALL be decidable before the
+ connector is spawned. The runtime SHALL NOT accept a connector-supplied
+ claim of eligibility.
+2. Every record the cursor covers is already durably ingested at the moment the
+ checkpoint is persisted.
+3. The connector has reported no gap for that stream in the current run.
+
+A checkpoint stream that fails any condition SHALL remain staged and SHALL
+commit only under the existing successful-DONE rule, so that a DONE-time
+`DETAIL_COVERAGE` verdict can never be bypassed.
+
+The runtime SHALL make an eagerly persisted checkpoint distinguishable from one
+that is merely staged, so a reader of the run timeline can tell which
+checkpoints survive a restart.
+
+A run ended by a controller restart SHALL be reported with a terminal
+disposition distinct from `failed`, reflecting that its outcome was never
+observed rather than observed to be bad.
+
+#### Scenario: An interrupted run resumes from its last eligible checkpoint
+
+- **WHEN** a connector emits records for an eligible checkpoint stream, then a
+ `STATE` message for that stream, and the process then exits without a valid
+ DONE
+- **THEN** the runtime SHALL have persisted that stream's cursor durably
+- **AND** the next run SHALL resume from that cursor rather than from zero
+- **AND** the next run SHALL NOT re-fetch records committed before that cursor
+
+#### Scenario: A detail parent's checkpoint never commits early
+
+- **WHEN** an in-scope stream declares a checkpoint stream as its detail parent
+- **AND** the connector emits a `STATE` message for that parent stream
+- **THEN** the runtime SHALL NOT persist that parent's cursor at `STATE` time
+- **AND** the parent's cursor SHALL commit only after a successful DONE whose
+ `DETAIL_COVERAGE` accounting is complete
+
+#### Scenario: Fail-closed defaults are unchanged
+
+- **WHEN** a run terminates with a connector-reported failure, an owner
+ cancellation, an invalid terminal count or exit code, or a protocol violation
+- **THEN** the runtime SHALL persist no staged STATE beyond the existing
+ certified stream-scoped failure exception
+- **AND** an ineligible checkpoint stream SHALL persist nothing in every such
+ case
+
+#### Scenario: Re-collection after a restart cannot duplicate records
+
+- **WHEN** a run interrupted by a restart re-collects a range whose records were
+ already durably ingested
+- **THEN** ingest SHALL upsert on the existing record identity rather than
+ create duplicates
+- **AND** the observable cost of an interrupted run SHALL be repeated work only,
+ never lost or duplicated records
diff --git a/openspec/changes/allow-resumable-checkpoints-after-restart/tasks.md b/openspec/changes/allow-resumable-checkpoints-after-restart/tasks.md
new file mode 100644
index 000000000..45b97ccf6
--- /dev/null
+++ b/openspec/changes/allow-resumable-checkpoints-after-restart/tasks.md
@@ -0,0 +1,53 @@
+## 1. Owner decision (blocking)
+
+- [ ] Owner rules on the narrow exception in `proposal.md`. Nothing below starts
+ until the spec change is accepted — the current rule is normative and was
+ deliberately written to cover this case (`spec-collection-profile.md`
+ lines 151 and 497).
+
+## 2. Spec
+
+- [ ] Amend `spec-collection-profile.md` to admit the restart exception, stating
+ the three eligibility conditions and keeping the fail-closed default for
+ every other terminal shape.
+- [ ] Name the restart disposition explicitly in the spec's terminal vocabulary,
+ distinct from `failed` (outcome never observed vs. observed bad).
+
+## 3. Implementation
+
+- [ ] Add the manifest-derived eligibility predicate (a checkpoint stream named
+ as a detail parent by any in-scope stream is ineligible), reusing the same
+ reading of `parent_streams`/`state_stream` that
+ `missingDetailCoverageReports` already applies.
+- [ ] Commit an eligible checkpoint from `handleStateMessage`, after the
+ existing `flushBatch(stateStream)` await.
+- [ ] Report the checkpoint as already-durable in `run.state_staged`, so the
+ timeline distinguishes a checkpoint that survives a restart from one that
+ only survives to DONE.
+- [ ] Leave the DONE-time commit path unchanged for every ineligible stream.
+
+## 4. Tests
+
+- [ ] Prove a connector that emits STATE and then exits WITHOUT DONE leaves its
+ cursor durable — read back over `GET /v1/state/:connectorId`, the same
+ surface the next run reads. Simulate the restart by killing the child
+ after a flushed write; do not use a sleep or any timing proxy.
+- [ ] Prove a detail-parent checkpoint does NOT commit early, so an unproven
+ coverage verdict can never be skipped.
+- [ ] Prove the existing contracts still hold: `DONE(failed)`,
+ `DONE(cancelled)`, and every protocol-violation case in
+ `test/collection-profile.test.ts` still commit nothing.
+- [ ] Mutation-prove each new test in both directions (remove the eager commit;
+ remove the safety predicate). A predicate mutation MUST fail the
+ detail-parent test — an earlier attempt at this work had a safety test
+ that passed with the guard removed, because its fixture manifest was
+ being rejected and the assertion never exercised the path.
+
+## 5. Validation
+
+- [ ] D15 canary: kill a real run mid-walk, then verify the next run re-fetches
+ nothing before the last committed boundary and that no detail-parent
+ cursor advanced.
+- [ ] Confirm idempotency still bounds the failure direction: re-walking an
+ already-committed range re-upserts via
+ `records_connector_instance_stream_key` rather than duplicating.
diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/design.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/design.md
new file mode 100644
index 000000000..c8f12f8ca
--- /dev/null
+++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/design.md
@@ -0,0 +1,116 @@
+## Context
+
+`runRequestedStreams` (`packages/polyfill-connectors/connectors/slack/
+index.ts:2568`) runs the messages/reactions/message_attachments unified pass
+via `runMessagesUnifiedPass`, then declared coverage for all three streams
+before this change. It is called from two places:
+
+1. `collect()`'s main flow (~line 2925), once per run, against the base
+ archive.
+2. `mergeScopedMessageArchivePasses` (~line 1347), inside a `for (const
+ archive of deps.scopedArchives)` loop, once per scoped archive that
+ `reconcileMessageSourceCache` selected to heal a channel missing from the
+ base archive.
+
+Call site 2 only runs when `messageFamilyRequested && isUnscopedMessageBoundary
+&& reconciledSourceCache.scopedArchives.length > 0` (collect(), ~line 2932).
+Production's base archive drops channels over time (export-scope drift,
+workspace membership changes); `reconcileMessageSourceCache` heals each
+missing channel from its own existing scoped archive. Production has always
+needed 3 scoped archives to heal, so call site 2's loop has always run 3
+times per run, each one calling `runRequestedStreams` again on the SAME
+shared `emit`.
+
+`runRequestedStreams`, before this change, emitted DETAIL_COVERAGE for
+`messages` (self-coverage) and for `reactions`/`message_attachments`
+(family coverage, via `declareMessageFamilyCoverage`) unconditionally,
+every time it ran the messages/reactions/message_attachments branch. So one
+base-archive call (site 1) plus 3 scoped-archive calls (site 2's loop) =
+4 emissions of each (state_stream, stream) pair through the same `emit`.
+`reference-implementation/runtime/index.ts:3128`'s `trackDetailCoverage`
+tracks `(state_stream, stream)` pairs it has already seen this run and
+throws `Connector emitted duplicate DETAIL_COVERAGE for
+state_stream=${stateStream} stream=${stream}` on any repeat. Every Slack
+run touching 2+ scoped archives has hit this and failed; 8/8 recorded runs.
+
+## Decisions
+
+### (a) Move the emission out of `runRequestedStreams`, into `collect()`, after the fold
+
+`runRequestedStreams` returns a `MessagesPassResult` (`{ channelMaxTs,
+maxMessageTs, considered }`) whether or not it's the function emitting
+coverage. `mergeMessagesPassResults` (line ~849) already sums `considered`
+across every result it merges — `merged.considered` (or in `collect()`,
+`messageResult.considered` after the merge assignment) is the correct
+summed denominator across the base archive plus every scoped archive
+folded in this run.
+
+The fix: `runRequestedStreams` no longer emits `messages`/`reactions`/
+`message_attachments` DETAIL_COVERAGE at all. `collect()` emits it exactly
+once, unconditionally, via a new `declareMergedMessageCoverage(deps,
+messageResult.considered)` helper, placed after both:
+
+- the base-archive `runRequestedStreams` call, and
+- the conditional `mergeScopedMessageArchivePasses` fold (which reassigns
+ `messageResult` to the folded total when it runs).
+
+`declareMergedMessageCoverage` internally no-ops when the message family
+wasn't requested this run (mirrors the guard `runRequestedStreams` used to
+apply inline), so `collect()` calls it unconditionally rather than adding
+an `if` at the call site — this keeps `collect()`'s branch count from
+growing (it was already at the connector's biome cognitive-complexity
+ceiling).
+
+### (b) Why not gate `runRequestedStreams`'s emission on "am I the last call"
+
+An alternative considered: keep the emission inside `runRequestedStreams`
+but only emit when it's the final call in a fold (e.g. thread an `isLast`
+flag through the loop). Rejected: it requires the loop's caller
+(`mergeScopedMessageArchivePasses`) to know about `runRequestedStreams`'s
+internal emission timing, and it does nothing for the fact that the base
+archive's call (site 1, outside the loop entirely) ALSO emits — the
+base-archive emission and the final scoped-archive emission would still be
+two separate emissions of the same pair. The loop and the base call are two
+independent call sites; the only place that has ever seen the FINAL summed
+`considered` for both is `collect()`, after both have run. Correctness
+requires the emission to live where the merge result is observed, not
+inside either producer.
+
+### (c) Single-archive path is unchanged
+
+When `reconciledSourceCache.scopedArchives.length === 0` (the common case:
+no channel needs healing this run), `mergeScopedMessageArchivePasses` never
+runs and `messageResult` is exactly what the base-archive
+`runRequestedStreams` call returned — the same value the removed inline
+emission used to read. `declareMergedMessageCoverage` still runs exactly
+once, with the same `considered`/`covered` value as before this change.
+This is the case `slack-collection-report.test.ts` and
+`canvases-considered.test.ts`-style unit coverage exercise; the constraint
+is that this change is behavior-invisible on that path.
+
+### Acceptance checks
+
+- 0 scoped archives: exactly one messages/reactions/message_attachments
+ DETAIL_COVERAGE emission, `considered` equal to the base archive's row
+ count — identical to pre-change behavior.
+- 1 scoped archive: pre-fix, `runRequestedStreams` ran twice (base + the one
+ scoped archive), emitting each pair twice — `reference-implementation/
+ runtime/index.ts`'s `trackDetailCoverage` would reject the second
+ occurrence. This was never actually observed failing in
+ `archive-reclaim.test.ts`'s existing single-scoped-archive coverage
+ because that harness (`runConnectorProtocolSubprocess`) spawns the
+ connector standalone and never runs it through the RI runtime process
+ that owns `trackDetailCoverage` — so a duplicate emission there is
+ silently accepted by the test harness, not proof the pair is duplicate
+ -safe in production. Post-fix: exactly one emission per pair, `considered`
+ equal to base + scoped row count.
+- 2+ scoped archives: this is the shape production has always needed (3
+ scoped archives every run) and the shape that has never once completed —
+ pre-fix, N+1 emissions of each pair (base + N scoped archives), and the
+ RI runtime throws on the 2nd. Post-fix: exactly one emission per pair,
+ `considered` equal to the summed row count across base + every scoped
+ archive folded. `scoped-archive-coverage.test.ts` drives the real
+ connector end-to-end (`PDPP_SLACK_SKIP_SLACKDUMP=1`, 2 distinct scoped
+ archives, full messages/reactions/message_attachments scope) and asserts
+ directly on the emitted DETAIL_COVERAGE sequence — this is the
+ regression test for this change.
diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/proposal.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/proposal.md
new file mode 100644
index 000000000..d910d5636
--- /dev/null
+++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/proposal.md
@@ -0,0 +1,56 @@
+## Why
+
+The Slack connector has 8 recorded production runs and 0 successes. Every
+run that folds 2+ scoped archives (`packages/polyfill-connectors/connectors/
+slack/index.ts`, `mergeScopedMessageArchivePasses`, ~line 1364) dies with
+`reference-implementation/runtime/index.ts:3135`'s `Connector emitted
+duplicate DETAIL_COVERAGE for state_stream=messages stream=messages`.
+Production touches 3 scoped archives per run.
+
+`mergeScopedMessageArchivePasses`'s `for (const archive of
+deps.scopedArchives)` loop calls `runRequestedStreams` once per archive.
+Before this change, `runRequestedStreams` itself emitted the messages
+self-coverage `DETAIL_COVERAGE` (`state_stream=messages`, `stream=messages`)
+and the message-family `DETAIL_COVERAGE` (`state_stream=messages`,
+`stream=reactions|message_attachments`, via `declareMessageFamilyCoverage`)
+on every call. With N scoped archives this emitted each (state_stream,
+stream) pair N times through the same `emit` side-channel. `polyfill-runtime`
+already requires a connector emit `DETAIL_COVERAGE` exactly once per run
+(see `openspec/specs/polyfill-runtime/spec.md`, "Connectors with a detail
+lane SHALL emit DETAIL_COVERAGE once per run") — Slack's scoped-archive fold
+violated it.
+
+The summed denominator this emission needs already exists:
+`mergeMessagesPassResults` (line ~849) sums `considered` across archives,
+and the loop's `merged` accumulator (returned at the end of
+`mergeScopedMessageArchivePasses`) carries the correct total. The emission
+was simply happening at the wrong call site — inside the per-archive
+function, instead of once after the fold completes.
+
+## What Changes
+
+- Remove the messages self-coverage and message-family `DETAIL_COVERAGE`
+ emission from inside `runRequestedStreams`.
+- Add a single post-fold emission in `collect()`, using
+ `messageResult.considered` — the value after `mergeScopedMessageArchivePasses`
+ (when it runs) has folded every scoped archive into the base archive's
+ total. When no scoped-archive fold happens (the ordinary single-archive
+ run), `messageResult` is just the base archive's own result, so the
+ emitted value is unchanged from today.
+- No change to `mergeMessagesPassResults`, `mergeScopedMessageArchivePasses`'s
+ fold loop, or the manifest's `state_stream`/`coverage_strategy` declarations
+ for `reactions`/`message_attachments`.
+
+## Capabilities
+
+Modified:
+- `polyfill-runtime`
+
+## Impact
+
+- Affects `packages/polyfill-connectors/connectors/slack/index.ts` only.
+- Fixes every Slack run that touches 2+ scoped archives (the only shape
+ production has ever exercised past the first channel-recovery run) —
+ these have never once completed.
+- No change to the single-scoped-archive or no-scoped-archive cases: same
+ emission count (one), same `considered`/`covered` value as today.
diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/specs/polyfill-runtime/spec.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/specs/polyfill-runtime/spec.md
new file mode 100644
index 000000000..dc7622d3d
--- /dev/null
+++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/specs/polyfill-runtime/spec.md
@@ -0,0 +1,64 @@
+## MODIFIED Requirements
+
+### Requirement: Connectors with a detail lane SHALL emit DETAIL_COVERAGE once per run
+
+A connector that runs a list+detail lane SHALL emit exactly one `DETAIL_COVERAGE`
+message per run, after the detail lane completes. A list+detail lane is one that
+fetches a list of records and then fetches per-record detail for at least a
+subset of those records. The message SHALL carry:
+
+- `stream`: the detail stream name.
+- `state_stream`: the list/parent stream whose cursor anchors the detail pass.
+- `required_keys`: the full set of record keys the connector considered for
+ detail fetch in this run.
+- `hydrated_keys`: the subset of `required_keys` for which detail was
+ successfully fetched and emitted.
+- `gap_keys` (optional): keys for which a `DETAIL_GAP` was emitted.
+- `optional_skip_keys` (optional): keys skipped by explicit policy (e.g.
+ rate-limited voluntarily, filtered by selection scope).
+
+Connectors that emit only flat streams with no per-record detail fetch are
+exempt from this requirement.
+
+When a connector internally folds multiple independent sources into one
+run's coverage for a `(state_stream, stream)` pair — for example, healing
+missing partitions from separate archives and merging their results — the
+"once per run" requirement applies to the run's single externally observed
+DETAIL_COVERAGE emission for that pair, not to each internal source the
+connector folds. The connector SHALL compute the merged denominator (e.g. a
+summed `considered`) across every source folded, then emit exactly once
+using that merged value. An internal per-source pass SHALL NOT itself emit
+DETAIL_COVERAGE for a pair that the run also emits once, merged.
+
+#### Scenario: list+detail run emits DETAIL_COVERAGE after the detail lane
+
+**WHEN** a connector completes a list+detail run
+**THEN** the connector SHALL emit a `DETAIL_COVERAGE` message
+**AND** the message SHALL appear after the last RECORD or DETAIL_GAP emitted by
+the detail lane in the same run
+**AND** `required_keys` SHALL equal the set of keys the connector scanned for
+detail
+
+#### Scenario: fully hydrated run emits DETAIL_COVERAGE with no gap_keys
+
+**WHEN** a list+detail run completes with no DETAIL_GAP messages
+**THEN** `DETAIL_COVERAGE.hydrated_keys` SHALL equal `DETAIL_COVERAGE.required_keys`
+**AND** `gap_keys` SHALL be absent or empty
+
+#### Scenario: partially hydrated run carries gap_keys matching emitted DETAIL_GAPs
+
+**WHEN** a list+detail run emits N DETAIL_GAP messages
+**THEN** `DETAIL_COVERAGE.gap_keys` SHALL contain those N keys
+**AND** `hydrated_keys` SHALL NOT contain keys that also appear in `gap_keys`
+
+#### Scenario: a run folds multiple internal sources into one coverage pair
+
+**WHEN** a connector's run internally reads from 2 or more independent
+sources (e.g. a base archive plus one or more supplementary archives healing
+missing partitions) that each contribute to the same `(state_stream,
+stream)` DETAIL_COVERAGE pair
+**THEN** the connector SHALL merge each source's contribution into one
+denominator (e.g. summed `considered`) before emitting
+**AND** the connector SHALL emit that `(state_stream, stream)`
+DETAIL_COVERAGE pair exactly once for the run, using the merged denominator
+**AND** no internal per-source pass SHALL emit that pair on its own
diff --git a/openspec/changes/fix-slack-scoped-archive-coverage-duplication/tasks.md b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/tasks.md
new file mode 100644
index 000000000..cb7720b04
--- /dev/null
+++ b/openspec/changes/fix-slack-scoped-archive-coverage-duplication/tasks.md
@@ -0,0 +1,24 @@
+## 1. Reproduction test (write first, before touching the connector)
+
+- [x] Add `packages/polyfill-connectors/connectors/slack/scoped-archive-coverage.test.ts`: seed a base archive missing 2 channels, each recoverable from its own separate pre-existing scoped archive, drive the real connector via `runConnectorProtocolSubprocess` (`PDPP_SLACK_SKIP_SLACKDUMP=1`) with `messages`/`reactions`/`message_attachments` in scope, and assert each `(state_stream, stream)` DETAIL_COVERAGE pair appears exactly once with the summed `considered`.
+- [x] Confirm the test fails against unmodified `connectors/slack/index.ts` (3 emissions of the `messages` pair: base + 2 scoped archives), then leave it in the suite.
+
+## 2. Fix
+
+- [x] Remove the messages self-coverage and message-family DETAIL_COVERAGE emission from inside `runRequestedStreams`.
+- [x] Add `declareMergedMessageCoverage(deps, considered)`, gated internally on the message family being requested (mirrors the removed inline guard), emitting the self-coverage then delegating to the existing `declareMessageFamilyCoverage`.
+- [x] Call `declareMergedMessageCoverage` once from `collect()`, after both the base-archive `runRequestedStreams` call and the conditional `mergeScopedMessageArchivePasses` fold, using the (possibly folded) `messageResult.considered`.
+
+## 3. Existing test repair
+
+- [x] Update `integration.test.ts`'s `"runRequestedStreams: archive message enumeration bounds both derived streams"` test, which asserted DETAIL_COVERAGE emission from inside `runRequestedStreams` directly — that emission moved to the caller, so assert on the returned `MessagesPassResult.considered` instead, and assert no DETAIL_COVERAGE is emitted from this call in isolation.
+
+## 4. Validation
+
+- [x] Run the new test; confirm GREEN with the fix applied.
+- [x] Run the full `connectors/slack/**/*.test.ts` suite; confirm no regressions (201/201 pass).
+- [x] Run `reference-implementation`'s `test/slack-collection-report.test.ts` (unaffected — projection layer, not connector).
+- [x] `npm --prefix reference-implementation run typecheck` — clean.
+- [x] `npm --prefix packages/polyfill-connectors run typecheck` — clean.
+- [x] `npx biome check` on every changed file — clean (required extracting `declareMergedMessageCoverage` and gating the message-family check inside it, rather than an `if` in `collect()`, to stay under the cognitive-complexity ceiling).
+- [x] `openspec validate fix-slack-scoped-archive-coverage-duplication --strict` — passes.
diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/design.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/design.md
new file mode 100644
index 000000000..64292cfc0
--- /dev/null
+++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/design.md
@@ -0,0 +1,48 @@
+## Context
+
+The coverage snapshot answers one question: did the collector account for
+everything the server requires? The previous gate also failed when the collector
+accounted for something the server no longer asks about, which is a different
+question and not one that bears on completeness.
+
+## Decision: the missing/unexpected asymmetry is the whole change
+
+**Unexpected is not fatal.** A collector reporting a store this build no longer
+declares scanned *more* than was asked of it. That cannot weaken a completeness
+claim. The parser already excludes unexpected entries from `rows` — it
+`continue`s before pushing — so they cannot corrupt the proof either. Their only
+effect was to fail the gate.
+
+**Missing stays fatal.** A missing store means the collector did not account for
+something the server requires. The snapshot genuinely is not committed, and
+saying otherwise would be the dishonesty this codebase is trying to remove.
+
+**Duplicate and malformed stay fatal.** Both indicate a report that cannot be
+trusted to mean what it says, which is distinct from a report that means more
+than needed.
+
+## Why not fix the device instead
+
+Updating the collector on the affected device resolves this instance and leaves
+the class untouched. Any user whose device and server versions ever diverge —
+which is every user eventually — hits the same wall, and the failure presents as
+"Not measured" with no diagnostic naming the drift. The lenient gate fixes the
+class and fixes this instance without touching the device.
+
+## Prior art
+
+restic exits `0` on full success and `3` on "some source files could not be
+read" — an incomplete-but-usable snapshot is still created. borg distinguishes
+`0` success, `1` warning, `2` error. rclone's `check` reports `--missing-on-dst`,
+`--differ` and `--match` as separate named categories. None discards a result
+because it is imperfect in one direction; partial coverage is a tagged status on
+still-valid data.
+
+## Acceptance
+
+- Every expected store present plus one unexpected store: snapshot commits,
+ `unexpectedStores` still reports the drift, the unexpected entry never appears
+ in `rows`.
+- A missing required store: snapshot does not commit.
+- A duplicate store: snapshot does not commit.
+- A malformed entry: snapshot does not commit.
diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/proposal.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/proposal.md
new file mode 100644
index 000000000..c46be7773
--- /dev/null
+++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/proposal.md
@@ -0,0 +1,37 @@
+## Why
+
+`packages/polyfill-connectors/src/local-source-inventory.ts` treated an
+unexpected store as fatal to a coverage snapshot:
+
+```ts
+const hasCommittedSnapshot =
+ !malformed && duplicateStores.length === 0 &&
+ unexpectedStores.length === 0 && missingStores.length === 0;
+```
+
+An unexpected store is one the collector reported that the server's descriptor
+table no longer declares — the normal result of a device running a build older
+than the server. One such name set `hasCommittedSnapshot = false`, then
+`reliable = false`, then coverage axis `unknown`, rendered as "Not measured".
+
+Measured on production: connection `cin_ece4bfe5096b8bf67a1468c2` ("peregrine
+Codex") has **1,293,596 collected records**, summary evidence `state=fresh` with
+every component `current`, a current heartbeat and a drained outbox — and
+displays "Not measured". The only drift is a single legacy `logs` store. Nothing
+is missing: every declared store is reported.
+
+## What Changes
+
+- An unexpected store no longer disqualifies a coverage snapshot.
+- `unexpectedStores` is still returned, so the drift stays observable — it
+ becomes informational rather than disqualifying.
+- Missing, duplicate, and malformed remain fatal, unchanged.
+
+## Capabilities
+
+- Modified: local-agent-collector-completeness
+
+## Impact
+
+- `packages/polyfill-connectors/src/local-source-inventory.ts`
+- `packages/polyfill-connectors/src/local-source-inventory.test.ts`
diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/specs/local-agent-collector-completeness/spec.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/specs/local-agent-collector-completeness/spec.md
new file mode 100644
index 000000000..0a8f9c568
--- /dev/null
+++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/specs/local-agent-collector-completeness/spec.md
@@ -0,0 +1,33 @@
+## MODIFIED Requirements
+
+### Requirement: Coverage snapshots SHALL tolerate unexpected stores and refuse missing ones
+
+A local coverage snapshot SHALL commit when every store the server declares is
+accounted for, even if the collector also reports stores the server does not
+declare. A store the server does not declare SHALL NOT contribute to the proof and
+SHALL NOT disqualify it. A snapshot SHALL NOT commit when a declared store is
+missing, when a store is reported more than once, or when an entry is malformed.
+
+#### Scenario: Older collector reports a store the server no longer declares
+
+- **WHEN** a coverage snapshot contains every declared store and one additional
+ store the server does not declare
+- **THEN** the snapshot SHALL commit
+- **AND** the undeclared store SHALL be reported as unexpected
+- **AND** the undeclared store SHALL NOT appear in the accounted rows
+
+#### Scenario: A declared store is absent
+
+- **WHEN** a coverage snapshot omits a store the server declares
+- **THEN** the snapshot SHALL NOT commit
+- **AND** the absent store SHALL be reported as missing
+
+#### Scenario: A store is reported twice
+
+- **WHEN** a coverage snapshot reports the same store more than once
+- **THEN** the snapshot SHALL NOT commit
+
+#### Scenario: An entry is malformed
+
+- **WHEN** a coverage snapshot contains an entry that cannot be parsed
+- **THEN** the snapshot SHALL NOT commit
diff --git a/openspec/changes/make-local-coverage-tolerate-unexpected-stores/tasks.md b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/tasks.md
new file mode 100644
index 000000000..7e6c42f0e
--- /dev/null
+++ b/openspec/changes/make-local-coverage-tolerate-unexpected-stores/tasks.md
@@ -0,0 +1,19 @@
+## 1. Test first
+
+- [x] Failing test: every expected store present plus one unexpected store, asserting
+ the snapshot commits. Red against unmodified source.
+- [x] Regression test: a missing required store still fails closed.
+- [x] Existing regression coverage for duplicate and malformed entries retained.
+
+## 2. Implement
+
+- [x] Remove `unexpectedStores.length === 0` from the `hasCommittedSnapshot` gate.
+- [x] Keep `unexpectedStores` in the returned result so drift stays observable.
+- [x] Document the missing/unexpected asymmetry at the decision site.
+
+## 3. Validate
+
+- [x] Module tests pass (33 pass, 0 fail).
+- [x] Biome clean on changed files.
+- [ ] Full reference-implementation suite green.
+- [ ] `openspec validate make-local-coverage-tolerate-unexpected-stores --strict`
diff --git a/openspec/changes/own-run-lifecycle-state-machine/.openspec.yaml b/openspec/changes/own-run-lifecycle-state-machine/.openspec.yaml
new file mode 100644
index 000000000..d160e09cf
--- /dev/null
+++ b/openspec/changes/own-run-lifecycle-state-machine/.openspec.yaml
@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-08-21
diff --git a/openspec/changes/own-run-lifecycle-state-machine/design.md b/openspec/changes/own-run-lifecycle-state-machine/design.md
new file mode 100644
index 000000000..28d624aa6
--- /dev/null
+++ b/openspec/changes/own-run-lifecycle-state-machine/design.md
@@ -0,0 +1,500 @@
+## Context
+
+The vocabulary for run lifecycle already exists and is mostly correct. What is
+missing is an owner. `run.abandoned` is a first-class terminal event in
+`check-run-terminal.sql` and `run-history-writer.ts`; `run_generation` is a
+textbook Kleppmann fencing token; `finalizeRunCleanup` implements a correct
+compare-and-swap. Each is right on its own. None of them is *the* authority,
+so each had to be invented where it was needed, and the copies drift.
+
+This design is therefore about consolidation, not invention. The states are
+already implied by the terminal set; the transitions are already implied by the
+guards; the fence already ships in `controller_identity`. The work is to state
+them once, in one module, with the database — not a comment — enforcing them.
+
+Every claim below cites code read on `fix/sweep-fairness-and-transformer-bounds`
+at `39d19704a`. Where the brief's figures did not reproduce, the measured value
+is given and the discrepancy is named.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- One closed state set, declared once, that every consumer reads.
+- One writer per transition, named in the table.
+- Every durable transition is a compare-and-swap fenced by owner epoch, failing
+ at the database rather than by convention.
+- Every historical incident cluster is expressible as an illegal transition, or
+ is recorded as a finding explaining why it is not.
+- Property tests that fail before implementation and pass after.
+
+**Non-Goals:**
+
+- Scheduling policy of any kind (D4's guard).
+- A generic "workflow engine." Six boot reconcilers should share a predicate,
+ not an executor — the sibling change already ruled on this and it holds here.
+- Changing observable connector behavior. This is a refactor of truth-keeping
+ (D14).
+
+## (a) The closed state set
+
+Nine states. `run_history.status` is the durable projection of this set, and
+its existing literals map one-to-one — the set is not new vocabulary, it is the
+existing vocabulary closed and named.
+
+| State | Terminal | Durable projection | Meaning |
+|---|---|---|---|
+| `pending` | no | `run_history.status='pending'` | Admitted by the executor, not yet started. Exists so admission has a durable pre-state to CAS against. |
+| `running` | no | `status='running'` | `run.started` emitted; the owning epoch is executing. |
+| `awaiting_interaction` | no | `status='running'` + derived | The connector asked the owner for input and is blocked. Today derived in SQL by `controller-boot.ts:374-387`. |
+| `cancel_requested` | no | `status='running'` + derived | `run.cancel_requested` seen; the executor has not yet stopped. |
+| `succeeded` | **yes** | `status='succeeded'` | `run.completed`. |
+| `failed` | **yes** | `status='failed'` | `run.failed`. An observed failure. |
+| `surface_failed` | **yes** | `status='surface_failed'` | `run.browser_surface_failed`. Terminal pre-launch. |
+| `cancelled` | **yes** | `status='cancelled'` | `run.cancelled`. Owner-initiated. |
+| `abandoned` | **yes** | `status='abandoned'` | `run.abandoned`. No owner will ever report. Distinct from `failed` per the landed owner-epoch change. |
+
+`pending` is the one addition. Everything else already exists as a
+`toTerminalStatus` output (`run-history-writer.ts:91-108`) or a `status`
+literal. `awaiting_interaction` and `cancel_requested` are named here because
+the machine must be able to *refuse* transitions out of them; they project onto
+`running` so no reader changes.
+
+`skipped` is deliberately **not** a run state. It is written to
+`run_history.status` by the scheduler's pre-run gate
+(`scheduler/pre-run-gate.ts:99,119,141,157,173,193`) and by
+`scheduler/run-executor.ts:555,572,592` for attempts that never started a run.
+A skipped attempt has no `run.started` event and no run to transition — it is a
+*dispatch outcome*, which is planner territory under D4. Recording it in the
+same column as run state is how "scheduler-generated `status:"skipped"` records
+fed into health classification" (the Gmail identity self-poisoning loop) became
+possible. The machine does not adopt it; the migration keeps it writable by the
+planner and forbids the machine from reading it as a run state.
+
+**The set SHALL be declared once.** Today the terminal event set is declared in
+at least **thirteen** places and **six of them disagree**. `lib/spine.ts:1066`
+claims authority in its own comment — "All run-status projection code must read
+from this set; never hardcode subset checks" — and is then not read by the
+divergent copies:
+
+| Declaration | Contents | Verdict |
+|---|---|---|
+| `lib/spine.ts:1066` (claims canonical) | 5 | correct |
+| `stores/run-history-writer.ts:60`, `stores/connector-attention-store.ts:50`, `scripts/repair/adjudicate-orphaned-runs.ts:123`, `controller-boot.ts:707`, `queries/spine/check-run-terminal.sql:15`, `queries/spine/get-run-terminal-event.sql:13`, `db.ts:5992` | 5 | correct |
+| `connector-summary-read-model.ts:1253` | **4 — omits `run.abandoned`** | **defect** |
+| `db.ts:5437` `SPINE_TERMINAL_EVENT_TYPES_SQL` | **4 — omits `run.abandoned`** | **defect** |
+| `postgres-storage.ts:2692,2716,2744,2793` | **4 — omits `run.abandoned`** | **defect** |
+| `connector-summary-evidence-engine.ts:1599,1799` | **4 — omits `run.abandoned`** | **defect** |
+| `lib/postgres-spine.ts:570` | **4 — omits `run.browser_surface_failed`** | **defect** |
+| `postgres-storage.ts:2336` | **4 — omits `run.browser_surface_failed`** | **defect** |
+
+`db.ts:5433` states it is "kept in sync with" `connector-summary-read-model.ts`.
+The two agree with each other and both disagree with `lib/spine.ts`. The
+observable consequence: an abandoned run is invisible to the connector-summary
+fold and to the `connector_instance_id` backfill the fold's partial index
+serves. The 121 runs adjudicated by the sibling change are exactly the
+population this omission hides. Separately, four PostgreSQL-side declarations
+omit a *different* member than the SQLite-side ones, so the two backends
+disagree about what "terminal" means — cluster 4's defect class, in the
+terminal vocabulary itself.
+
+A comment asking two constants to stay in sync is not a mechanism. This is the
+completeness test for D3 working as intended: writing the state set down found
+six live divergences that no test catches.
+
+### Audit: every side-state that lives beside run state
+
+Verdicts are **absorb** (becomes a real state), **derive** (computed from the
+machine, never stored), or **delete**.
+
+| Side-state | Location | Verdict | Rationale |
+|---|---|---|---|
+| `activeRuns` map | `controller.ts:981` | **derive** | Currently the real admission authority; `finalizeRunCleanup:3249` hand-rolls a CAS on it. Becomes a read-through cache of `state ∈ {pending, running}` fenced by epoch. Its correctness stops depending on process memory. |
+| `activeRunPromises` | `controller.ts:986` | **keep, out of scope** | Holds a JS promise, not state. "Await this run" is a runtime affordance with no durable meaning. Not a second truth. |
+| `settledRunIds` | `controller.ts:997` | **delete** | Exists only to answer "did finalize already run?" — which `state` in a terminal set answers exactly. The `isStale` probe at `:3312` becomes a state read. |
+| `controller_active_runs` | `db.ts:1149`, `postgres-storage.ts:1945` | **absorb** | The durable flight table. Becomes the `pending`/`running` rows of the machine rather than a parallel table. `reconcileBrowserSurfaceLeasesAfterBoot` reads it and keeps working — it is being re-homed, not removed. |
+| `run_generation` | `db.ts:1156`, `controller.ts:1046` | **absorb → replaced by `owner_epoch`** | A per-instance monotonic fencing token, correct in kind but reinvented and *in-process* (`runGenerations` Map, cleared only by test reset). The owner epoch is the same idea already durable in `controller_identity`. One fence, not two. Note: the run-generation column shipped to SQLite before Postgres once — the exact dual-backend gap this design forbids. |
+| `runWatchdogSettlements` | `controller.ts:1031` | **derive** | Watchdog fires ⇒ attempt an `abandoned` transition. If the CAS loses, the run already terminalized. Removes the timer/completion race entirely. |
+| `needsHumanAttention` | `controller.ts:1055` | **delete (not absorb)** | In-memory `Set`, lost on restart, and **not a run state** — it is a per-*connection* automation-suppression policy. Absorbing it would smuggle scheduling policy into the machine, violating D4. It moves to the scheduler's own policy input, derived from terminal runs whose reason is an unresolved interaction. |
+| `cooling_off` / `blocked` | `scheduler-backoff.ts:252` | **delete from run state** | Already not run state — a `recommendedHealthState` derived from a failure streak. Named here to record it as correctly-placed policy, and to forbid it moving in. |
+| `scheduler_dispatch_wedged` | `scheduler.ts:197-215` | **derive** | A synthetic `status:"failed"` `RunRecord` fabricated when the pre-launch gate misses its liveness ceiling. Under the machine this is a real `failed` transition with `terminal_reason='scheduler_dispatch_wedged'`, not a fabricated record. The word "wedged" survives as a reason, not a shadow state. |
+| `run_history.status` | `db.ts:1365` | **absorb** | The durable projection. Gains `owner_epoch` (below). |
+| `scheduler_managed` | `db.ts:1379` | **keep, out of scope** | Provenance, not lifecycle. Marks which writer touched the row; scheduler cadence readers filter on it. Orthogonal. |
+| `manual_upload_artifacts.owner_epoch` | landed sibling | **precedent** | Not run state. Cited because it is the pattern this design generalizes: an epoch column that replaced a wall clock. |
+
+### F1 cluster completeness (D3's test)
+
+| Cluster | Expressible as illegal transition? |
+|---|---|
+| GroupMe 503 — dispatch probe vs. active run | **Yes.** Planner attempting any write while `state ∈ {pending, running}`. Forbidden transition F1 below. |
+| Ingest 503 never retried (Aug recurrence) | **Yes**, partially. The batch-level retry is HTTP policy, but the *observable* defect — a run reporting `failed` while records were durably committed — is `running → failed` attempted by a non-owner. Forbidden F5. |
+| YNAB stuck-run wedge (`activeRuns` leak) | **Yes.** A run with no live owner epoch stuck non-terminal; `abandoned` becomes reachable, so the 409 cannot be permanent. Forbidden F6. |
+| Collector-runner drain boundary race | **Yes.** Two clock reads with opposite boundary semantics; under CAS the second read cannot act on the first read's stale premise. Forbidden F3. |
+| Run-generation fencing / stale controller write | **Yes.** Precisely the epoch CAS. Forbidden F2. |
+| Controller-restart misclassification (`failed` vs `abandoned`) | **Yes.** `running → failed` by the boot path is forbidden; only `running → abandoned` is legal there. Forbidden F4. |
+| **Maintenance-sweep shared-deadline starvation (×3)** | **NO — reported as a finding.** |
+
+**The cluster that does not fit.** The maintenance-sweep starvation family
+(2026-08-01 ×2, 2026-08-03) is not a run-lifecycle defect. No illegal run
+transition occurs: every run in those incidents was in a legal state the whole
+time. The defect is that a *shared budget* was divided unfairly across
+page-mates, so some connections never got serviced. Forcing it into this table
+would require modelling scheduler fairness as run state — which is exactly the
+policy-in-the-machine error D4 forbids. Recording it as out of scope is the
+honest answer, and it belongs to the sweep-fairness work already in flight on
+this very branch. The state set is not missing a state on its account.
+
+## (b) The legal transition table
+
+**Single writer per transition.** "Executor" means the one owner module this
+design mandates; today its responsibilities are split across `runtime/index.ts`,
+`runtime/controller.ts`, and `runtime/scheduler/run-executor.ts`.
+
+| # | Transition | Precondition | Single writer |
+|---|---|---|---|
+| T1 | `∅ → pending` | No non-terminal run for this `connector_instance_id` | Executor (admission) |
+| T2 | `pending → running` | Caller holds the current owner epoch; `run.started` emitted | Executor |
+| T3 | `running → awaiting_interaction` | Connector emitted `run.interaction_required` / `run.assistance_requested` | Executor |
+| T4 | `awaiting_interaction → running` | Interaction reached a terminal interaction event | Executor |
+| T5 | `running → cancel_requested` | Owner requested cancel (`run.cancel_requested`) | Executor, on owner intent |
+| T6 | `running → succeeded` | Connector reported DONE; terminal commit gate passed | Executor |
+| T7 | `running → failed` | Observed failure attributable to this run | Executor |
+| T8 | `running → surface_failed` | Browser surface failed pre-launch | Executor |
+| T9 | `cancel_requested → cancelled` | Executor observed the request and stopped | Executor |
+| T10 | `{pending, running, awaiting_interaction, cancel_requested} → abandoned` | Run's `owner_epoch` is neither the actor's nor the newest boot epoch | **Boot adjudicator** (the sole exception to "executor only"; see (f)) |
+| T11 | `awaiting_interaction → abandoned` | As T10, with reason `controller_terminated_while_awaiting_owner_interaction` | Boot adjudicator |
+
+Terminal states have no outgoing transitions. That is the whole point of the
+set being closed.
+
+### Forbidden transitions, each naming the incident it prevents
+
+| # | Forbidden | Incident prevented |
+|---|---|---|
+| F1 | Any run-state write by the **planner/scheduler** | **GroupMe 503.** `dispatchIfDue` probed `getForwardEvidenceDebt`'s reconcile write against an instance with a run in flight, contending on the per-instance mutex and turning committed batches into `connector_instance_busy` failures. Fixed at `scheduler.ts:636` by a guard whose own comment says it mirrors "the guard `executeRun` already applies one step later" — one rule stated twice. Under F1 the planner has no write path to guard. |
+| F2 | Any transition whose actor's epoch ≠ the run's `owner_epoch` | **Stale-controller writes / run-generation fencing.** A predecessor that resumes after a successor took over must lose *at the database*. Today `run_history` has no epoch column, so this is enforced only in process memory (`controller.ts:1046`). |
+| F3 | Any transition whose observed `state` ≠ the CAS expected state | **Collector-runner drain boundary race.** Two clock reads with opposite boundary semantics (`<=` on claim, `>` on `nextRetryTime`) let a deadline landing between them produce a false "empty" exit. A CAS cannot act on a premise that changed under it. |
+| F4 | `{pending, running, …} → failed` by the boot/adjudication path | **Controller-restart misclassification.** 134 production runs recorded `run.failed`/`controller_restarted`; 55 had staged a cursor and 34 had durably ingested a batch. Interruption is not observed failure. Only T10/T11 are legal there. |
+| F5 | A second terminal transition on an already-terminal run | **Double-terminal / ingest-503 misreporting.** The existing `AND status='running'` fence (`run-history-writer.ts:355`) already provides this for `run_history`; F5 makes it a property of the machine rather than of each writer remembering to add it. |
+| F6 | A run remaining non-terminal with no live owner epoch | **YNAB stuck-run wedge.** A hung subprocess left an `activeRuns` entry forever, 409-ing every future manual run until restart. `abandoned` must always be reachable, so "wedged forever" is unrepresentable. |
+| F7 | Any transition out of a terminal state | Terminal means terminal. Makes `records_emitted` revision-after-terminal — explicitly forbidden by the landed change — structurally impossible. |
+| F8 | Any write to run state not routed through the owner module | **D1.** A `transitionRun()` helper that five modules import is today's distributed writes wearing a uniform. Raw `INSERT`s into `spine_events` (`controller-boot.ts:526`, `:597`) are the current instance of this. |
+
+### Three live defects the table would have prevented
+
+Found while auditing writers. Each is a real, current divergence — reported as
+findings, not fixed here.
+
+1. **An unfenced upsert can overwrite a terminal status.**
+ `queries/controller/insert-run-history.sql:40` and its PostgreSQL twin
+ `stores/scheduler-store.ts:1018` are `ON CONFLICT (run_id,
+ connector_instance_id) DO UPDATE SET status = excluded.status` with **no
+ `status = 'running'` fence**. Every other status writer has one
+ (`run-history-writer.ts:355`, `:406`; `controller-boot.ts:670`, `:791`,
+ `:859`). The scheduler's `appendRunHistory` normally runs *after* the generic
+ writer has already finalized the row, so a scheduler retry can revise an
+ already-terminal outcome. This is forbidden transition **F5** and **F7**,
+ both violated by one statement.
+2. **The PostgreSQL drift repair fences on `run_id` alone.**
+ `controller-boot.ts:829` joins `WHERE h.run_id = t.run_id AND h.status =
+ 'running' AND h.connector_instance_id IS NOT NULL` — an `IS NOT NULL` check,
+ not an equality. Its SQLite twin at `:788-790` correctly fences
+ `AND connector_instance_id = ?`. The codebase documents in several places
+ that `run_id` is not unique across connections, which is why every other
+ writer fences on the pair. A dual-backend asymmetry in the repair path
+ itself — cluster 4's defect class, inside the adjudication code.
+3. **`run_generation` is written from two incompatible sources.**
+ `controller.ts:3139-3187` derives it from a monotonic per-instance counter;
+ `scheduler/run-executor.ts:819` sets it from `attempt`, a retry counter.
+ One column, two meanings. Absorbing the fence into `owner_epoch` (M5)
+ retires the ambiguity rather than arbitrating it.
+
+4. **Two `createReferenceSchedulerManager` definitions exist.** One is inline
+ at `server/index.ts:8828` and is what production calls (`:8378`). The other
+ is exported from `server/scheduler-manager-factory.ts:374`, carries its own
+ copy of all four dispatch probes — including the same
+ `reconcileDirtyConnectorSummaryEvidence` write at `:558` — and is imported
+ by **nothing outside tests**; the two importing tests take only
+ `createRunManagedConnectorViaController`. A parallel copy of the exact
+ subsystem this design governs, whose divergence no test can see. M2 must
+ pick one before cutting writers over. This is D5's masking-pair hazard
+ sitting in the tree already.
+
+A fifth observation, not a defect: **`drainActiveRuns` has no production
+caller.** All 116 call sites are test teardown. Its own doc comments
+(`controller.ts:987`, `server/index.ts:8567`) still describe it as the
+graceful-shutdown path and are stale relative to `server/index.ts:9670-9691`.
+See the migration section for the full verdict.
+
+## (c) The CAS + owner-epoch schema
+
+### Schema change
+
+`run_history` gains one column on both backends, NULL-tolerant so existing rows
+migrate without a backfill:
+
+- SQLite (`server/db.ts`): `owner_epoch TEXT` via the existing
+ `addColumnIfMissing` helper — the same mechanism `run_generation` used at
+ `db.ts:6171`.
+- PostgreSQL (`server/postgres-storage.ts`):
+ `ALTER TABLE run_history ADD COLUMN IF NOT EXISTS owner_epoch TEXT;` — the
+ same shape used for `controller_active_runs.run_generation` at
+ `postgres-storage.ts:1960`.
+
+Verified absent today: `db.ts:1358-1395` and `postgres-storage.ts:2169-2197`
+list every column, and neither has an epoch. The epoch exists only inside the
+spine event's `data_json` (`runtime/index.ts:2745-2749`), which no `UPDATE` can
+fence on cheaply. **Both backends in the same change** — `run_generation`
+shipped to SQLite without Postgres once and was caught only as a deploy
+blocker; the landed owner-epoch change repeats the warning. This design treats
+a single-backend schema change as a defect by construction.
+
+The epoch value is the boot epoch already stashed by
+`emitControllerBootedAndStashEpoch` (`controller-boot.ts:202-231`), whose
+identity half is durable in `controller_identity` (`controller-boot.ts:96-151`,
+one row `id='singleton'`, live in production and verified by reading the file).
+
+### The predicate
+
+Every durable transition is one statement. No read-then-write.
+
+**SQLite** (`better-sqlite3`, synchronous; decide by `.changes`):
+
+```sql
+UPDATE run_history
+ SET status = ?, -- new state
+ completed_at = ?, -- terminal transitions only
+ terminal_reason = ?
+ WHERE run_id = ?
+ AND connector_instance_id = ? -- run_id alone is not unique
+ AND status = ? -- expected state
+ AND (owner_epoch = ? OR owner_epoch IS NULL);
+```
+
+**PostgreSQL** (decide by `rowCount`):
+
+```sql
+UPDATE run_history
+ SET status = $1,
+ completed_at = $2,
+ terminal_reason = $3
+ WHERE run_id = $4
+ AND connector_instance_id = $5
+ AND status = $6
+ AND (owner_epoch = $7 OR owner_epoch IS NULL);
+```
+
+Four properties of this predicate are load-bearing:
+
+1. **`changes`/`rowCount` = 0 means the transition was refused**, and the caller
+ must treat that as an ordinary outcome — someone else already moved the run.
+ It is never retried blindly and never escalated to the owner.
+2. **`connector_instance_id` is part of the fence, not decoration.** `run_id`
+ alone is not unique across connections; both `run-history-writer.ts:355` and
+ `controller-boot.ts:670` already fence on the pair, and this preserves that.
+3. **The NULL arm is spelled out explicitly, not written `IS DISTINCT FROM`.**
+ The landed sibling change was bitten by exactly this: on PostgreSQL,
+ `owner_epoch IS DISTINCT FROM NULL` reduces to `owner_epoch IS NOT NULL` and
+ would spare precisely the legacy rows that most need claiming. Writing
+ `(owner_epoch = $7 OR owner_epoch IS NULL)` is identical on both backends
+ and cannot silently reduce.
+4. **The NULL arm is `OR`, not `AND`.** A legacy row written before the column
+ existed has no claimant, so any epoch may adjudicate it. A row *with* a
+ different epoch may not.
+
+The adjudication transition (T10/T11) inverts arm 4 — it must match rows whose
+epoch is *not* the actor's — and additionally excludes the newest boot epoch, so
+live work is never adjudicated:
+
+```sql
+ AND (owner_epoch IS NULL OR owner_epoch <> :mine)
+ AND (owner_epoch IS NULL OR owner_epoch <> :newest_boot_epoch)
+```
+
+This is the predicate the landed change already proved against production: its
+dry run reported 123 before the newest-epoch exclusion and 121 after, the two
+extras being runs a live container had started ninety seconds earlier.
+
+### Why CAS and not a lock
+
+An advisory lock answers "may I proceed?" at a moment; a CAS answers "was the
+world still as I assumed when I wrote?" — which is the actual question after an
+`await`. This repo already has at least five independently-invented admission
+mechanisms (F1's count), and adding a sixth lock would extend that list. The CAS
+adds no new mechanism: it is a `WHERE` clause on writes that already happen.
+
+## (d) Property-test skeletons
+
+Listed in `tasks.md` §5 with file names. Each is `.todo` or asserts against the
+not-yet-existing owner module, so **none can pass before implementation**. That
+is the requirement: this program has already shipped a conformance test that
+asserted behavior it never exercised, and a guard that cannot fail certifies a
+regression as safe.
+
+Each skeleton names its property, its generator, and its invariant, and each
+maps to a row in the forbidden table above. Dual-backend skeletons run against
+SQLite and PostgreSQL from one body, because a Postgres-only divergence is
+invisible to this suite by construction — that is a named defect class here
+(6,792 tests stayed green while production could not paginate).
+
+## (e) The D4 split, stated precisely
+
+- The **planner** (`runtime/scheduler.ts`, `scheduler/dispatch-governor.ts`,
+ `scheduler-backoff.ts`) READS run state and emits **intents**. It writes no
+ run state, ever.
+- The **executor** is the sole writer of transitions T1-T9, and the boot
+ adjudicator of T10-T11.
+
+**The split does not hold today.** A first pass suggested it nearly did —
+`runtime/scheduler.ts` emits no spine event and contains no literal `UPDATE
+run_history`. That reading was wrong, and the correction matters more than the
+original claim. The scheduler writes run state four ways:
+
+| # | Write | Site |
+|---|---|---|
+| P1 | `INSERT INTO run_history` for skip and back-off records | `scheduler.ts:432`, `:445` (`recordAndNotify` → `schedulerStore.appendRunHistory`), called from `dispatchIfDue:655`, `:662`; SQL at `scheduler-store.ts:570` / `:995` |
+| P2 | `scheduler_last_run_times` upsert | `scheduler.ts:477`; `scheduler-store.ts:1607` |
+| P3 | In-process `activeRuns` set and five announcement-dedup maps | `scheduler.ts:547`, `:576`; `dispatch-governor.ts:559`, `:563`, `:588`, `:590`, `:593` |
+| P4 | **A durable repair write inside the dispatch-eligibility probe** | `server/index.ts:9057` — `getForwardEvidenceDebt` calls `reconcileDirtyConnectorSummaryEvidence([instanceId])` *before* its read at `:9058`; that reconcile takes `withConnectorInstanceWrite` and issues `withPostgresTransaction({ lockConnectorInstanceId })` upserts (`connector-summary-evidence-engine.ts:1281`, `:1866`, `:2691`) |
+
+**P4 is the GroupMe 503 mechanism**, and it is the clearest violation of D4:
+a read-only-looking eligibility probe that takes the per-instance write mutex.
+So F1 must forbid *side-effecting reads*, not merely direct writes. A planner
+that "only reads" but whose read reconciles is a writer.
+
+P1 is the second-order version of the same error: the scheduler writes rows
+that feed back into `runtime.history`, which is the next tick's own decision
+input. The Gmail identity self-poisoning loop is exactly this — scheduler-
+generated `status:"skipped"` records read back as run outcomes.
+
+Under this design: P4 moves to the executor or becomes genuinely read-only.
+P1's records stay writable by the planner but are not run states (see
+`skipped`, above) and the machine never reads them. P2 is planner-owned cadence
+state, unaffected. P3's `activeRuns` set is replaced by an epoch-fenced read of
+the machine — today it is an in-process `Set` that dies with the process, so
+the `dispatchIfDue` guard does not suppress a probe against a run started by a
+different process or surviving a restart.
+
+**The policy guard.** The machine answers *is this transition legal*. It never
+answers *which connector runs next*. Concretely: `cooling_off`, `blocked`,
+backoff curves, fairness rotation, admission deadlines, and per-statement
+budgets stay in the planner. "Runnable" means "no legal impediment to a T1"; it
+does not mean "chosen." `needsHumanAttention` is deleted from the controller
+for exactly this reason — it reads like run state and is actually automation
+policy.
+
+## (f) The D6 formalization
+
+`scripts/repair/adjudicate-orphaned-runs.ts` and
+`reconcileOrphanedRunsAtBoot` (`controller-boot.ts:344`) implement the same
+adjudication with different scopes. The script is the owner-operated backlog
+tool: dry-run by default, ignores `controller_id` (the field that was broken),
+excludes the newest boot epoch, snapshots pre-images into an `aor_backup` table
+inside the write transaction. The boot reconciler is the steady-state path:
+filters on `controller_id`, runs before HTTP routes mount, aborts boot on error.
+
+Under this design the boot reconciler **is** transitions T10/T11 — the same
+predicate, expressed once in the owner module. Three properties must be
+preserved exactly, because this is a behavior-preserving move (D14):
+
+1. **Idempotency stays on `caused_by_event_id`,** via the
+ `spine_run_abandoned_cause_unique` partial index. Repeated passes emit
+ exactly one terminal event per orphan. The existing code catches *only* the
+ named constraint and never blanket-catches `23505` / `SQLITE_CONSTRAINT_UNIQUE`;
+ the owner module must keep that discipline.
+2. **Newest-epoch exclusion stays,** and stays load-bearing. Without it,
+ adjudication declares live work abandoned and frees its resource for a
+ competing run — reintroducing the duplicate-execution hazard the fence
+ exists to prevent.
+3. **`records_emitted` is never revised.** Records durably ingested before the
+ interruption stay committed. F7 makes this structural.
+
+The one real change: the raw `INSERT`s at `controller-boot.ts:526` and `:597`
+that bypass `emitSpineEvent` — and therefore had to hand-write their own
+`run_history` projection at `:657` and `:837` to avoid stranding rows at
+`running` — route through the owner module instead. The event and its
+projection already commit in one transaction; that stays. The bypass is the F8
+violation this design closes.
+
+The repair script remains a script. It is owner-operated, dry-run-first,
+`--limit`-bounded, and deliberately ignores the ownership filter — properties
+of an operational tool, not of a lifecycle transition. It should call the same
+predicate rather than reimplement it, which is the whole benefit.
+
+## (g) The D5 migration plan
+
+Writers cut over atomically per subsystem, old paths deleted in the same change,
+readers migrate after. Never let old and new writers coexist across a deploy
+boundary — every "parallel systems" period in this repo produced a masking pair.
+
+| Tranche | Writers cut over | Deleted in the same change |
+|---|---|---|
+| M1 | Schema + owner module (no callers yet) | nothing — additive only |
+| M2 | Terminal writes: `runtime/index.ts` ×6, `controller.ts` ×2, `terminal-run-commit-store.ts`, `local-device-terminal-collection.ts` | the direct `emitSpineEvent` terminal calls at those sites |
+| M3 | Admission + finalize: `activeRuns`, `settledRunIds`, `runWatchdogSettlements` | `settledRunIds`; hand-rolled CAS at `controller.ts:3249` |
+| M4 | Boot adjudication (T10/T11) | raw `INSERT`s at `controller-boot.ts:526`, `:597` |
+| M5 | Fence unification | `run_generation` column + `runGenerations` map |
+| M6 | Readers: the four terminal-set declarations collapse to one | `TERMINAL_RUN_EVENT_TYPES`, `SPINE_TERMINAL_EVENT_TYPES_SQL` |
+
+M6 is where the `run.abandoned` omission defect is repaired — as a consequence
+of single declaration, not as a separate fix.
+
+### The `drainActiveRuns` collision — verdict: **the brief's premise no longer holds**
+
+The brief asks me to confirm the Sidekiq *quiet* vs *drain* split and say which
+references migrate. Measured on this branch, `39d19704a`:
+
+- **139 total references** — the brief's number reproduces exactly.
+- **3** are in `openspec/changes/adjudicate-interrupted-runs-by-owner-epoch/**`
+ (prose in the landed sibling change).
+- **127** are in `reference-implementation/test/**`, of which **116** are call
+ expressions — overwhelmingly `await controller.drainActiveRuns(1000)` as
+ test teardown.
+- **9** are in source: `runtime/controller.ts` ×7 (one definition at `:4144`,
+ one interface member at `:786`, one export at `:4394`, four comments) and
+ `server/index.ts` ×2 — **both of which are comments, not call sites**.
+
+**There is no shutdown call site left to delete. It is already gone**, removed
+by the landed sibling change. `server/index.ts:8570` now reads "The shutdown
+path deliberately does NOT drain," and `:9686-9689` records that
+"`drainActiveRuns` itself stays on the controller: it means 'await in-flight
+runs' … Sidekiq draws the same line between *quiet* and *drain*."
+
+So: **the split is correct and is already implemented.** My verdict confirms the
+brief's reading of the semantics and corrects its state — this is not open work.
+The residual figures also differ slightly from the sibling change's prose, which
+says "137 references … 136 remain." Today's count is 139 including its own 3
+lines of prose, i.e. 136 outside that change plus 3 inside it. The numbers are
+consistent; the sibling was counting before it wrote about itself.
+
+Which references migrate under this design: **none of the 116 test calls.**
+`drainActiveRuns` means "await in-flight run promises" — a runtime affordance
+over `activeRunPromises` (JS promises, ruled "keep, out of scope" above), not a
+run-state mutation. It never transitions a run. It stays exactly where it is
+and keeps its name.
+
+## Risks / Trade-offs
+
+- [Design authored against a moving branch] -> A concurrent agent owns the
+ scheduler-conformance helper and the store drivers. The single-writer
+ inventory is a point-in-time read of `39d19704a` and must be re-verified
+ before M2 begins. Named as a residual risk, not mitigated here.
+- [`pending` is a new state with no existing projection] -> It is the only
+ addition, and it exists so admission has something to CAS against. If M1
+ measurement shows admission can safely CAS `∅ → running` directly, `pending`
+ should be dropped rather than kept for symmetry — a state that no transition
+ needs is a second truth in waiting.
+- [Absorbing `controller_active_runs` touches browser-surface leases] ->
+ `reconcileBrowserSurfaceLeasesAfterBoot` reads that table to decide which
+ leases are still held. Absorption must preserve that read or re-home it in
+ the same tranche; it is explicitly not a deletion.
+- [Deleting `needsHumanAttention` changes automation behavior] -> It is
+ in-memory and already lost on every restart, so its current behavior is
+ "suppress until restart." Moving it to a derived scheduler policy input makes
+ it durable, which is a *behavior change* and must be measured on the canary
+ rather than assumed neutral. This is the one item in the audit that is not
+ behavior-preserving on its face.
+- [CAS refusals become invisible] -> A transition that returns 0 rows is a
+ normal outcome, which means a bug that refuses *every* transition looks like
+ a quiet system. The canary metrics in `tasks.md` §6 therefore count
+ successful transitions, not just absence of anomalies.
diff --git a/openspec/changes/own-run-lifecycle-state-machine/proposal.md b/openspec/changes/own-run-lifecycle-state-machine/proposal.md
new file mode 100644
index 000000000..826d95672
--- /dev/null
+++ b/openspec/changes/own-run-lifecycle-state-machine/proposal.md
@@ -0,0 +1,142 @@
+## Why
+
+Run state has no owner. It is spread across an append-only spine, a durable
+`run_history` projection, a durable `controller_active_runs` flight table, and
+at least five in-process maps in `runtime/controller.ts` — with no single
+component that decides which transitions are legal.
+
+The consequence is measured, not theorized. Terminal events are emitted from at
+least five modules: `runtime/index.ts` (lines 4071, 5117, 5172, 5367, 5414,
+5563), `runtime/controller.ts` (3707, 4083), `server/stores/terminal-run-commit-store.ts:117`,
+`operations/local-device-terminal-collection.ts:211`, and — bypassing
+`emitSpineEvent` entirely with raw `INSERT`s — `lib/controller-boot.ts`. Each
+site re-derives for itself what a legal terminal write is.
+
+Two facts found while writing this proposal show the cost directly:
+
+1. **The terminal set disagrees with itself.** `check-run-terminal.sql:15` and
+ `get-run-terminal-event.sql:13` list five terminal events including
+ `run.abandoned`. `connector-summary-read-model.ts:1253`
+ (`TERMINAL_RUN_EVENT_TYPES`) and `db.ts:5437`
+ (`SPINE_TERMINAL_EVENT_TYPES_SQL`) list four and omit `run.abandoned` — and
+ `db.ts`'s own comment says the two are "kept in exact sync." They are in
+ sync with each other and out of sync with the spine. An abandoned run is
+ therefore invisible to the connector-summary fold and to the
+ `connector_instance_id` backfill that fold depends on. The change that made
+ `abandoned` a first-class terminal state could not update these because
+ nothing declares the set once.
+
+2. **The epoch cannot fence a durable write.** `run.started` stamps
+ `boot_epoch`, `controller_id`, and `seq` into the spine event's `data_json`
+ (`runtime/index.ts:2745-2749`), but `run_history` has no epoch column on
+ either backend (`db.ts:1358`, `postgres-storage.ts:2169`). Every durable
+ run-state UPDATE is fenced on `AND status = 'running'` alone
+ (`run-history-writer.ts:355`, `:406`; `controller-boot.ts:670`). That fence
+ stops a double-terminal write but cannot stop a *stale epoch's* write: a
+ predecessor container that comes back from a pause still sees `running` and
+ still wins. Fencing exists in-process instead, as a hand-rolled
+ compare-and-swap on a JavaScript map (`controller.ts:3249`) plus a
+ `run_generation` counter (`controller.ts:1046`) — correct, and invisible to
+ the database that actually arbitrates.
+
+The historical incidents are all one shape. The GroupMe 503 was two components
+mutating run state through the same per-instance mutex, fixed by a guard in the
+scheduler (`scheduler.ts:636`) that mirrors "the guard `executeRun` already
+applies one step later" — the same rule stated twice because no component owns
+it. The collector-runner drain race was two clock reads with opposite boundary
+semantics. The YNAB wedge was an `activeRuns` entry that outlived its run and
+409-ed the instance until restart.
+
+This change writes down the machine: a closed state set, a transition table
+whose forbidden entries name the incident each one prevents, and a
+compare-and-swap predicate fenced by the owner epoch that already ships
+durably in `controller_identity` (`lib/controller-boot.ts:96-151`).
+
+## What Changes
+
+This change is **design and tests-first only**. It authors the contract and
+failing property-test skeletons; it writes no implementation. That is
+deliberate — the implementation is single-threaded and starts from a settled
+design.
+
+- Define the **closed run state set** and require it be declared exactly once,
+ so a terminal-set divergence like `TERMINAL_RUN_EVENT_TYPES` becomes a
+ compile-time impossibility rather than a comment asking two constants to
+ agree.
+- Rule on every side-state that lives beside run state today: `activeRuns`,
+ `controller_active_runs`, `run_generation`, `settledRunIds`,
+ `needsHumanAttention`, the scheduler's `cooling_off` and
+ `scheduler_dispatch_wedged` markers — each **absorbed**, **derived**, or
+ **deleted**, with the verdict recorded in `design.md`.
+- Define the **legal transition table**: each transition's precondition and its
+ single writer. Define the forbidden transitions, each annotated with the
+ historical incident it prevents.
+- Require every durable transition to be a **compare-and-swap fenced by owner
+ epoch** — `WHERE state = AND owner_epoch = ` — and specify
+ the predicate for SQLite and PostgreSQL together, since this repo runs both
+ and a Postgres-only design has already shipped silent divergence here once.
+- Add an `owner_epoch` column to `run_history` on both backends, NULL-tolerant,
+ so the fence has a column to read. It does not exist today.
+- State the **D4 split**: the planner reads the machine and emits intents; only
+ the executor transitions. Scheduling *policy* stays out of the machine —
+ "runnable" is not "chosen to run next."
+- Formalize **successor adjudication** as a legal transition executed at boot,
+ replacing the sibling-script framing, with observable behavior unchanged.
+- Author **property-test skeletons**, one per historical incident cluster, each
+ failing or `.todo` and marked as a skeleton. A skeleton that passes without
+ implementation is the hollow-test defect this program exists to kill.
+
+## Capabilities
+
+### New Capabilities
+
+None.
+
+### Modified Capabilities
+
+- `reference-implementation-architecture`: Define the closed run state set and
+ its single declaration, the legal-transition table and its single-writer
+ rule, the epoch-fenced compare-and-swap requirement and its dual-backend
+ predicate, the planner/executor split, and successor adjudication as a
+ lifecycle transition.
+
+## Impact
+
+- Design-only in this change. No runtime file is modified.
+- New failing property-test skeletons under `reference-implementation/test/`,
+ named `run-lifecycle-*.property.test.ts`, excluded from the green gate until
+ the implementation change lands.
+- The implementation this design governs will touch
+ `reference-implementation/runtime/controller.ts`,
+ `runtime/index.ts`, `runtime/scheduler.ts`,
+ `server/stores/run-history-writer.ts`, `lib/controller-boot.ts`,
+ `server/db.ts`, and `server/postgres-storage.ts`.
+- Additive `run_history.owner_epoch` on SQLite and PostgreSQL, NULL-tolerant,
+ so existing databases migrate without a backfill.
+
+## Non-Goals
+
+- **Any implementation.** No writer is cut over here, no state is renamed in
+ code, no migration runs.
+- **Scheduling policy.** Fairness, throttling, backoff curves, and dispatch
+ ordering stay where they are. The machine answers whether a transition is
+ legal, never which connector should run next.
+- **Coverage-ledger and health work.** Steps 2 and 3 of this program depend on
+ this machine and are sequenced after it (D17).
+- **Auto-resume or auto-retry of interrupted runs.** Unchanged from the landed
+ owner-epoch change: adjudication is silent and the normal schedule picks the
+ work up.
+- **Committing staged cursors under interruption.** Still gated on the
+ bounded-`DETAIL_COVERAGE` evidence that production does not yet emit.
+
+## Residual risks
+
+- The transition table is authored against the code as read on
+ `fix/sweep-fairness-and-transformer-bounds` at `39d19704a`. A concurrent
+ agent owns the scheduler-conformance helper and the store drivers; if those
+ land new run-state writers, the single-writer inventory needs a re-read
+ before implementation begins.
+- One historical cluster — the maintenance-sweep shared-deadline starvation
+ family — is **not** expressible as an illegal run transition. It is a
+ fairness defect in a different subsystem, recorded in `design.md` as a
+ finding rather than forced into the table.
diff --git a/openspec/changes/own-run-lifecycle-state-machine/specs/reference-implementation-architecture/spec.md b/openspec/changes/own-run-lifecycle-state-machine/specs/reference-implementation-architecture/spec.md
new file mode 100644
index 000000000..cbd847d95
--- /dev/null
+++ b/openspec/changes/own-run-lifecycle-state-machine/specs/reference-implementation-architecture/spec.md
@@ -0,0 +1,213 @@
+## ADDED Requirements
+
+### Requirement: Run lifecycle SHALL be a closed state set declared exactly once
+
+The reference implementation SHALL define the run lifecycle as a closed set of
+states: `pending`, `running`, `awaiting_interaction`, `cancel_requested`,
+`succeeded`, `failed`, `surface_failed`, `cancelled`, and `abandoned`. The last
+five are terminal.
+
+The state set and its terminal subset SHALL be declared in exactly one place,
+and every consumer — projection, fold, query, backfill, and health read — SHALL
+derive from that declaration. No component SHALL restate the terminal set as
+its own literal collection.
+
+A run state SHALL NOT be represented by a boolean or marker stored beside the
+state. Any condition that describes where a run is in its lifecycle SHALL be a
+state in this set or SHALL be derived from one.
+
+A dispatch outcome that never started a run SHALL NOT be a run state. The
+scheduler MAY record such an attempt for cadence purposes, but the lifecycle
+SHALL NOT read it as the outcome of a run.
+
+#### Scenario: The terminal set has one declaration
+
+- **WHEN** any component tests whether a run event is terminal
+- **THEN** it SHALL consult the single declared terminal set
+- **AND** adding a terminal state SHALL require exactly one edit
+
+#### Scenario: An abandoned run is visible to every terminal reader
+
+- **WHEN** a run reaches `abandoned`
+- **THEN** every consumer of the terminal set SHALL observe that run as terminal
+- **AND** no consumer SHALL classify it as non-terminal because its own copy of
+ the terminal set omits `abandoned`
+
+#### Scenario: Both backends agree on the terminal set
+
+- **WHEN** the same run is read through the SQLite path and the PostgreSQL path
+- **THEN** both SHALL report the same terminal classification
+
+### Requirement: Every run-state transition SHALL be an epoch-fenced compare-and-swap
+
+Every durable run-state transition SHALL be expressed as a single conditional
+statement predicated on both the expected current state and the acting owner
+epoch. A transition SHALL NOT read the current state and then write it in a
+separate statement.
+
+A transition whose predicate matches no row SHALL be treated as refused. A
+refusal SHALL be an ordinary outcome: it SHALL NOT be retried by re-reading and
+re-writing, SHALL NOT raise owner attention, and SHALL NOT be recorded as a
+failure of the run.
+
+A run SHALL carry the identity of the owner epoch entitled to transition it,
+recorded durably in the same write that admits the run. A process whose epoch
+is not the run's owner epoch SHALL fail to transition that run **at the
+database**, not by an in-process check.
+
+The predicate SHALL be expressed so that it is identical in effect on SQLite
+and PostgreSQL. A null owner epoch SHALL be treated as unclaimed. The null arm
+SHALL be spelled out explicitly and SHALL NOT be written in a form that reduces
+to excluding null rows on either backend.
+
+Run identity for fencing purposes SHALL be the pair of run identifier and
+connector instance identifier, because a run identifier alone is not unique
+across connections.
+
+#### Scenario: A stale epoch's write is refused by the database
+
+- **WHEN** a process whose owner epoch is not the run's owner epoch attempts any
+ transition on that run
+- **THEN** the statement SHALL match no row
+- **AND** the run's state SHALL be unchanged
+
+#### Scenario: A transition from an unexpected state is refused
+
+- **WHEN** a transition expects a run to be in one state and the run is in
+ another
+- **THEN** the statement SHALL match no row
+- **AND** the caller SHALL treat the refusal as an ordinary outcome
+
+#### Scenario: A legacy run with no recorded owner epoch is claimable
+
+- **WHEN** a transition targets a run whose owner epoch is null
+- **THEN** the transition SHALL be permitted
+- **AND** the predicate SHALL NOT spare that run on either backend
+
+#### Scenario: Two connections sharing a run identifier are fenced apart
+
+- **WHEN** two runs on different connector instances share a run identifier
+- **THEN** a transition on one SHALL NOT match the other
+
+### Requirement: Exactly one component SHALL write run state
+
+All run-state transitions SHALL pass through a single owner module. No other
+component SHALL emit a run lifecycle event, update the durable run projection,
+or mutate an in-process structure that other components read as run state.
+
+A shared helper that many components import SHALL NOT satisfy this requirement.
+The requirement is one code path that owns the mutations, not one function that
+many code paths call.
+
+A component that writes run state SHALL NOT also decide which work runs next.
+
+#### Scenario: A second writer cannot terminalize a run
+
+- **WHEN** a component other than the owner module attempts to record a run's
+ terminal state
+- **THEN** the attempt SHALL be rejected
+- **AND** the run's terminal state SHALL remain whatever the owner module wrote
+
+#### Scenario: A terminal run cannot be revised
+
+- **WHEN** any component attempts a transition on a run already in a terminal
+ state
+- **THEN** the transition SHALL be refused
+- **AND** the run's recorded record count SHALL NOT be revised
+
+### Requirement: The planner SHALL read run state and SHALL NOT write it
+
+The component that decides which connector to run SHALL read the run lifecycle
+and emit intents. It SHALL NOT transition run state.
+
+A planner read SHALL NOT produce a durable side effect. An eligibility probe
+that reconciles, repairs, or otherwise writes on the read path SHALL count as
+writing run state, regardless of the name of the function that performs it.
+
+The lifecycle SHALL NOT encode scheduling policy. Backoff curves, cooldown
+windows, fairness rotation, admission budgets, and automation suppression SHALL
+remain the planner's. The lifecycle SHALL answer only whether a transition is
+legal.
+
+A run being eligible for a transition SHALL NOT mean the run has been chosen to
+execute.
+
+#### Scenario: The planner's eligibility probe writes nothing
+
+- **WHEN** the planner evaluates whether a connector instance is due
+- **THEN** that evaluation SHALL perform no durable write
+- **AND** it SHALL NOT acquire a write lock on the connector instance
+
+#### Scenario: Scheduling policy stays out of the lifecycle
+
+- **WHEN** a backoff, cooldown, or fairness rule changes
+- **THEN** the run lifecycle's states and transitions SHALL be unchanged
+
+### Requirement: Illegal transitions SHALL be refused rather than reconciled later
+
+The reference implementation SHALL define which transitions are legal, and
+SHALL refuse every transition outside that definition at the point of the
+write.
+
+The following SHALL be illegal: a transition attempted by any component other
+than the transition's declared writer; a transition whose actor epoch is not
+the run's owner epoch; a second terminal transition on a terminal run; any
+transition out of a terminal state; and the recording of an interrupted run as
+an observed failure.
+
+A run SHALL NOT be able to remain non-terminal indefinitely with no live owner
+epoch. A terminal state SHALL always be reachable for such a run, so that a run
+whose owner has died cannot permanently block its connector instance.
+
+#### Scenario: An interrupted run is not recorded as a failure
+
+- **WHEN** a run's owner process died without reporting an outcome
+- **THEN** the run SHALL transition to `abandoned`
+- **AND** no transition to `failed` SHALL be permitted for that reason
+
+#### Scenario: An ownerless run cannot block its connection forever
+
+- **WHEN** a run is non-terminal and no live epoch owns it
+- **THEN** a terminal transition SHALL be reachable for that run
+- **AND** its connector instance SHALL become able to admit a new run
+
+### Requirement: Successor adjudication SHALL be a lifecycle transition
+
+Marking a predecessor epoch's non-terminal runs as abandoned SHALL be a legal
+transition of the run lifecycle, executed by the owner module at boot. It SHALL
+NOT be a separate reconciliation path with its own copy of the state
+vocabulary.
+
+The transition SHALL preserve the adjudication behavior already established:
+it SHALL be idempotent per originating run-start event so repeated passes
+produce exactly one terminal event; it SHALL NOT adjudicate any run belonging
+to the newest boot epoch; it SHALL decide eligibility by epoch comparison
+rather than by an age threshold; and it SHALL NOT revise the record count of a
+run that ingested records before being interrupted.
+
+The terminal event and its durable projection SHALL be written in one
+transaction, so a run cannot be terminal in the event log while its projection
+still claims to be running.
+
+An owner-operated repair tool MAY apply this transition over a historical
+backlog with different scoping, but SHALL use the same transition rather than
+reimplementing it, and SHALL remain distinguishable in the recorded provenance
+of the events it writes.
+
+#### Scenario: Adjudication at boot is the same transition as everywhere else
+
+- **WHEN** the owner module adjudicates a predecessor epoch's non-terminal run
+- **THEN** it SHALL apply the same epoch-fenced transition used for every other
+ run-state change
+
+#### Scenario: The event and its projection commit together
+
+- **WHEN** adjudication terminalizes a run
+- **THEN** the terminal event and the durable projection SHALL commit in one
+ transaction
+- **AND** no run SHALL be observable as terminal in one and running in the other
+
+#### Scenario: Repeated adjudication is idempotent
+
+- **WHEN** adjudication runs twice over the same interrupted run
+- **THEN** exactly one terminal event SHALL exist for that run
diff --git a/openspec/changes/own-run-lifecycle-state-machine/tasks.md b/openspec/changes/own-run-lifecycle-state-machine/tasks.md
new file mode 100644
index 000000000..d6da49b7c
--- /dev/null
+++ b/openspec/changes/own-run-lifecycle-state-machine/tasks.md
@@ -0,0 +1,142 @@
+## 1. Design (this change)
+
+- [x] 1.1 Enumerate the closed state set, including `abandoned`, and rule
+ `skipped` out as a dispatch outcome rather than a run state.
+- [x] 1.2 Audit every boolean and side-state living beside run state and record
+ an absorb / derive / delete verdict for each in `design.md`.
+- [x] 1.3 Express every F1 incident cluster as a forbidden transition, and
+ report the one cluster that cannot be expressed as a finding.
+- [x] 1.4 Write the legal-transition table with each transition's precondition
+ and single writer, and the forbidden table annotated with the incident
+ each entry prevents.
+- [x] 1.5 Specify the compare-and-swap predicate for SQLite and PostgreSQL
+ together, including the explicit null arm.
+- [x] 1.6 State the planner/executor split and its policy guard.
+- [x] 1.7 Specify successor adjudication as a legal transition preserving
+ observable behavior.
+- [x] 1.8 Write the writers-first migration plan and the `drainActiveRuns`
+ verdict.
+
+## 2. Findings raised by this design (not fixed here)
+
+- [ ] 2.1 Six terminal-set declarations disagree with `lib/spine.ts:1066`. Four
+ omit `run.abandoned` (`connector-summary-read-model.ts:1253`,
+ `db.ts:5437`, `postgres-storage.ts:2692`, `connector-summary-evidence-engine.ts:1599`);
+ two omit `run.browser_surface_failed` (`lib/postgres-spine.ts:570`,
+ `postgres-storage.ts:2336`). Closed by M6, not before.
+- [ ] 2.2 `insert-run-history.sql:40` and `scheduler-store.ts:1018` upsert
+ `status = excluded.status` with no `status = 'running'` fence, so a
+ scheduler retry can overwrite a terminal status.
+- [ ] 2.3 `controller-boot.ts:829` (PostgreSQL drift repair) matches on
+ `run_id` alone while its SQLite twin at `:788` fences on
+ `connector_instance_id`.
+- [ ] 2.4 `run_generation` is written from a monotonic counter
+ (`controller.ts:3187`) and from a retry `attempt`
+ (`scheduler/run-executor.ts:819`) — one column, two meanings.
+- [ ] 2.5 Two `createReferenceSchedulerManager` definitions exist
+ (`server/index.ts:8828` live, `scheduler-manager-factory.ts:374`
+ test-only). Resolve before M2.
+
+## 3. Schema (implementation, not this change)
+
+- [ ] 3.1 Add NULL-tolerant `run_history.owner_epoch TEXT` on SQLite via
+ `addColumnIfMissing`, mirroring the `run_generation` migration.
+- [ ] 3.2 Add `run_history.owner_epoch TEXT` on PostgreSQL via
+ `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`, **in the same change** as 3.1.
+- [ ] 3.3 Stamp `owner_epoch` in the same write that admits a run.
+- [ ] 3.4 Add a dual-backend test proving the column exists and is written on
+ both, so a single-backend migration cannot ship.
+
+## 4. The owner module (implementation, not this change)
+
+- [ ] 4.1 Declare the state set and terminal subset once; every consumer derives.
+- [ ] 4.2 Implement the transition table T1-T11 as epoch-fenced compare-and-swap.
+- [ ] 4.3 Return refusal as an ordinary outcome; never retry blindly.
+- [ ] 4.4 Route boot adjudication (T10/T11) through the same predicate.
+- [ ] 4.5 Cut writers over per the M1-M6 tranches, deleting each old path in the
+ same change as its replacement.
+
+## 5. Property-test skeletons (this change — must fail or `.todo`)
+
+Each skeleton is authored `.todo` or asserting against the not-yet-existing
+owner module, and is marked in-file as a skeleton. **A skeleton that passes
+before implementation is the defect this program exists to kill.** Each names
+its property, generator, and invariant, and maps to a forbidden transition.
+
+- [x] 5.1 `run-lifecycle-transition-legality.property.test.ts` — **F7, F5.**
+ Generator: random sequences of transition attempts drawn from the full
+ state × transition cross-product. Invariant: the observed state after any
+ sequence is reachable from the initial state by legal transitions only,
+ and a terminal state is never left.
+- [x] 5.2 `run-lifecycle-epoch-fencing.property.test.ts` — **F2.** Generator:
+ interleavings of two actors with distinct epochs attempting transitions on
+ one run. Invariant: only the run's owner epoch ever changes state; the
+ stale actor's statement matches zero rows. Runs on both backends.
+- [x] 5.3 `run-lifecycle-single-terminal.property.test.ts` — **F5, F4.**
+ Generator: concurrent terminal attempts of differing kinds, including a
+ boot adjudicator racing an executor. Invariant: exactly one terminal state
+ results; an interrupted run terminalizes `abandoned`, never `failed`.
+- [x] 5.4 `run-lifecycle-planner-writes-nothing.property.test.ts` — **F1.**
+ Generator: planner eligibility evaluations against instances in every
+ state, including one with a run in flight. Invariant: zero durable writes
+ and zero connector-instance write-lock acquisitions occur during a planner
+ read. This is the GroupMe 503 as a property.
+- [x] 5.5 `run-lifecycle-no-permanent-wedge.property.test.ts` — **F6.**
+ Generator: runs abandoned mid-flight by an epoch that never returns.
+ Invariant: a terminal state is always reachable, and the connector
+ instance can admit a new run afterwards. This is the YNAB wedge.
+- [x] 5.6 `run-lifecycle-cas-no-stale-premise.property.test.ts` — **F3.**
+ Generator: transitions whose observed state is mutated by a competing
+ actor between the caller's read and its write. Invariant: no transition
+ commits on a premise that changed under it. This is the drain/clock race.
+- [x] 5.7 `run-lifecycle-terminal-set-agreement.property.test.ts` — **the
+ declaration requirement.** Generator: every terminal state, read through
+ every consumer of the terminal set. Invariant: all consumers agree, and
+ both backends agree. Fails today on the six divergent declarations.
+- [x] 5.8 `run-lifecycle-adjudication-idempotence.property.test.ts` — **T10/T11.**
+ Generator: repeated adjudication passes over overlapping orphan sets,
+ including a run in the newest boot epoch. Invariant: exactly one terminal
+ event per orphan; newest-epoch runs are never adjudicated; record counts
+ are never revised.
+
+- [ ] 5.9 Register 5.1-5.8 in the suite as expected-failing until the owner
+ module lands, so their red state is deliberate and visible rather than
+ an unexplained broken build.
+
+## 6. Pre-registered canary metrics (D15)
+
+Registered **before** any Step 1 deploy. Post-hoc criteria are how "green"
+claims died in this program.
+
+- [ ] 6.1 **Restarts.** 20 consecutive controller restarts in a replaced
+ container with zero adjudication anomalies: every `run.started` without a
+ terminal event belongs to the newest boot epoch, and no run holds two
+ terminal events. Measured by query, not by log reading.
+- [ ] 6.2 **Race-class property tests green.** All of 5.1-5.8 pass on both
+ SQLite and PostgreSQL. Any skipped test counts as a failure.
+- [ ] 6.3 **Transition throughput is non-zero.** Count of successful transitions
+ per state pair over the canary window is greater than zero for T1, T2, and
+ at least one terminal transition. A machine that refuses everything would
+ otherwise satisfy every "zero anomalies" metric.
+- [ ] 6.4 **Refusal rate is bounded.** Refused transitions stay under 1% of
+ attempts outside adjudication. A rising refusal rate means the expected
+ state is being computed wrong somewhere.
+- [ ] 6.5 **Behavior preservation (D14).** Per-connector run outcomes over the
+ canary window match the pre-change baseline in kind and count. Terminal
+ state distribution shifts only by the `failed` → `abandoned`
+ reclassification the design intends.
+- [ ] 6.6 **No permanent wedge.** Zero connector instances refusing a new run
+ with `active_run_exists` for longer than one scheduling interval.
+ Baseline: the UAT instance previously held 7 of 8 `running` rows as
+ zombies up to two days old.
+- [ ] 6.7 **Dual-backend parity.** 6.1-6.6 measured on both backends. A
+ PostgreSQL-only or SQLite-only pass is a failure, not a partial success.
+
+## 7. Validation
+
+- [ ] 7.1 `openspec validate own-run-lifecycle-state-machine --strict`.
+- [ ] 7.2 `openspec validate --all --strict` against the recorded baseline of 11
+ pre-existing failures, so this change's contribution is distinguishable.
+- [ ] 7.3 Re-verify the single-writer inventory against the branch head before
+ M2 begins; it is a point-in-time read and concurrent agents own adjacent
+ files.
diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/design.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/design.md
new file mode 100644
index 000000000..9ff2cb9e6
--- /dev/null
+++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/design.md
@@ -0,0 +1,315 @@
+## Context
+
+The sibling change `adjudicate-interrupted-runs-by-owner-epoch` settled who
+writes an interrupted run's terminal state: the successor, by owner epoch, with
+no drain. It deliberately left one question open — whether a staged cursor may
+be committed when a run does not reach `succeeded`.
+
+That question cannot be answered by policy, because the runtime has no
+information to reason with. `commitState(stream, cursor)` is already a
+per-stream idempotent `PUT`, and the plumbing is exercised on every successful
+run. Only the *decision* is deferred to DONE, at a single gate:
+
+```
+if (persistState && (terminalStatus === "succeeded" || isCertifiedStreamCollectionFailure))
+```
+
+`reference-implementation/runtime/index.ts` — located by content, not by line
+number, since the research entry's line numbers have since drifted.
+
+So the missing thing is not a policy. It is a vocabulary.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- A connector can state a checkpoint claim the runtime can check without
+ knowing anything about the connector.
+- A connector that cannot state a truthful claim is *rejected*, and that
+ rejection is the standard working, not a failure of the standard.
+- The representation admits coarse-granularity truth, so a connector with a
+ coarse cursor is judged on whether it can prove a coarse unit closed, not
+ punished for lacking a finer one.
+- The representation is storage-free, so the terminal form is additive.
+
+**Non-Goals:**
+
+- Fleet-wide adoption. See the proposal; this is the point, not a concession.
+- A storage, compaction, or interval-merge layer.
+- Rescuing a connector whose scan is unordered.
+
+## Decisions
+
+### Qualification is the product, not a side effect
+
+A prototype tested a four-field claim contract against `gmail`, `slack`, and
+`heb`. Two of the three could not state a truthful claim, and that was first
+recorded as the design failing.
+
+That framing is wrong and is corrected here. **A contract that admits every
+connector would be worthless.** Its entire job is to separate cursors that can
+express a truthful boundary from cursors that cannot. `slack` and `heb` failing
+at fine granularity is the qualifier discriminating correctly — it is the
+evidence that the standard has teeth, not evidence against it.
+
+The correct reading of the prototype:
+
+| Connector | Fine granularity | Day granularity | Why |
+| --- | --- | --- | --- |
+| `gmail` | Qualifies | n/a | IMAP UID order, and it already carries `uidvalidity` as an identifier-space epoch |
+| `heb` | Fails | **Qualifies, under the closed-day rule** | Cursor is `YYYY-MM-DD`; orders sort by full timestamp, so within-day position is inexpressible |
+| `slack` | Fails | Fails | No ordered scan at all; no granularity rescues it |
+
+### The representation is covered intervals; debt is derived
+
+The prototype's contract declared a `complete_through` boundary *and* a
+separate `debt` list. Two fields describing one fact can disagree, and nothing
+in the protocol would catch it.
+
+This change adopts **covered intervals** instead: the claim carries a set of
+intervals over a declared space, and outstanding debt is whatever is *not*
+covered. Debt is derived, so the two cannot contradict each other. A
+newest-first walk states its truth directly — after page 1 `heb` covers
+`[newest_day, newest_day]` and owes everything below, and that is a correct
+claim at every instant rather than a promise about the future.
+
+This also subsumes the two-pointer shape without special-casing it. `gmail`'s
+`forward_uidnext` plus `backfilled_through_uid`/`target_uid` is exactly two
+covered intervals with a gap between them.
+
+No storage layer is specified. An implementation may keep one interval or a
+thousand; the representation does not care, which is what makes a later
+compaction layer additive.
+
+### A claim carries a granularity, and a coarse claim is still true
+
+The prototype tested claims at the granularity of the provider's sort key. That
+is the wrong test for a connector whose cursor is coarser than its sort key,
+which is precisely `heb`'s situation.
+
+A day-boundary claim — "every order dated D or later is accounted for" — is
+truthful even when within-day ordering is inexpressible, **provided the
+connector can prove the day is closed**. Coarsening the granularity does not
+weaken the claim's meaning; it widens the unit over which completeness must be
+proven, which makes the claim *harder* to earn, not easier.
+
+The claim's granularity is declared, and every interval endpoint is a position
+at that granularity. A day-granularity claim covering `[D, D]` asserts that
+every item in the whole of day `D` is accounted for. It does not assert
+anything about ordering inside `D`, and it does not need to.
+
+### The `heb` day-granularity verdict: qualifies, under a closed-day rule
+
+**Verdict: `heb` `orders` qualifies at day granularity, and only for days
+proven closed. The naive day claim is unsafe and is rejected.**
+
+Two independent hazards put holes *inside* a day, and each is answered:
+
+**Hazard 1 — the newest day is open.** `runForwardScan`
+(`packages/polyfill-connectors/connectors/heb/index.ts:868-925`) walks list
+pages newest-first and tracks `newestOrderDate` as a running max
+(`:901-903`). At any instant mid-walk, the newest day seen may be only
+partially enumerated: orders from the same day sort by full timestamp and can
+straddle a page boundary, so page 1 may hold three of that day's five orders.
+Claiming `[newestOrderDate, ...]` from page 1 would claim a day whose remaining
+orders have not been seen.
+
+The rule that answers it: **a day may be claimed only once a strictly older day
+has been observed.** Seeing an order dated `D-1` proves the newest-first walk
+has passed every order dated `D`. This is a property of the walk order, which
+`heb` genuinely has (globally reverse-chronological, per `resumeBoundary`'s
+own doc comment: "H-E-B's order list is globally reverse-chronological (not
+year-partitioned like Amazon)"). The newest day observed is therefore always
+excluded from the claim — it is exactly the open unit.
+
+**Hazard 2 — `dateDropped` puts a hole in a day that cannot be located.** This
+is the sharper one, and it is why the rule must be a *closed*-day rule rather
+than just an *older-day-seen* rule.
+
+`processListOrder` (`:829-854`) parses each order's date. When
+`parseOrderDate` returns null — it is a `new Date(raw)` parse of DOM free text
+(`connectors/heb/parsers.ts:341-350`) — the order is pushed to
+`ordersCoverage.dateDropped` (`:851`) and the function `return`s before
+`emitOrderAndItems`. The order is *considered but not covered*, exactly as the
+`OrdersCoverage` doc comment states.
+
+The hazard is that **a date-dropped order has no date**. It cannot be
+attributed to any day, so it cannot be excluded from a specific day's interval.
+It is a hole of unknown position, and any interval claimed on a run that
+dropped a date might be the interval containing it.
+
+The rule that answers it: **a claim is emitted only when `dateDropped` is empty
+for the run.** A single unparseable date withholds the entire claim for that
+run. This is coarse and deliberately so — it fails closed, it needs no
+attribution machinery, and the fallback is today's behavior. `heb` runs are
+small, so withholding a whole run's claim is cheap.
+
+A third, lesser hazard is worth naming because it bounds the claim rather than
+blocking it: `MAX_LIST_PAGES = 50` (`:64`) and the `maxPage` exhaustion at
+`:919-921` can end a walk early. This does not threaten the claim, because a
+truncated walk simply covers fewer days — the *upper* part of the space is
+still proven, and the untraversed remainder is uncovered by construction. This
+is the covered-intervals representation earning its place: truncation is
+expressible as a smaller covered set, whereas under a single `complete_through`
+watermark it would be indistinguishable from completion.
+
+Note that `CHECKPOINT_OVERLAP_DAYS = 60` (`:70`) is **not** part of the safety
+argument. It re-scans a 60-day window to catch status transitions on
+already-seen orders (`resumeBoundary`, `:1064-1073`). It is a freshness device,
+not a completeness device, and treating it as a safety margin would be exactly
+the kind of accidental-timing invariant this contract exists to replace.
+
+**Confidence.** High on the source reading: the scan order, the `dateDropped`
+path, and the single post-scan `STATE` emission (`:1183-1186`) were each read
+directly. Not verified live — `heb` sign-in costs the owner a real OTP, so no
+run was triggered. Both rules fail closed, so a misreading withholds a commit
+rather than losing data.
+
+### `slack` remains disqualified, and the emitted-watermark fix does not change it
+
+**Verdict: disqualified at every granularity.**
+
+`grep -c "ORDER BY" packages/polyfill-connectors/connectors/slack/index.ts`
+returns **0**, reproduced on this branch. Slack's messages pass is one flat
+interleaved query over the archive with no ordering, so any mid-run watermark
+is a maximum over an arbitrary subset of rows, not the top of a contiguous
+prefix. An interval claimed over that input asserts coverage of everything
+below the maximum, while rows below it may not have been visited at all. That
+is not a weaker claim than `heb`'s — it is a false one, and coarsening the
+granularity makes it a more confident falsehood rather than a safer claim. A
+day-granularity claim over an unordered scan asserts *more*.
+
+The sibling branch `fix/slack-emitted-watermark-0821` (head `9541a10db`) was
+checked rather than assumed. It is a real and valuable fix: it advances the
+durable watermarks only for rows actually emitted and accepted, removes the
+`COALESCE(t.last_ts, ?)` global-floor inheritance, and parenthesizes an
+operator-precedence bug in the same predicate. **It does not add an ordering.**
+`grep -c "ORDER BY"` on that branch's `connectors/slack/index.ts` also returns
+**0**. The fix makes the watermark honest about *which rows it counted*; it
+does not make the scan a prefix. Disqualification stands, and it stands for the
+same reason it did before.
+
+This is worth stating plainly because the two defects are easy to conflate: the
+emitted-vs-iterated defect was a bug *within* an unclaimable design, and fixing
+it does not make the design claimable.
+
+### Rejected representations
+
+- **`complete_through` plus a declared `debt` list.** Two fields for one fact,
+ free to disagree. Covered intervals derive debt instead.
+- **A `covered`/`considered` ratio.** Measures detail-hydration honesty, not
+ position. `github` `starred` reports itself `partial` yet still advances its
+ watermark past dropped entries; `jellyfin` `items` writes a cursor nothing
+ reads. A ratio both over- and under-approximates cursor safety.
+- **A `safe: true` boolean.** Unfalsifiable self-attestation.
+- **A connector version or capability flag.** An allowlist with extra steps.
+- **A global floor for unseen partitions.** Deliberately inexpressible. A
+ partition-scoped claim may move only its own partition, which makes Slack's
+ `COALESCE` shape unrepresentable rather than merely discouraged.
+
+## The ideal-compatibility rule
+
+Every later fix on this program obeys three constraints:
+
+1. **No new health logic reading evidence projections.** Health is derived from
+ the ledger, not from re-deriving state out of projections.
+2. **No new implicit run-state flags.** Run state is explicit and durable, not
+ inferred from the presence or absence of a side-channel.
+3. **No new cursor shapes violating the claim schema.** A new cursor either
+ states a checkpoint claim or stays silent; it does not invent a third
+ dialect.
+
+**Placement, and why.** The rule lives here in `design.md` as its normative
+home, and is restated as a requirement in this change's `polyfill-runtime`
+spec delta so it survives archival into `openspec/specs/`.
+
+It is deliberately **not** added to `openspec/README.md`. That file documents
+the OpenSpec process — artifact kinds, lifecycle, validation commands, the
+closeout checklist — and is scoped to how changes are written, not to what any
+particular change may contain. A program-specific engineering constraint placed
+there would be read by every contributor to every unrelated change, which is
+how process docs accrete rules nobody applies. There is also no
+CONTRIBUTING-adjacent surface in this repository that governs runtime
+architecture.
+
+The spec delta is the surface that actually binds. A design.md is advisory and
+is archived with its change; a requirement in `openspec/specs/polyfill-runtime/`
+is the repository's standing statement of how the runtime behaves, and is what a
+later contributor or agent validates against. Putting the rule anywhere that
+does not survive `openspec archive` would guarantee it is forgotten by the step
+that needs it most, which is Step 3 of this program.
+
+## Corrections to prior statements
+
+### The restart worst case is redo since the last COMMITTED cursor
+
+Earlier statements — including the research entry
+`ai/research/pdpp/the-checkpoint-protocol-must-carry-a-proven-boundary-and-an-identifier-space-epoch-...md`
+§5 — said the residue of an interruption is that "work done *since* the last
+checkpoint is redone."
+
+**That is wrong, and it understates the cost.** Staged cursors are discarded on
+interruption, so nothing a run staged mid-flight survives. The correct statement
+is that for a `commit_on_success` connector, an interruption redoes work since
+the last **committed** cursor — which is the cursor written by the last
+*successful* run, effectively the run start.
+
+Verified by content on this branch, since line numbers have drifted:
+
+- `newState` is a plain in-memory `Record`
+ (`reference-implementation/runtime/index.ts:2762`), assigned on each `STATE`
+ message (`:4237`).
+- `commitState` has exactly two call sites (`:5278`, `:5322`), both inside the
+ DONE gate at `:5266`. There is no mid-run commit path.
+
+So a staged cursor has no durable effect until DONE, and an interrupted run
+leaves the committed cursor exactly where the previous successful run left it.
+
+This matters to the program's cost-benefit and is corrected because it is
+accurate, not because it is convenient: it makes the pain *larger* than
+previously stated, which strengthens the case for the ledger. The 465 runs that
+ingested 897,916 records and advanced no cursor are the direct measurement of
+this, and they are not "since the last checkpoint" losses — they are whole-run
+losses.
+
+## Risks / Trade-offs
+
+- [A connector states a claim it cannot honor] -> The runtime does not trust the
+ claim alone. It checks the claim against a fact it wrote itself: a claim that
+ advances a position with zero durably ingested records for that stream this
+ run is staged, not committed. Flink's rule applies — the durable ingest *is*
+ the pre-commit.
+- [The closed-day rule withholds too often on `heb`] -> It withholds a run's
+ claim whenever any date fails to parse. The fallback is today's behavior
+ (`commit_on_success`), so the failure mode is a redone run, never a lost
+ order. If withholding proves common in practice, the fix is to attribute
+ dropped orders to a day, not to weaken the rule.
+- [Covered intervals grow unbounded] -> Out of scope by construction. The
+ representation is storage-free precisely so a compaction layer can be added
+ additively once a real growth pattern is measured rather than guessed.
+- [Coarse granularity becomes an escape hatch] -> It is the opposite: a coarser
+ unit must be proven complete over a wider range, so it is harder to earn.
+ `slack` demonstrates the floor — coarsening does not rescue a scan that has no
+ order.
+- [The `heb` verdict is wrong because it was not run live] -> Both rules fail
+ closed. The cost of a misreading is a withheld commit.
+
+## Migration Plan
+
+Sequenced cheapest-value-first. This change lands only the standard; the
+numbered steps below are later tranches.
+
+1. Add the optional `checkpoint_claim` field to the protocol. Inert — no
+ connector emits it and no behavior changes.
+2. Implement the runtime decision procedure. Still inert for every connector
+ that stays silent.
+3. Claim it in `gmail` `messages` first: it already carries all the needed
+ structure including `uidvalidity`, and it is 203,417 of the lost records.
+4. `chatgpt` and `codex` next — together 657,477 records, the largest single
+ win in the fleet.
+5. `heb` under the closed-day rule, if and only if its interruption pain is
+ shown to be real. `heb` p50 run duration is 25.2s, so it may correctly never
+ qualify for the *investment* even though it qualifies for the *contract*.
+
+Rollback at any step is deletion of an optional field; a connector that stops
+claiming reverts to `commit_on_success` with no data migration.
diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/proposal.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/proposal.md
new file mode 100644
index 000000000..374a93796
--- /dev/null
+++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/proposal.md
@@ -0,0 +1,106 @@
+## Why
+
+The connector→runtime protocol types a checkpoint payload as `cursor: unknown`
+(`packages/polyfill-connectors/src/connector-runtime-protocol.ts`), and the
+runtime's complete validation of it is that it is a non-array object or null.
+The runtime therefore has zero information with which to tell a safe cursor
+from an unsafe one, and `commit_on_success` is what that absence of information
+forces — a no-information fallback, not a risk judgment.
+
+The cost is measured on the live spine: fleet-wide, 130,517 `run.state_staged`
+against 75,513 `run.state_advanced`, so 42% of staged checkpoint work is
+discarded. Restricting to terminal runs that staged at least one stream and
+reported `checkpoint_commit_status = not_committed`, 465 runs across 14
+connectors durably ingested 897,916 records and advanced no cursor.
+
+Today, cursor safety is not a property of the cursor. It is a property of when
+a connector happens to emit. `heb` `orders` sets `newestOrderDate` from page 1
+before the older pages are walked, and is safe only because its single
+`emit({ type: "STATE" ... })` sits after `runForwardScan` returns. Move that
+emit inside the loop and it becomes permanent data loss, with no runtime check
+firing and no test failing. That is an unowned invariant held in place by code
+review alone.
+
+This change writes down the contract that lets a connector *say* whether its
+checkpoint is safe, and lets the runtime check that claim against a fact the
+runtime itself wrote.
+
+## What Changes
+
+This change is a **qualification standard**, not a migration plan. It defines
+what a connector must be able to express to earn incremental commit, and
+accepts that most of the fleet will never express it.
+
+- Define an optional `checkpoint_claim` on the `STATE` message carrying a
+ declared identifier space with an epoch, a set of **covered intervals** over
+ that space, and an optional partition key. Outstanding debt is **derived from
+ the gaps** between covered intervals; it is never separately declared, so the
+ two cannot disagree.
+- Define the runtime's connector-agnostic decision procedure. Absent claim,
+ mismatched epoch, or a claim advancing with no durably ingested records this
+ run ⇒ stage only. A partition-scoped claim moving any other partition's
+ position ⇒ protocol violation.
+- Define **claim granularity**. A claim's interval endpoints are positions in a
+ declared space at a declared granularity. A coarse granularity (a day) is a
+ truthful claim when a finer one is not expressible, provided the connector
+ can prove every item in that coarse unit was accounted for.
+- Record the qualification results for the three prototyped connectors as
+ evidence that the qualifier discriminates: `gmail` qualifies, `slack` is
+ disqualified at every granularity, and `heb` qualifies **only** at day
+ granularity and **only** under a closed-day rule this change specifies.
+- Adopt an ideal-compatibility rule constraining every later fix on this
+ program: no new health logic reading evidence projections, no new implicit
+ run-state flags, no new cursor shapes violating the claim schema.
+
+Non-qualifying connectors keep `commit_on_success` unchanged and forever. That
+is a correct permanent answer, not a temporary one.
+
+## Capabilities
+
+### New Capabilities
+
+None.
+
+### Modified Capabilities
+
+- `polyfill-runtime`: Define the checkpoint claim's covered-intervals
+ representation, the identifier-space epoch, claim granularity and the
+ closed-unit rule, the runtime's commit decision procedure, the
+ disqualification criteria, and the ideal-compatibility rule.
+
+## Impact
+
+- No production code changes in this tranche. This change is the qualification
+ standard; implementation is sequenced behind it.
+- When implemented: `packages/polyfill-connectors/src/connector-runtime-protocol.ts`
+ gains an optional field, and `reference-implementation/runtime/index.ts`
+ gains the decision procedure at its `STATE` handler. Both are additive — a
+ connector that says nothing keeps today's behavior exactly.
+- The representation is deliberately storage-free, so a later compaction or
+ interval-store layer is an additive migration rather than a rewrite.
+
+## Non-Goals
+
+- **Fleet-wide adoption.** Explicitly not a goal. A connector whose runs take
+ 40 seconds should redo its run; buying the contract there is cost with no
+ benefit. The contract is bought only where interruption pain is real, which
+ the live measurement localizes to a handful of connectors.
+- **A storage or compaction layer for intervals.** This change specifies the
+ representation only. How intervals are stored, merged, or bounded is left
+ open so the terminal form is an additive migration.
+- **Committing staged cursors under an `INTERRUPTED` terminal state.** That
+ remains gated on the sibling change
+ `adjudicate-interrupted-runs-by-owner-epoch`, whose measurement stands: zero
+ of 34,928 `run.detail_coverage_declared` events carry `boundary`,
+ `slice_start`, or `slice_end`.
+- **Rescuing `slack`.** No granularity coarsening makes an unordered scan
+ claimable. Slack needs an ordered scan first; that is separate work.
+- **Any per-connector allowlist.** If one becomes necessary, the contract has
+ failed — the claim is meant to travel with the data.
+
+## Residual risks
+
+- The `heb` day-granularity verdict is derived from source reading, not from a
+ live run. `heb` sign-in costs the owner a real OTP, so live confirmation is
+ deferred rather than performed. The closed-day rule is written to fail closed,
+ so a wrong reading withholds a commit rather than losing data.
diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/specs/polyfill-runtime/spec.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/specs/polyfill-runtime/spec.md
new file mode 100644
index 000000000..56ad1ee4a
--- /dev/null
+++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/specs/polyfill-runtime/spec.md
@@ -0,0 +1,267 @@
+## ADDED Requirements
+
+### Requirement: A checkpoint claim SHALL be optional, and its absence SHALL mean commit_on_success
+
+A connector `STATE` message MAY carry a `checkpoint_claim`. A connector that
+emits no claim SHALL retain `commit_on_success` behavior unchanged: its staged
+cursor SHALL be committed only when the run reaches a terminal status that
+already permits commit.
+
+The runtime SHALL NOT require a claim from any connector, and SHALL NOT
+maintain a per-connector allowlist of connectors permitted to claim. Eligibility
+SHALL be carried by the claim itself, so a connector qualifies or fails to
+qualify by what it can express, not by enumeration.
+
+Fleet-wide adoption SHALL NOT be treated as a goal. A connector whose runs are
+short enough that redoing one is acceptable recovery MAY correctly never emit a
+claim.
+
+#### Scenario: A silent connector keeps today's behavior
+
+- **WHEN** a connector emits a `STATE` message with no `checkpoint_claim`
+- **THEN** the runtime SHALL stage the cursor
+- **AND** it SHALL NOT commit that cursor before the run reaches a terminal
+ status that already permits commit
+
+#### Scenario: Qualification is not an allowlist
+
+- **WHEN** the runtime evaluates whether a checkpoint may be committed
+ incrementally
+- **THEN** the decision SHALL depend only on the claim and on facts the runtime
+ itself recorded
+- **AND** it SHALL NOT depend on the connector's name or version
+
+### Requirement: A checkpoint claim SHALL declare an identifier space with an epoch
+
+A `checkpoint_claim` SHALL name the identifier space its positions belong to,
+and that name SHALL incorporate an epoch that changes whenever the provider
+re-seeds the space.
+
+The runtime SHALL compare a claim's space to the stored space by equality
+alone. When they differ, the runtime SHALL stage the cursor only, SHALL discard
+the prior position, and SHALL NOT interpret positions from one epoch as
+comparable to positions from another.
+
+The runtime SHALL NOT require any provider-specific knowledge to perform this
+comparison.
+
+#### Scenario: A re-seeded identifier space invalidates the prior position
+
+- **WHEN** a claim declares a space that differs from the stored space
+- **THEN** the runtime SHALL NOT commit the claimed position
+- **AND** the prior position SHALL be discarded rather than compared
+
+#### Scenario: A matching space permits comparison
+
+- **WHEN** a claim declares a space equal to the stored space
+- **THEN** the claim's positions SHALL be treated as comparable to the stored
+ position
+
+### Requirement: A checkpoint claim SHALL represent coverage as intervals, with debt derived from the gaps
+
+A `checkpoint_claim` SHALL express what it has accounted for as a set of
+covered intervals over its declared space. Outstanding debt SHALL be derived as
+the complement of the covered set within that space, and SHALL NOT be declared
+as a separate field.
+
+A claim SHALL NOT carry both a coverage assertion and an independently declared
+debt assertion, because two fields describing one fact can disagree and the
+runtime cannot adjudicate between them.
+
+An empty covered set SHALL be a valid claim asserting that nothing is yet
+accounted for. Omitting the covered set entirely SHALL be rejected as
+malformed; the two SHALL NOT be treated as equivalent.
+
+This requirement SHALL constrain the representation only. It SHALL NOT mandate
+any storage, compaction, or interval-merge strategy, so that such a layer may
+be added later without changing the claim's meaning.
+
+#### Scenario: A newest-first walk states a truthful partial claim
+
+- **WHEN** a connector walking newest-first has accounted for only the newest
+ part of its space
+- **THEN** it MAY claim exactly that part as covered
+- **AND** the remainder SHALL be treated as outstanding debt without being
+ separately declared
+
+#### Scenario: A two-pointer connector needs no special case
+
+- **WHEN** a connector maintains both a forward watermark and a backfill floor
+- **THEN** it SHALL express that state as two covered intervals separated by a
+ gap
+
+#### Scenario: An omitted covered set is malformed
+
+- **WHEN** a claim omits its covered set entirely
+- **THEN** the runtime SHALL reject the claim as malformed
+- **AND** it SHALL NOT treat the claim as asserting empty coverage
+
+### Requirement: A checkpoint claim SHALL declare its granularity, and a coarse claim SHALL cover whole units
+
+A `checkpoint_claim` SHALL declare the granularity at which its interval
+endpoints are positions. A claim MAY declare a granularity coarser than the
+provider's own sort order when a finer position is not expressible from the
+connector's cursor.
+
+A coarse-granularity interval SHALL assert that every item in every unit it
+covers has been accounted for. It SHALL NOT assert anything about ordering
+within a unit, and SHALL NOT be required to.
+
+A unit SHALL be claimed only when the connector can prove that unit closed. A
+unit that may still receive items the connector has not seen SHALL NOT be
+claimed, regardless of how many of its items have been accounted for.
+
+Coarsening granularity SHALL NOT weaken what a claim asserts. A coarser unit
+SHALL require completeness to be proven over a wider range, so it SHALL be
+harder to earn than a finer one, not easier.
+
+#### Scenario: A coarse claim is truthful when the unit is closed
+
+- **WHEN** a connector's cursor cannot express a position finer than a whole
+ unit
+- **AND** it can prove every item in that unit was accounted for
+- **THEN** it MAY claim that unit as covered at that granularity
+
+#### Scenario: The open unit at the frontier is not claimed
+
+- **WHEN** a connector walking in one direction has reached a unit but cannot
+ prove it has passed every item in that unit
+- **THEN** it SHALL NOT claim that unit
+- **AND** it MAY claim only units it has provably passed
+
+#### Scenario: An unattributable omission withholds the claim
+
+- **WHEN** a connector enumerated an item but could not determine which unit it
+ belongs to
+- **THEN** it SHALL NOT emit a claim covering any unit for that run
+- **AND** the run SHALL fall back to `commit_on_success` behavior
+
+#### Scenario: A truncated walk claims less rather than claiming falsely
+
+- **WHEN** a walk ends early because a pagination ceiling was reached
+- **THEN** the claim SHALL cover only the units actually traversed
+- **AND** the untraversed remainder SHALL remain outstanding debt
+
+### Requirement: The runtime SHALL check a claim against durable ingest before committing
+
+The runtime SHALL NOT commit a claimed position on the strength of the claim
+alone. A claim that advances a stream's position SHALL be committed only when
+the runtime itself recorded durable ingest for that stream during the run.
+
+A claim that advances a position with no durably ingested records for that
+stream in that run SHALL be staged only.
+
+The runtime's decision procedure SHALL be connector-agnostic and SHALL depend
+only on the claim's declared fields and on facts the runtime recorded.
+
+#### Scenario: A claim without corresponding durable ingest is not committed
+
+- **WHEN** a claim advances a stream's position
+- **AND** the runtime recorded no durable ingest for that stream during the run
+- **THEN** the runtime SHALL stage the cursor without committing it
+
+#### Scenario: A claim backed by durable ingest commits incrementally
+
+- **WHEN** a claim is well-formed, its space matches, and the runtime recorded
+ durable ingest for that stream during the run
+- **THEN** the runtime SHALL commit that stream's cursor without waiting for the
+ run's terminal status
+
+### Requirement: A partition-scoped claim SHALL move only its own partition, and a global floor SHALL be inexpressible
+
+A `checkpoint_claim` MAY declare a partition key. A partition-scoped claim SHALL
+move only the position of the partition it names.
+
+A claim that would move the position of any partition other than the one it
+names SHALL be rejected as a protocol violation.
+
+The claim schema SHALL provide no field by which a connector can assign a
+position to partitions it has not enumerated. A floor inherited by unseen
+partitions SHALL therefore be unrepresentable rather than merely discouraged,
+because such a floor makes every item below it permanently unreachable for a
+partition that was never walked.
+
+#### Scenario: A claim cannot move a partition it does not name
+
+- **WHEN** a partition-scoped claim would advance the position of a different
+ partition
+- **THEN** the runtime SHALL reject the claim as a protocol violation
+
+#### Scenario: An unenumerated partition inherits no position
+
+- **WHEN** a connector has not enumerated a partition
+- **THEN** no claim SHALL assign that partition a position
+- **AND** that partition SHALL remain fully outstanding
+
+### Requirement: A connector whose scan has no order SHALL NOT qualify at any granularity
+
+A connector SHALL NOT emit a `checkpoint_claim` for a stream whose items are
+retrieved without an ordering over the claimed space.
+
+A maximum taken over an arbitrary subset of items SHALL NOT be treated as the
+upper bound of a covered interval. Without an ordering, items below that maximum
+may never have been visited, so an interval claimed up to it asserts coverage
+the connector cannot demonstrate.
+
+Declaring a coarser granularity SHALL NOT qualify an unordered scan. A coarser
+unit asserts completeness over a wider range, so coarsening an unordered scan
+SHALL produce a broader false claim rather than a safer one.
+
+Advancing a watermark only for items actually emitted SHALL NOT by itself
+qualify a stream. Restricting the watermark to emitted items makes it honest
+about which items were counted; it does not establish that the counted items
+form a contiguous prefix of the space.
+
+#### Scenario: An unordered scan is disqualified
+
+- **WHEN** a stream's items are retrieved by a query with no ordering over the
+ claimed space
+- **THEN** that stream SHALL NOT emit a checkpoint claim
+- **AND** it SHALL retain `commit_on_success` behavior
+
+#### Scenario: Coarsening does not rescue an unordered scan
+
+- **WHEN** an unordered stream declares a coarser claim granularity
+- **THEN** it SHALL still be disqualified
+
+#### Scenario: An emitted-only watermark does not establish a prefix
+
+- **WHEN** a stream advances its watermark only over items it emitted, but still
+ retrieves those items without an ordering
+- **THEN** it SHALL remain disqualified
+
+### Requirement: New work on the checkpoint program SHALL preserve ideal compatibility
+
+Changes made under this program SHALL observe the following constraints, so that
+later tranches compose rather than accumulating incompatible mechanisms.
+
+New health logic SHALL NOT be derived by reading evidence projections. Health
+SHALL be derived from the coverage ledger.
+
+New implicit run-state flags SHALL NOT be introduced. Run state SHALL be
+explicit and durable, and SHALL NOT be inferred from the presence or absence of
+a side-channel.
+
+New cursor shapes SHALL NOT violate the claim schema. A new cursor SHALL either
+state a well-formed `checkpoint_claim` or emit no claim at all.
+
+#### Scenario: Health is not re-derived from a projection
+
+- **WHEN** a change introduces health logic under this program
+- **THEN** that logic SHALL read the coverage ledger
+- **AND** it SHALL NOT re-derive health by reading an evidence projection
+
+#### Scenario: Run state stays explicit
+
+- **WHEN** a change under this program needs to record that a run is in some
+ state
+- **THEN** it SHALL record that state explicitly and durably
+- **AND** it SHALL NOT encode it as the presence or absence of an unrelated
+ side-channel
+
+#### Scenario: A new cursor either claims well-formed or stays silent
+
+- **WHEN** a change under this program introduces a new cursor shape
+- **THEN** that cursor SHALL either carry a well-formed `checkpoint_claim` or
+ carry none
+- **AND** it SHALL NOT introduce a third representation of checkpoint safety
diff --git a/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/tasks.md b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/tasks.md
new file mode 100644
index 000000000..aed75203e
--- /dev/null
+++ b/openspec/changes/qualify-connectors-for-incremental-checkpoint-commit/tasks.md
@@ -0,0 +1,80 @@
+## 1. Establish the qualification standard (this tranche)
+
+- [x] 1.1 Frame the checkpoint contract as a qualification standard rather than
+ a failed design, and state that fleet-wide adoption is not a goal.
+- [x] 1.2 Adopt the covered-intervals representation, with debt derived from the
+ gaps, and specify no storage or compaction layer so the terminal form is
+ an additive migration.
+- [x] 1.3 Define claim granularity and the closed-unit rule, so a connector with
+ a coarse cursor is judged on whether it can prove a coarse unit complete.
+- [x] 1.4 Record the disqualification criteria for unordered scans, including
+ that coarsening does not rescue them and that an emitted-only watermark
+ does not establish a prefix.
+
+## 2. Measure the coarse-granularity question
+
+- [x] 2.1 Re-test `heb` `orders` at day granularity by reading
+ `packages/polyfill-connectors/connectors/heb/index.ts`. Verdict:
+ **qualifies at day granularity, under a closed-day rule**.
+- [x] 2.2 Identify the frontier hazard: `runForwardScan` tracks
+ `newestOrderDate` as a running max (`:901-903`) while orders within a day
+ sort by full timestamp, so the newest day observed may be partially
+ enumerated. Answered by claiming a day only once a strictly older day has
+ been observed.
+- [x] 2.3 Identify the `dateDropped` hazard: an order whose date does not parse
+ is recorded as considered-but-not-covered (`:849-852`) and has no day to
+ be attributed to, so it is a hole of unknown position. Answered by
+ withholding the whole run's claim when `dateDropped` is non-empty.
+- [x] 2.4 Confirm truncation is expressible rather than blocking:
+ `MAX_LIST_PAGES = 50` (`:64`) and `maxPage` exhaustion (`:919-921`) end a
+ walk early, which under covered intervals is simply a smaller covered set.
+- [x] 2.5 Confirm `CHECKPOINT_OVERLAP_DAYS = 60` (`:70`) is a freshness device
+ via `resumeBoundary` (`:1064-1073`), not part of the safety argument.
+- [x] 2.6 Re-confirm `slack` disqualification:
+ `grep -c "ORDER BY" connectors/slack/index.ts` returns `0` on this branch.
+- [x] 2.7 Check whether `fix/slack-emitted-watermark-0821` changes that.
+ It does not: `grep -c "ORDER BY"` on that branch's
+ `connectors/slack/index.ts` also returns `0`. The fix corrects
+ emitted-vs-iterated watermark advance and removes the global-floor
+ `COALESCE`, but adds no ordering.
+
+## 3. Correct the redo-cost statement
+
+- [x] 3.1 Verify by content that no mid-run commit path exists: `newState` is
+ in-memory (`reference-implementation/runtime/index.ts:2762`, assigned at
+ `:4237`) and `commitState` has exactly two call sites (`:5278`, `:5322`),
+ both inside the DONE gate at `:5266`.
+- [x] 3.2 Correct the wording in the research entry
+ `ai/research/pdpp/the-checkpoint-protocol-must-carry-a-proven-boundary-...md`
+ §5, from "work done since the last checkpoint is redone" to redo since the
+ last *committed* cursor.
+- [x] 3.3 Record the correction in this change's `design.md` so the corrected
+ cost-benefit is visible to a reviewer who does not read the corpus.
+- [x] 3.4 Confirm the wording does not appear in any other OpenSpec artifact,
+ branch design note, or research entry.
+
+## 4. Record the ideal-compatibility rule
+
+- [x] 4.1 State the rule in `design.md` with its placement justification.
+- [x] 4.2 Restate it as a requirement in the `polyfill-runtime` spec delta so it
+ survives archival into `openspec/specs/`.
+- [x] 4.3 Decide against `openspec/README.md`: that file is scoped to OpenSpec
+ process, not to program-specific engineering constraints.
+
+## 5. Validation
+
+- [x] 5.1 `openspec validate qualify-connectors-for-incremental-checkpoint-commit --strict`
+ passes.
+- [x] 5.2 `openspec validate --all --strict` shows the same 11 pre-existing
+ failures as the baseline captured before this change, with this change
+ passing.
+
+## 6. Deferred to later tranches
+
+- [ ] 6.1 Add the optional `checkpoint_claim` field to
+ `packages/polyfill-connectors/src/connector-runtime-protocol.ts`.
+- [ ] 6.2 Implement the runtime decision procedure at the `STATE` handler.
+- [ ] 6.3 Claim in `gmail` `messages` first.
+- [ ] 6.4 Owner-authorized live confirmation of the `heb` day-granularity
+ verdict. Deferred here because a `heb` run costs the owner a real OTP, and
+ both rules fail closed so the cost of a misreading is a withheld commit.
diff --git a/packages/cli/scripts/package-contract.ts b/packages/cli/scripts/package-contract.ts
index 205222544..9b47241fe 100644
--- a/packages/cli/scripts/package-contract.ts
+++ b/packages/cli/scripts/package-contract.ts
@@ -3,12 +3,27 @@
import assert from "node:assert/strict";
import { existsSync, readFileSync, statSync } from "node:fs";
+import { builtinModules } from "node:module";
import { relative, resolve, sep } from "node:path";
const TEST_ARTIFACT = /(^|\/)\.?.+\.test\.(?:js|mjs|cjs|ts|mts|cts)$/;
const WHITESPACE = /\s/;
const NPM_PACK_OUTPUT_MAX_BYTES = 8 * 1024 * 1024;
+// Node's built-in module names, with and without the `node:` prefix.
+const NODE_BUILTIN_SPECIFIERS = new Set(builtinModules.flatMap((name) => [name, `node:${name}`]));
+
+// Static `import`/`export` are anchored to the start of a line (optionally
+// indented): tsc/esbuild output always emits these as statements starting a
+// line, never mid-expression, so anchoring avoids false positives on runtime
+// code that merely contains the words "import"/"export" inside a string or
+// property access. `export` additionally requires a trailing `from "…"` —
+// the only valid syntax for a re-export naming a module specifier.
+const STATIC_IMPORT_OR_EXPORT_FROM =
+ /^[ \t]*(?:import\s+(?:[^"'\n;]*?\s+from\s+)?["']([^"'.][^"']*)["']|export\s+[^"'\n;]*?\s+from\s+["']([^"'.][^"']*)["'])/gm;
+const DYNAMIC_IMPORT = /\bimport\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g;
+const REQUIRE_CALL = /\brequire\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g;
+
interface ExportTarget {
label: string;
target: string;
@@ -75,9 +90,12 @@ function collectExportTargets(value: unknown, label: string, targets: ExportTarg
export interface PackageManifest {
bin: Record;
+ dependencies?: Record;
exports: Record;
files: string[];
name: string;
+ optionalDependencies?: Record;
+ peerDependencies?: Record;
}
export function assertManifestTargets(manifest: unknown, packageRoot: string): asserts manifest is PackageManifest {
@@ -146,6 +164,107 @@ export function assertPackedFiles(manifest: PackageManifest, packedFiles: string
}
}
+/**
+ * Resolve a bare import specifier to the npm package name it names: the
+ * whole specifier for an unscoped package (`zod` from `zod/v4`), or the
+ * first two path segments for a scoped package (`@pdpp/read-core` from
+ * `@pdpp/read-core/records`).
+ */
+function bareSpecifierPackageName(specifier: string): string {
+ const segments = specifier.split("/");
+ if (specifier.startsWith("@")) {
+ return segments.slice(0, 2).join("/");
+ }
+ return segments[0];
+}
+
+function isBareSpecifier(specifier: string): boolean {
+ return !(specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("node:"));
+}
+
+/**
+ * Extract every bare (non-relative, non-absolute) import/export/require
+ * specifier a compiled `.js`/`.mjs`/`.d.ts` file references: static
+ * `import … from "x"` (including the bare side-effect form `import "x"`),
+ * `export … from "x"`, dynamic `import("x")`, and `require("x")`.
+ */
+function bareImportSpecifiers(source: string): Set {
+ const specifiers = new Set();
+ for (const match of source.matchAll(STATIC_IMPORT_OR_EXPORT_FROM)) {
+ const specifier = match[1] ?? match[2];
+ if (specifier && isBareSpecifier(specifier) && !specifier.startsWith("node:")) {
+ specifiers.add(specifier);
+ }
+ }
+ for (const pattern of [DYNAMIC_IMPORT, REQUIRE_CALL]) {
+ for (const [, specifier] of source.matchAll(pattern)) {
+ if (isBareSpecifier(specifier) && !specifier.startsWith("node:")) {
+ specifiers.add(specifier);
+ }
+ }
+ }
+ return specifiers;
+}
+
+/**
+ * Every published `@pdpp/local-collector` 1.5.1-1.5.4 shipped a compiled
+ * `import … from "@pdpp/reference-contract/common"` that was not in
+ * `dependencies` and does not exist on the npm registry: it resolved for
+ * every developer through the pnpm workspace link and failed closed for
+ * every real npm install with `ERR_MODULE_NOT_FOUND`. Neither
+ * `assertManifestTargets` (declared dependency sections only) nor
+ * `assertPackedFiles` (packed file layout only) looks at what the packed
+ * code actually imports, so a bare specifier undeclared in package.json can
+ * slip through both untouched. This closes that gap: every bare import,
+ * export-from, dynamic import(), and require() specifier compiled into the
+ * packed `.js`/`.mjs`/`.d.ts` files must resolve to either a Node builtin or
+ * a package the manifest actually declares as a real (non-workspace,
+ * non-file:) dependency. `@pdpp/cli` currently declares no runtime
+ * dependencies at all, so today this means: no bare specifiers other than
+ * Node builtins may appear in the packed output.
+ */
+export function assertBareSpecifiersResolve(
+ manifest: PackageManifest,
+ extractedRoot: string,
+ packedFiles: string[]
+): void {
+ const declaredPackages = new Set([
+ ...Object.keys(manifest.dependencies ?? {}),
+ ...Object.keys(manifest.peerDependencies ?? {}),
+ ...Object.keys(manifest.optionalDependencies ?? {}),
+ ]);
+
+ for (const file of packedFiles) {
+ if (!(file.endsWith(".js") || file.endsWith(".mjs") || file.endsWith(".d.ts"))) {
+ continue;
+ }
+ const source = readFileSync(resolve(extractedRoot, file), "utf8");
+ for (const specifier of bareImportSpecifiers(source)) {
+ const packageName = bareSpecifierPackageName(specifier);
+ if (NODE_BUILTIN_SPECIFIERS.has(specifier) || NODE_BUILTIN_SPECIFIERS.has(packageName)) {
+ continue;
+ }
+ if (declaredPackages.has(packageName)) {
+ continue;
+ }
+ if (packageName.startsWith("@pdpp/")) {
+ throw new Error(
+ `${file} imports private workspace package "${packageName}" (specifier "${specifier}") which is not ` +
+ "declared in dependencies/peerDependencies/optionalDependencies. This is the exact defect that made " +
+ "every published @pdpp/local-collector 1.5.1-1.5.4 unrunnable (ERR_MODULE_NOT_FOUND on every " +
+ "install). Declare a real dependency, vendor the needed symbol, or rewrite the specifier at build " +
+ "time before packing."
+ );
+ }
+ throw new Error(
+ `${file} imports "${specifier}" (package "${packageName}") which is not declared in ` +
+ "dependencies/peerDependencies/optionalDependencies and is not a Node builtin. A clean npm install of " +
+ "this package would fail to resolve this import at runtime."
+ );
+ }
+ }
+}
+
export function parseNpmPackOutput(output: string): NpmPackResult[] {
assert.ok(
Buffer.byteLength(output, "utf8") <= NPM_PACK_OUTPUT_MAX_BYTES,
diff --git a/packages/cli/scripts/validate-package.ts b/packages/cli/scripts/validate-package.ts
index a2a7794c1..9398cba62 100644
--- a/packages/cli/scripts/validate-package.ts
+++ b/packages/cli/scripts/validate-package.ts
@@ -8,6 +8,7 @@ import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import {
+ assertBareSpecifiersResolve,
assertManifestTargets,
assertPackedFiles,
type PackageManifest,
@@ -51,6 +52,7 @@ try {
encoding: "utf8",
});
assertManifestTargets(manifest, join(extractionRoot, "package"));
+ assertBareSpecifiersResolve(manifest, join(extractionRoot, "package"), packedFiles);
process.stdout.write(`Validated ${pack.filename} (${packedFiles.length} files).\n`);
} finally {
rmSync(tempRoot, { force: true, recursive: true });
diff --git a/packages/cli/test/artifact-contract.test.ts b/packages/cli/test/artifact-contract.test.ts
index b2a104ebf..fe472a8f4 100644
--- a/packages/cli/test/artifact-contract.test.ts
+++ b/packages/cli/test/artifact-contract.test.ts
@@ -10,9 +10,12 @@ import { fileURLToPath } from "node:url";
import { assertArtifactReceipt, bindNodeEnvironment, gitHeadSha } from "../scripts/artifact-receipt.ts";
import { discoverTestFiles, needsTsx } from "../scripts/discover-tests.ts";
-import { assertManifestTargets } from "../scripts/package-contract.ts";
+import { assertBareSpecifiersResolve, assertManifestTargets } from "../scripts/package-contract.ts";
const packageRoot = fileURLToPath(new URL("..", import.meta.url));
+const UNDECLARED_PDPP_IMPORT =
+ /imports private workspace package "@pdpp\/reference-contract".*1\.5\.1-1\.5\.4 unrunnable/s;
+const UNDECLARED_THIRD_PARTY_IMPORT = /imports "left-pad".*not declared in dependencies/s;
function makeManifest(overrides = {}) {
return {
@@ -76,6 +79,44 @@ test("artifact contract rejects a bin that loses its shebang", () => {
assert.throws(() => assertManifestTargets(makeManifest(), root), /must retain its node shebang/);
});
+test("bare-specifier check rejects the exact @pdpp/local-collector 1.5.1-1.5.4 defect shape: an undeclared private-package import", () => {
+ const root = emittedFixture();
+ writeFileSync(
+ join(root, "dist", "src", "index.js"),
+ 'import { canonicalTerminalRunCommitEnvelope } from "@pdpp/reference-contract/common";\nexport const artifact = true;\n'
+ );
+ assert.throws(
+ () => assertBareSpecifiersResolve(makeManifest(), root, ["dist/src/index.js", "dist/bin/pdpp.js"]),
+ UNDECLARED_PDPP_IMPORT
+ );
+});
+
+test("bare-specifier check rejects any undeclared bare import, not just @pdpp/* ones", () => {
+ const root = emittedFixture();
+ writeFileSync(
+ join(root, "dist", "src", "index.js"),
+ 'import leftPad from "left-pad";\nexport const artifact = true;\n'
+ );
+ assert.throws(
+ () => assertBareSpecifiersResolve(makeManifest(), root, ["dist/src/index.js", "dist/bin/pdpp.js"]),
+ UNDECLARED_THIRD_PARTY_IMPORT
+ );
+});
+
+test("bare-specifier check accepts declared dependencies and Node builtins", () => {
+ const root = emittedFixture();
+ writeFileSync(
+ join(root, "dist", "src", "index.js"),
+ 'import { z } from "zod";\nimport path from "node:path";\nexport const artifact = true;\n'
+ );
+ assert.doesNotThrow(() =>
+ assertBareSpecifiersResolve(makeManifest({ dependencies: { zod: "^4.4.3" } }), root, [
+ "dist/src/index.js",
+ "dist/bin/pdpp.js",
+ ])
+ );
+});
+
test("extension-complete discovery and loader selection are exact", async () => {
const root = mkdtempSync(join(tmpdir(), "pdpp-cli-test-discovery-"));
const cases = [
diff --git a/packages/display/package.json b/packages/display/package.json
index 1526d57a7..1bdf408ab 100644
--- a/packages/display/package.json
+++ b/packages/display/package.json
@@ -5,7 +5,8 @@
"type": "module",
"description": "Framework-independent rules that turn PDPP values and metadata into human-readable display models.",
"exports": {
- ".": "./src/index.ts"
+ ".": "./src/index.ts",
+ "./health": "./src/health/axis-vocabulary.ts"
},
"scripts": {
"typecheck": "tsc --noEmit",
diff --git a/packages/display/src/health/axis-vocabulary.test.ts b/packages/display/src/health/axis-vocabulary.test.ts
new file mode 100644
index 000000000..608a54d89
--- /dev/null
+++ b/packages/display/src/health/axis-vocabulary.test.ts
@@ -0,0 +1,84 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+ formatAttentionAxis,
+ formatCoverageAxis,
+ formatFreshnessAxis,
+ formatOutboxAxis,
+} from "./axis-vocabulary.ts";
+
+test("coverage axis maps all known states to owner-facing words with correct tone", () => {
+ assert.equal(formatCoverageAxis("complete").value, "complete");
+ assert.equal(formatCoverageAxis("complete").tone, "success");
+
+ assert.equal(formatCoverageAxis("deferred").value, "optional, not collected");
+ assert.equal(formatCoverageAxis("deferred").tone, "neutral");
+
+ assert.equal(formatCoverageAxis("gaps").value, "gaps");
+ assert.equal(formatCoverageAxis("gaps").tone, "warning");
+
+ assert.equal(formatCoverageAxis("inventory_only").value, "complete (list only, by design)");
+ assert.equal(formatCoverageAxis("inventory_only").tone, "neutral");
+
+ assert.equal(formatCoverageAxis("partial").value, "partial");
+ assert.equal(formatCoverageAxis("partial").tone, "warning");
+
+ assert.equal(formatCoverageAxis("retryable_gap").value, "retryable gap");
+ assert.equal(formatCoverageAxis("retryable_gap").tone, "warning");
+
+ assert.equal(formatCoverageAxis("terminal_gap").value, "won't backfill");
+ assert.equal(formatCoverageAxis("terminal_gap").tone, "danger");
+
+ assert.equal(formatCoverageAxis("unavailable").value, "unavailable");
+ assert.equal(formatCoverageAxis("unavailable").tone, "neutral");
+
+ assert.equal(formatCoverageAxis("unknown").value, "not measured");
+ assert.equal(formatCoverageAxis("unknown").tone, "neutral");
+
+ assert.equal(formatCoverageAxis("unsupported").value, "unsupported");
+ assert.equal(formatCoverageAxis("unsupported").tone, "neutral");
+});
+
+test("freshness axis maps known states to owner-facing words with correct tone", () => {
+ assert.equal(formatFreshnessAxis("fresh").value, "fresh");
+ assert.equal(formatFreshnessAxis("fresh").tone, "success");
+
+ assert.equal(formatFreshnessAxis("stale").value, "stale");
+ assert.equal(formatFreshnessAxis("stale").tone, "warning");
+
+ assert.equal(formatFreshnessAxis("unknown").value, "not measured");
+ assert.equal(formatFreshnessAxis("unknown").tone, "neutral");
+});
+
+test("outbox axis maps known states to owner-facing words with correct tone", () => {
+ assert.equal(formatOutboxAxis("active").value, "active");
+ assert.equal(formatOutboxAxis("active").tone, "success");
+
+ assert.equal(formatOutboxAxis("idle").value, "idle");
+ assert.equal(formatOutboxAxis("idle").tone, "success");
+
+ assert.equal(formatOutboxAxis("stalled").value, "stalled");
+ assert.equal(formatOutboxAxis("stalled").tone, "danger");
+
+ assert.equal(formatOutboxAxis("unknown").value, "not measured");
+ assert.equal(formatOutboxAxis("unknown").tone, "neutral");
+});
+
+test("attention axis maps known states to owner-facing words with correct tone", () => {
+ assert.equal(formatAttentionAxis("acknowledged")?.value, "acknowledged");
+ assert.equal(formatAttentionAxis("acknowledged")?.tone, "warning");
+
+ assert.equal(formatAttentionAxis("in_progress")?.value, "in progress");
+ assert.equal(formatAttentionAxis("in_progress")?.tone, "warning");
+
+ assert.equal(formatAttentionAxis("none"), null);
+
+ assert.equal(formatAttentionAxis("open")?.value, "open");
+ assert.equal(formatAttentionAxis("open")?.tone, "warning");
+
+ assert.equal(formatAttentionAxis("unknown_state")?.value, "not measured");
+ assert.equal(formatAttentionAxis("unknown_state")?.tone, "neutral");
+});
diff --git a/packages/display/src/health/axis-vocabulary.ts b/packages/display/src/health/axis-vocabulary.ts
new file mode 100644
index 000000000..c1ac47122
--- /dev/null
+++ b/packages/display/src/health/axis-vocabulary.ts
@@ -0,0 +1,313 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * The owner-facing vocabulary for the reference's connection-health axes.
+ *
+ * This module is the SINGLE source of the words an owner reads for a
+ * coverage / freshness / outbox / attention axis. It lives in
+ * `@pdpp/display` — not in the console app — because more than one surface
+ * must say the same thing about the same evidence:
+ *
+ * - the console `/sources` page and its source-detail subpages, and
+ * - `pdpp ref sources`, the headless CLI that exists so an agent and the
+ * owner can read the SAME rendered verdict rather than the agent reading
+ * raw `connector_summary_evidence` rows (the INPUTS) while the owner reads
+ * the rendered UI (the OUTPUT). Those two diverged badly enough to cause
+ * repeated miscommunication; a second copy of these strings would
+ * reintroduce exactly that drift.
+ *
+ * Nothing here reads the network, touches React, or depends on a framework.
+ * The axis KEYS are the reference server's durable wire contract; the
+ * `value`/`title` text is owner-facing copy this module owns. Where the two
+ * deliberately diverge (`deferred`, `terminal_gap`) the reason is commented
+ * inline — do not "fix" the mismatch by renaming a wire key.
+ */
+
+export type EvidenceTone = "neutral" | "success" | "warning" | "danger";
+
+export interface AxisChip {
+ /** The axis name (e.g. "Coverage", "Freshness"). Rendered muted. */
+ dimension: string;
+ /** Short owner-facing label (e.g. "Coverage · gaps"). Kept for backward compat/tooltips. */
+ label: string;
+ /** Long-form hover/tooltip — describes what the chip means. */
+ title: string;
+ tone: EvidenceTone;
+ /** The axis state value (e.g. "gaps", "fresh"). Rendered prominent. */
+ value: string;
+}
+
+/** The reference's `connection_health.axes.coverage` domain. */
+export type CoverageAxis =
+ | "complete"
+ | "deferred"
+ | "gaps"
+ | "inventory_only"
+ | "partial"
+ | "retryable_gap"
+ | "terminal_gap"
+ | "unavailable"
+ | "unknown"
+ | "unsupported";
+
+/** The reference's `connection_health.axes.freshness` domain. */
+export type FreshnessAxis = "fresh" | "stale" | "unknown";
+
+/** The reference's `connection_health.axes.outbox` domain. */
+export type OutboxAxis = "active" | "idle" | "stalled" | "unknown";
+
+/** The reference's `connection_health.axes.attention` domain. */
+export type AttentionAxis = "acknowledged" | "in_progress" | "none" | "open";
+
+const COVERAGE_LABELS: Record = {
+ complete: {
+ dimension: "Coverage",
+ label: "Coverage · complete",
+ title: "All required streams have durable evidence of complete coverage.",
+ tone: "success",
+ value: "complete",
+ },
+ deferred: {
+ dimension: "Coverage",
+ label: "Coverage · optional, not collected",
+ title:
+ "The manifest declares this coverage out of scope. This is an accepted, settled state — not a queued task — and does not block connection health.",
+ tone: "neutral",
+ // The underlying axis key stays "deferred" (durable manifest/runtime
+ // contract — see AcceptedCoveragePolicy in connector-coverage-policy.ts).
+ // "Deferred" read as queued/pending work to owners, contradicting the
+ // settled, non-degrading semantics this axis actually carries. The
+ // visible value/label now say plainly that this stream is optional and
+ // not collected; the manifest-declaration detail moves to the title.
+ value: "optional, not collected",
+ },
+ gaps: {
+ dimension: "Coverage",
+ label: "Coverage · gaps",
+ title: "Required coverage has known retryable or terminal gaps.",
+ tone: "warning",
+ value: "gaps",
+ },
+ inventory_only: {
+ dimension: "Coverage",
+ label: "Coverage · complete (list only, by design)",
+ title:
+ "The manifest declares that this stream only ever lists what exists, rather than downloading each item's full detail. Everything it is designed to collect has been collected, so nothing is missing and nothing is owed. This is a settled, finished state — not partial progress.",
+ tone: "neutral",
+ // B4 (owner ledger 2026-08-22): the owner asked whether the green/neutral
+ // tone here was intentional. It is, and it is honest: `inventory_only` is an
+ // AcceptedAbsencePolicy (packages/reference-contract/src/evidence/
+ // coherence.ts) that the manifest declares, `hasOutstandingGap` excludes,
+ // and `deriveForwardDisposition` resolves to `complete` — the connection
+ // genuinely owes no further data. Tone therefore stays `neutral`. What was
+ // wrong was the WORD: "inventory only" reads as a limitation the owner
+ // might need to act on, so the value now states plainly that this is
+ // complete by design.
+ value: "complete (list only, by design)",
+ },
+ partial: {
+ dimension: "Coverage",
+ label: "Coverage · partial",
+ title: "Some required streams collected only partial data.",
+ tone: "warning",
+ value: "partial",
+ },
+ retryable_gap: {
+ dimension: "Coverage",
+ label: "Coverage · retryable gap",
+ title:
+ "Some required detail is missing, but the runtime expects to fill it on a later run. Records already collected stay valid; no owner action is needed yet.",
+ tone: "warning",
+ value: "retryable gap",
+ },
+ terminal_gap: {
+ dimension: "Coverage",
+ label: "Coverage · won't backfill",
+ title:
+ "Some required detail will not backfill on its own — the connector or source cannot recover it without a change. Records already collected stay valid and usable; this is not current data loss. Open the connection's latest run to see which streams are affected and the recovery step.",
+ tone: "danger",
+ // "terminal gap" is jargon. The value stays short for the chip; the title
+ // carries the three things the owner actually needs (per design-notes/
+ // dashboard-health-semantics-and-reliability-2026-06-01.md): what state this
+ // is, whether current records are safe, and what can recover coverage. The
+ // reference's coverage condition carries a `Review source coverage gaps`
+ // remediation but not the specific cause/stream/time — that contract gap is
+ // noted in the workstream report; the per-stream detail lives in the latest
+ // run's known_gaps, which the connection detail page links to.
+ value: "won't backfill",
+ },
+ unavailable: {
+ dimension: "Coverage",
+ label: "Coverage · unavailable",
+ title:
+ "The manifest accepts that the source does not expose this coverage. This is a settled state, not a temporary gap awaiting a retry.",
+ tone: "neutral",
+ value: "unavailable",
+ },
+ unknown: {
+ dimension: "Coverage",
+ label: "Coverage · not measured",
+ title:
+ "Nothing has measured how much of this stream was collected, so we can't say whether it is complete. This is missing measurement, not proven missing data — records already collected stay valid.",
+ tone: "neutral",
+ // B2 (owner ledger 2026-08-22): the chip said "unknown" while the adjacent
+ // forward-disposition line said "not measured" for the SAME underlying
+ // state, so the owner read two vocabularies for one thing. The axis key
+ // stays `unknown` (durable wire contract); the owner-facing word is now
+ // "not measured" everywhere, matching FORWARD_DISPOSITION_LABELS.unmeasured
+ // and SOURCE_WORK_GROUP_COPY.notMeasured. Tone stays `neutral` because
+ // `reference-surface-topology` requires that unknown alone SHALL NOT
+ // produce degraded tone — this is an absence of measurement, not a defect.
+ value: "not measured",
+ },
+ unsupported: {
+ dimension: "Coverage",
+ label: "Coverage · unsupported",
+ title:
+ "The manifest accepts that the connector cannot collect this coverage. This is a settled state, not a temporary gap awaiting a retry.",
+ tone: "neutral",
+ value: "unsupported",
+ },
+};
+
+const FRESHNESS_LABELS: Record = {
+ fresh: {
+ dimension: "Freshness",
+ label: "Freshness · fresh",
+ title: "The last successful run is within policy.",
+ tone: "success",
+ value: "fresh",
+ },
+ stale: {
+ dimension: "Freshness",
+ label: "Freshness · stale",
+ title: "The last successful run is outside the configured freshness window.",
+ tone: "warning",
+ value: "stale",
+ },
+ unknown: {
+ dimension: "Freshness",
+ label: "Freshness · not measured",
+ title: "Nothing has measured how recent this data is, so we can't say whether it is up to date.",
+ tone: "neutral",
+ // B2: same one-word rule as the coverage axis — "not measured" is the
+ // single owner-facing word for "no evidence has been taken", on every axis.
+ value: "not measured",
+ },
+};
+
+const OUTBOX_LABELS: Record = {
+ active: {
+ dimension: "Outbox",
+ label: "Outbox · active",
+ title: "Outbound work is making progress.",
+ // `active` means the local-device outbox is draining — a healthy,
+ // progressing state. It previously shared `neutral` (muted grey) with
+ // `unknown`, so an operator could not tell a draining outbox from one
+ // whose evidence we could not read. `success` gives it a distinct,
+ // non-alarming colour (the same green as `idle`); the value text
+ // ("active" vs "idle") carries the finer distinction, and the row-level
+ // pill still escalates an actively-draining outbox to a "Syncing" badge.
+ tone: "success",
+ value: "active",
+ },
+ idle: {
+ dimension: "Outbox",
+ label: "Outbox · idle",
+ title: "No retryable outbound work is pending.",
+ tone: "success",
+ value: "idle",
+ },
+ stalled: {
+ dimension: "Outbox",
+ label: "Outbox · stalled",
+ title: "Retryable outbound work is stalled and not progressing.",
+ tone: "danger",
+ value: "stalled",
+ },
+ unknown: {
+ dimension: "Outbox",
+ label: "Outbox · not measured",
+ title: "Nothing has measured the upload queue on this device, so we can't say whether it is keeping up.",
+ tone: "neutral",
+ // B2: one word for "no evidence taken" across every axis.
+ value: "not measured",
+ },
+};
+
+const ATTENTION_LABELS: Record = {
+ acknowledged: {
+ dimension: "Attention",
+ label: "Attention · acknowledged",
+ title: "Owner action is acknowledged but not yet resolved.",
+ tone: "warning",
+ value: "acknowledged",
+ },
+ in_progress: {
+ dimension: "Attention",
+ label: "Attention · in progress",
+ title: "Owner action is in progress.",
+ tone: "warning",
+ value: "in progress",
+ },
+ none: null,
+ open: {
+ dimension: "Attention",
+ label: "Attention · open",
+ title: "Owner action is open.",
+ tone: "warning",
+ value: "open",
+ },
+};
+
+export function formatCoverageAxis(axis: CoverageAxis | null | string | undefined): AxisChip {
+ return formatKnownAxis(COVERAGE_LABELS, axis, "unknown", "Coverage");
+}
+
+export function formatFreshnessAxis(axis: FreshnessAxis | null | string | undefined): AxisChip {
+ return formatKnownAxis(FRESHNESS_LABELS, axis, "unknown", "Freshness");
+}
+
+export function formatOutboxAxis(axis: OutboxAxis | null | string | undefined): AxisChip {
+ return formatKnownAxis(OUTBOX_LABELS, axis, "unknown", "Outbox");
+}
+
+export function formatAttentionAxis(axis: AttentionAxis | null | string | undefined): AxisChip | null {
+ if (axis === null) {
+ return null;
+ }
+ if (axis !== undefined && Object.hasOwn(ATTENTION_LABELS, axis)) {
+ return ATTENTION_LABELS[axis as AttentionAxis];
+ }
+ return {
+ dimension: "Attention",
+ label: "Attention · not measured",
+ title: `This console does not recognize the attention state "${axis}" reported by the reference server, so it cannot say whether anything needs you.`,
+ tone: "neutral",
+ value: "not measured",
+ };
+}
+
+function formatKnownAxis(
+ labels: Record,
+ axis: T | null | string | undefined,
+ fallback: T,
+ labelPrefix: string
+): AxisChip {
+ if (axis !== null && axis !== undefined && Object.hasOwn(labels, axis)) {
+ return labels[axis as T];
+ }
+ const fallbackChip = labels[fallback];
+ if (axis === null) {
+ return fallbackChip;
+ }
+ return {
+ ...fallbackChip,
+ dimension: labelPrefix,
+ title: `This console does not recognize the ${labelPrefix.toLowerCase()} state "${axis}" reported by the reference server, so it cannot say what was measured.`,
+ // B2: an unrecognized axis is still "we have no usable measurement", so it
+ // uses the same one word rather than introducing a third vocabulary.
+ value: "not measured",
+ };
+}
diff --git a/packages/display/src/index.ts b/packages/display/src/index.ts
index d95da5e03..0dad1a0e1 100644
--- a/packages/display/src/index.ts
+++ b/packages/display/src/index.ts
@@ -22,6 +22,20 @@ export type {
TraceLabelInput,
} from "./identity/summary-row-label.ts";
export { grantRowLabel, runRowLabel, traceRowLabel } from "./identity/summary-row-label.ts";
+export type {
+ AttentionAxis,
+ AxisChip,
+ CoverageAxis,
+ EvidenceTone,
+ FreshnessAxis,
+ OutboxAxis,
+} from "./health/axis-vocabulary.ts";
+export {
+ formatAttentionAxis,
+ formatCoverageAxis,
+ formatFreshnessAxis,
+ formatOutboxAxis,
+} from "./health/axis-vocabulary.ts";
export type { DeclaredFieldRoles, FieldRole } from "./record/declared-field-roles.ts";
export {
EMPTY_DECLARED_FIELD_ROLES,
diff --git a/packages/display/src/record/structured-value.test.ts b/packages/display/src/record/structured-value.test.ts
index 782191306..727d43a51 100644
--- a/packages/display/src/record/structured-value.test.ts
+++ b/packages/display/src/record/structured-value.test.ts
@@ -19,11 +19,11 @@ test("an empty array reads as an explicit empty state, not a blank or raw '[]'",
test("an array of objects with `name` fields renders as joined names — the Gmail `cc` case", () => {
const cc = [
- { email: "mmarco@law.harvard.edu", name: "Meg Marco" },
- { email: "anna@opendatalabs.xyz", name: "Anna Kazlauskas" },
+ { email: "rowan.diaz@example.edu", name: "Rowan Diaz" },
+ { email: "sasha.lindqvist@example.org", name: "Sasha Lindqvist" },
];
const result = formatStructuredCell(cc);
- assert.equal(result?.text, "Meg Marco, Anna Kazlauskas");
+ assert.equal(result?.text, "Rowan Diaz, Sasha Lindqvist");
assert.equal(result?.detail, undefined, "under the item cap, no separate detail is needed");
});
diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json
index b2ef67239..cb89e59c7 100644
--- a/packages/mcp-server/package.json
+++ b/packages/mcp-server/package.json
@@ -28,7 +28,7 @@
"test:read-surface": "node --import tsx ../../scripts/test-scratch/run-command.ts -- node ../../scripts/run-node-tests.mjs --test --import tsx \"test/*.test.ts\" test/smoke-stdio.ts",
"validate:package": "pnpm build && node --import tsx scripts/package-contract.ts",
"verify:artifact": "pnpm build && node --import tsx scripts/pack-install-run.ts",
- "verify": "pnpm test && pnpm validate:package",
+ "verify": "pnpm test && pnpm validate:package && pnpm verify:artifact",
"pack:dry-run": "pnpm build && npm pack --dry-run --ignore-scripts"
},
"dependencies": {
diff --git a/packages/mcp-server/scripts/package-contract.ts b/packages/mcp-server/scripts/package-contract.ts
index f95affd27..6da4546bd 100644
--- a/packages/mcp-server/scripts/package-contract.ts
+++ b/packages/mcp-server/scripts/package-contract.ts
@@ -4,6 +4,7 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { existsSync, readFileSync, rmSync, statSync } from "node:fs";
+import { builtinModules } from "node:module";
import { dirname, relative, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
@@ -11,6 +12,21 @@ const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const EXECUTABLE_PERMISSION = /[1357]/;
const TEST_ARTIFACT_PATH = /(^|\/)\.?.+\.test\.(?:js|mjs|cjs|ts|mts|cts)$/;
const NPM_PACK_JSON = /(\[\s*\{[\s\S]*\])\s*$/;
+const NPM_PACK_JSON_OBJECT = /(\{\s*"[^"]*"\s*:\s*\{[\s\S]*\})\s*$/;
+
+// Node's built-in module names, with and without the `node:` prefix.
+const NODE_BUILTIN_SPECIFIERS = new Set(builtinModules.flatMap((name) => [name, `node:${name}`]));
+
+// Static `import`/`export` are anchored to the start of a line (optionally
+// indented): tsc/esbuild output always emits these as statements starting a
+// line, never mid-expression, so anchoring avoids false positives on runtime
+// code that merely contains the words "import"/"export" inside a string or
+// property access. `export` additionally requires a trailing `from "…"` —
+// the only valid syntax for a re-export naming a module specifier.
+const STATIC_IMPORT_OR_EXPORT_FROM =
+ /^[ \t]*(?:import\s+(?:[^"'\n;]*?\s+from\s+)?["']([^"'.][^"']*)["']|export\s+[^"'\n;]*?\s+from\s+["']([^"'.][^"']*)["'])/gm;
+const DYNAMIC_IMPORT = /\bimport\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g;
+const REQUIRE_CALL = /\brequire\s*\(\s*["']([^"'.][^"']*)["']\s*\)/g;
// Loosely typed on purpose: this describes the runtime shape of an untrusted
// `package.json` read from disk, which assertManifestTargets/assertPackedFiles
@@ -150,10 +166,109 @@ export function assertPackedFiles(manifest: PackageManifest, packedFiles: string
}
}
+/**
+ * Resolve a bare import specifier to the npm package name it names: the
+ * whole specifier for an unscoped package (`zod` from `zod/v4`), or the
+ * first two path segments for a scoped package (`@pdpp/read-core` from
+ * `@pdpp/read-core/records`).
+ */
+function bareSpecifierPackageName(specifier: string): string {
+ const segments = specifier.split("/");
+ if (specifier.startsWith("@")) {
+ return segments.slice(0, 2).join("/");
+ }
+ return segments[0] ?? specifier;
+}
+
+function isBareSpecifier(specifier: string): boolean {
+ return !(specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("node:"));
+}
+
+/**
+ * Extract every bare (non-relative, non-absolute) import/export/require
+ * specifier a compiled `.js`/`.mjs`/`.d.ts` file references: static
+ * `import … from "x"` (including the bare side-effect form `import "x"`),
+ * `export … from "x"`, dynamic `import("x")`, and `require("x")`.
+ */
+function bareImportSpecifiers(source: string): Set {
+ const specifiers = new Set();
+ for (const match of source.matchAll(STATIC_IMPORT_OR_EXPORT_FROM)) {
+ const specifier = match[1] ?? match[2];
+ if (specifier && isBareSpecifier(specifier) && !specifier.startsWith("node:")) {
+ specifiers.add(specifier);
+ }
+ }
+ for (const pattern of [DYNAMIC_IMPORT, REQUIRE_CALL]) {
+ for (const [, specifier] of source.matchAll(pattern)) {
+ if (specifier && isBareSpecifier(specifier) && !specifier.startsWith("node:")) {
+ specifiers.add(specifier);
+ }
+ }
+ }
+ return specifiers;
+}
+
+/**
+ * Every published `@pdpp/local-collector` 1.5.1-1.5.4 shipped a compiled
+ * `import … from "@pdpp/reference-contract/common"` that was not in
+ * `dependencies` and does not exist on the npm registry: it resolved for
+ * every developer through the pnpm workspace link and failed closed for
+ * every real npm install with `ERR_MODULE_NOT_FOUND`. Neither
+ * `assertManifestTargets` (declared dependency sections only) nor
+ * `assertPackedFiles` (packed file layout only) looks at what the packed
+ * code actually imports, so a bare specifier undeclared in package.json
+ * can slip through both untouched. This closes that gap: every bare import,
+ * export-from, dynamic import(), and require() specifier compiled into the
+ * packed `.js`/`.mjs`/`.d.ts` files must resolve to either a Node builtin or
+ * a package the manifest actually declares as a real (non-workspace,
+ * non-file:) dependency.
+ */
+export function assertBareSpecifiersResolve(manifest: PackageManifest, root: string, packedFiles: string[]): void {
+ const declaredPackages = new Set(Object.keys(manifest.dependencies ?? {}));
+
+ for (const file of packedFiles) {
+ if (!(file.endsWith(".js") || file.endsWith(".mjs") || file.endsWith(".d.ts"))) {
+ continue;
+ }
+ const source = readFileSync(resolve(root, file), "utf8");
+ for (const specifier of bareImportSpecifiers(source)) {
+ const packageName = bareSpecifierPackageName(specifier);
+ if (NODE_BUILTIN_SPECIFIERS.has(specifier) || NODE_BUILTIN_SPECIFIERS.has(packageName)) {
+ continue;
+ }
+ if (declaredPackages.has(packageName)) {
+ continue;
+ }
+ if (packageName.startsWith("@pdpp/")) {
+ throw new Error(
+ `${file} imports private workspace package "${packageName}" (specifier "${specifier}") which is not ` +
+ "declared in dependencies. This is the exact defect that made every published @pdpp/local-collector " +
+ "1.5.1-1.5.4 unrunnable (ERR_MODULE_NOT_FOUND on every install). Declare a real dependency, vendor " +
+ "the needed symbol, or rewrite the specifier at build time before packing."
+ );
+ }
+ throw new Error(
+ `${file} imports "${specifier}" (package "${packageName}") which is not declared in dependencies and is ` +
+ "not a Node builtin. A clean npm install of this package would fail to resolve this import at runtime."
+ );
+ }
+ }
+}
+
export function parseNpmPackOutput(output: string): NpmPackEntry[] {
- const match = output.match(NPM_PACK_JSON);
- assert.ok(match, "npm pack did not produce a trailing JSON payload");
- return JSON.parse(match[1] as string) as NpmPackEntry[];
+ // npm's `pack --json` output shape changed across major versions: older npm
+ // (<=11) emits a top-level array of one record; npm 12 emits an object
+ // keyed by package name instead. Accept either, and tolerate `npm warn`
+ // lines ahead of the JSON payload (observed in this environment), rather
+ // than pinning this check to one npm major/config shape.
+ const arrayMatch = output.match(NPM_PACK_JSON);
+ if (arrayMatch) {
+ return JSON.parse(arrayMatch[1] as string) as NpmPackEntry[];
+ }
+ const objectMatch = output.match(NPM_PACK_JSON_OBJECT);
+ assert.ok(objectMatch, "npm pack did not produce a trailing JSON payload");
+ const parsed = JSON.parse(objectMatch[1] as string) as Record;
+ return Object.values(parsed);
}
export function packAndInspect(root: string, manifest: PackageManifest): NpmPackEntry {
@@ -163,10 +278,9 @@ export function packAndInspect(root: string, manifest: PackageManifest): NpmPack
});
const [pack] = parseNpmPackOutput(output);
assert.ok(pack, "npm pack produced no entries");
- assertPackedFiles(
- manifest,
- pack.files.map((file) => file.path)
- );
+ const packedFiles = pack.files.map((file) => file.path);
+ assertPackedFiles(manifest, packedFiles);
+ assertBareSpecifiersResolve(manifest, root, packedFiles);
return pack;
}
diff --git a/packages/mcp-server/test/artifact-contract.test.ts b/packages/mcp-server/test/artifact-contract.test.ts
index 75c9589ae..8c7412fde 100644
--- a/packages/mcp-server/test/artifact-contract.test.ts
+++ b/packages/mcp-server/test/artifact-contract.test.ts
@@ -23,7 +23,12 @@ import {
type SiblingCandidateEvidence,
} from "../scripts/artifact-receipt.ts";
import { assertInstalledPackageMatchesTarball, resolveReceiptOutputPath } from "../scripts/pack-install-run.ts";
-import { assertManifestTargets, assertPackedFiles, type PackageManifest } from "../scripts/package-contract.ts";
+import {
+ assertBareSpecifiersResolve,
+ assertManifestTargets,
+ assertPackedFiles,
+ type PackageManifest,
+} from "../scripts/package-contract.ts";
const SYMLINK = /symlink/;
const STALE_OR_REPLAYED_RECEIPT = /stale or replayed receipt/;
@@ -45,6 +50,9 @@ const SOURCE_TARGET = /must point into \.\/dist\//;
const SOURCE_FILE = /source file leaked/;
const SOURCE_FALLBACK = /resolved from source instead of the offline consumer/;
const REPLAYED_RECEIPT = /stale or replayed receipt/;
+const UNDECLARED_PDPP_IMPORT =
+ /imports private workspace package "@pdpp\/reference-contract".*1\.5\.1-1\.5\.4 unrunnable/s;
+const UNDECLARED_THIRD_PARTY_IMPORT = /imports "left-pad".*not declared in dependencies/s;
function manifest(overrides: Partial = {}): PackageManifest {
return {
@@ -133,6 +141,70 @@ test("artifact contract rejects a source fallback target and packed source files
);
});
+test("bare-specifier check rejects the exact @pdpp/local-collector 1.5.1-1.5.4 defect shape: an undeclared private-package import", () => {
+ const root = emittedFixture();
+ writeFileSync(
+ join(root, "dist", "src", "index.js"),
+ 'import { canonicalTerminalRunCommitEnvelope } from "@pdpp/reference-contract/common";\nexport const artifact = true;\n'
+ );
+ assert.throws(
+ () =>
+ assertBareSpecifiersResolve(manifest(), root, [
+ "dist/src/index.js",
+ "dist/src/server.js",
+ "dist/bin/pdpp-mcp-server.js",
+ ]),
+ UNDECLARED_PDPP_IMPORT
+ );
+});
+
+test("bare-specifier check rejects any undeclared bare import, not just @pdpp/* ones", () => {
+ const root = emittedFixture();
+ writeFileSync(
+ join(root, "dist", "src", "index.js"),
+ 'import leftPad from "left-pad";\nexport const artifact = true;\n'
+ );
+ assert.throws(
+ () =>
+ assertBareSpecifiersResolve(manifest(), root, [
+ "dist/src/index.js",
+ "dist/src/server.js",
+ "dist/bin/pdpp-mcp-server.js",
+ ]),
+ UNDECLARED_THIRD_PARTY_IMPORT
+ );
+});
+
+test("bare-specifier check accepts declared dependencies and Node builtins", () => {
+ const root = emittedFixture();
+ writeFileSync(
+ join(root, "dist", "src", "index.js"),
+ 'import { z } from "zod";\nimport path from "node:path";\nimport { cliThing } from "@pdpp/cli";\nexport const artifact = true;\n'
+ );
+ assert.doesNotThrow(() =>
+ assertBareSpecifiersResolve(manifest({ dependencies: { "@pdpp/cli": ">=0.18.11 <1.0.0", zod: "^4.4.3" } }), root, [
+ "dist/src/index.js",
+ "dist/src/server.js",
+ "dist/bin/pdpp-mcp-server.js",
+ ])
+ );
+});
+
+test("bare-specifier check ignores 'import'/'export' inside string literals and property access", () => {
+ const root = emittedFixture();
+ writeFileSync(
+ join(root, "dist", "src", "index.js"),
+ 'const assignment = line.startsWith("export ") ? line.slice("export ".length) : line;\nexport const artifact = true;\n'
+ );
+ assert.doesNotThrow(() =>
+ assertBareSpecifiersResolve(manifest(), root, [
+ "dist/src/index.js",
+ "dist/src/server.js",
+ "dist/bin/pdpp-mcp-server.js",
+ ])
+ );
+});
+
test("consumer proof rejects an installed package symlinked to source", () => {
const root = mkdtempSync(join(tmpdir(), "pdpp-mcp-source-fallback-"));
const tarRoot = join(root, "tar");
diff --git a/packages/polyfill-connectors/connectors/_conformance/change-feed-is-not-inventory.test.ts b/packages/polyfill-connectors/connectors/_conformance/change-feed-is-not-inventory.test.ts
new file mode 100644
index 000000000..020fd21d2
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/_conformance/change-feed-is-not-inventory.test.ts
@@ -0,0 +1,197 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Change-feed-is-not-inventory gate.
+ *
+ * CONTRACT UNDER TEST
+ * -------------------
+ * A stream whose manifest declares `coverage_strategy: "full_inventory"` may
+ * NOT prove that claim from a run whose executed code path was an incremental
+ * DELTA. An empty change feed and an empty inventory are different facts, and
+ * only one of them is evidence of anything.
+ *
+ * RFC 6578 §3.2 is normative here and makes this statically decidable: a
+ * sync-collection REPORT carrying an EMPTY `` MUST return every
+ * member of the collection, while one carrying a NON-EMPTY token returns only
+ * what changed since that token. So "the response listed 0 resources" means
+ * "0 contacts exist" in the first case and "nothing changed" in the second.
+ *
+ * THE DEFECT THIS PINS
+ * --------------------
+ * Apple Contacts declares `full_inventory` on all three of its streams, but
+ * its steady-state run issues a sync-collection delta. A quiet run therefore
+ * observed 0 changed resources and, before the fix, reported
+ * `considered: 0, covered: 0` — which the coherence oracle reads as a
+ * measured `enumeration_boundary` proving a verified-EMPTY address book. A
+ * required stream holding hundreds of real contacts read Healthy with zero
+ * records.
+ *
+ * The connector now withholds the coverage claim entirely on an incremental
+ * run (`contactsBoundaryEstablished`), leaving the stream honestly unproven
+ * rather than falsely complete. This gate exists so that behavior cannot
+ * silently regress: it drives the REAL connector subprocess against a fake
+ * CardDAV server configured to obey RFC 6578 faithfully, with a prior sync
+ * token in START state, and asserts no fabricated boundary is emitted.
+ *
+ * WHY A CONFORMANCE TEST RATHER THAN A LINT OR A MANIFEST RULE
+ * -----------------------------------------------------------
+ * A manifest-validation rule cannot see which code path executes; it can only
+ * read declarations, and the declaration here (`full_inventory`) is correct —
+ * the address book genuinely is a full inventory. A lint would have to
+ * pattern-match call sites and would be defeated by any indirection.
+ *
+ * The falsifiable fact is behavioral: given a prior token, does the run emit a
+ * coverage boundary it did not measure? Only executing the connector answers
+ * that, and this package already has the machinery to do it cheaply and
+ * credential-free (`_conformance/coverage-conformance.test.ts` drives the same
+ * subprocess against the same fake server). So this extends that pattern
+ * rather than introducing a new mechanism.
+ */
+
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import type { EmittedMessage } from "@pdpp/connector-protocol";
+import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts";
+import { buildVCard, startFakeCardDavServer } from "../apple_contacts/test-carddav-server.ts";
+
+const PACKAGE_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+const BOOK_URL_KEY_SOURCE = "/addressbooks/owner/card/";
+
+interface DetailCoverageLike {
+ readonly considered?: number;
+ readonly covered?: number;
+ readonly stream?: string;
+ readonly type?: string;
+}
+
+/** Every DETAIL_COVERAGE message the run emitted for a given stream. */
+function detailCoverageFor(messages: readonly EmittedMessage[], stream: string): DetailCoverageLike[] {
+ return messages.filter(
+ (message): message is EmittedMessage & DetailCoverageLike =>
+ (message as DetailCoverageLike).type === "DETAIL_COVERAGE" && (message as DetailCoverageLike).stream === stream
+ );
+}
+
+function recordCountFor(messages: readonly EmittedMessage[], stream: string): number {
+ return messages.filter((message) => {
+ const shape = message as { stream?: string; type?: string };
+ return shape.type === "RECORD" && shape.stream === stream;
+ }).length;
+}
+
+/**
+ * Drive the real Apple Contacts entrypoint against a fake CardDAV server that
+ * obeys RFC 6578, optionally handing it a prior sync token so the run takes
+ * the incremental path.
+ */
+async function runAppleContacts(args: { readonly priorSyncToken?: string }): Promise {
+ const username = "owner@example.com";
+ const password = "app-specific-pw";
+ const server = await startFakeCardDavServer({
+ enforceRfc6578IncrementalSemantics: true,
+ password,
+ username,
+ });
+ try {
+ // A real, non-empty address book. The whole point is that these contacts
+ // exist while the change feed is empty.
+ for (const uid of ["contact-one", "contact-two", "contact-three"]) {
+ server.contacts.set(uid, {
+ href: `${BOOK_URL_KEY_SOURCE}${uid}.vcf`,
+ uid,
+ vcard: buildVCard({ email: `${uid}@example.com`, fn: `Fixture ${uid}`, uid }),
+ });
+ }
+
+ // The connector keys per-book cursor state by the book URL with trailing
+ // slashes stripped (`addressBookId`), so the state key must match exactly
+ // or the prior token is silently ignored and the run takes the full path.
+ const bookKeySource = `${server.origin}${BOOK_URL_KEY_SOURCE}`.replace(/\/+$/, "");
+ const state = args.priorSyncToken
+ ? { contacts: { [bookKeySource]: { fingerprints: {}, sync_token: args.priorSyncToken } } }
+ : {};
+
+ const result = await runConnectorProtocolSubprocess({
+ allowFailedDone: true,
+ cwd: PACKAGE_ROOT,
+ entrypoint: join(PACKAGE_ROOT, "connectors/apple_contacts/index.ts"),
+ env: { APPLE_APP_SPECIFIC_PASSWORD: password, APPLE_CARDDAV_ORIGIN: server.origin, APPLE_ID: username },
+ start: {
+ scope: { streams: [{ name: "address_books" }, { name: "contacts" }, { name: "contact_groups" }] },
+ state,
+ type: "START",
+ },
+ });
+ return result.messages;
+ } finally {
+ await server.close();
+ }
+}
+
+test("an initial run (empty sync token) DOES prove the full inventory", async () => {
+ // The control case. RFC 6578 requires an empty token to return every member,
+ // so this run genuinely enumerates the collection and is entitled to claim a
+ // measured boundary. If this stops holding, the gate below would pass
+ // vacuously.
+ const messages = await runAppleContacts({});
+
+ assert.equal(recordCountFor(messages, "contacts"), 3, "the initial run must emit every contact");
+
+ const coverage = detailCoverageFor(messages, "contacts");
+ assert.equal(coverage.length, 1, "the initial run must emit exactly one contacts coverage claim");
+ assert.equal(coverage[0]?.considered, 3, "considered must be the measured inventory size");
+ assert.equal(coverage[0]?.covered, 3, "covered must satisfy the denominator");
+});
+
+test("a quiet INCREMENTAL run must not report an empty change feed as an empty inventory", async () => {
+ // The regression guard for the real defect. A prior sync token puts the run
+ // on the delta path; the fixture's collection is unchanged, so the change
+ // feed is empty while three contacts still exist upstream.
+ const messages = await runAppleContacts({ priorSyncToken: "sync-token-1" });
+
+ // The delta legitimately carries no contact records — nothing changed.
+ assert.equal(recordCountFor(messages, "contacts"), 0, "a quiet delta emits no contact records");
+
+ const coverage = detailCoverageFor(messages, "contacts");
+
+ // THE ASSERTION THAT MATTERS. Emitting `considered: 0, covered: 0` here is
+ // exactly the defect: the coherence oracle reads a measured zero denominator
+ // as a proven-empty inventory, so a populated address book would read
+ // verified-empty and Healthy. Withholding the claim leaves the stream
+ // honestly unproven instead.
+ for (const claim of coverage) {
+ assert.notEqual(
+ claim.considered,
+ 0,
+ "an incremental delta must never emit considered: 0 — that is a change feed being " +
+ "reported as an inventory, which the coherence oracle reads as verified-empty"
+ );
+ }
+ assert.equal(
+ coverage.length,
+ 0,
+ "a run that established no full boundary must emit NO contacts coverage claim at all; " +
+ "silence is the honest verdict, not a fabricated zero"
+ );
+});
+
+test("the incremental run still commits its cursor, so withholding coverage does not stall sync", async () => {
+ // Withholding a coverage CLAIM must not be confused with failing the run.
+ // The delta is real progress: its sync token has to persist or the next run
+ // would re-walk from the same place forever.
+ const messages = await runAppleContacts({ priorSyncToken: "sync-token-1" });
+
+ const stateMessages = messages.filter((message) => {
+ const shape = message as { stream?: string; type?: string };
+ return shape.type === "STATE" && shape.stream === "contacts";
+ });
+ assert.ok(stateMessages.length > 0, "the incremental run must still checkpoint its sync token");
+
+ const done = messages.find((message) => (message as { type?: string }).type === "DONE") as
+ | { status?: string }
+ | undefined;
+ assert.equal(done?.status, "succeeded", "a quiet incremental run is a success, not a failure");
+});
diff --git a/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts b/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts
index 456e5c83f..30b61d503 100644
--- a/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts
+++ b/packages/polyfill-connectors/connectors/_conformance/coverage-conformance-drivers.ts
@@ -50,6 +50,7 @@
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import type { EmittedMessage } from "@pdpp/connector-protocol";
+import { REDDIT_JSON_ORIGIN } from "../../src/auto-login/reddit.ts";
import type { CollectContext } from "../../src/connector-runtime.ts";
import { makeRecordingEmit, type RecordingEmit } from "../../src/test-harness.ts";
@@ -169,6 +170,13 @@ function createMockRedditPage(fetchPath: (path: string) => Promise<{ status: num
const { path } = args as { path: string };
return fetchPath(path);
},
+ // `redditFetch` calls `ensureRedditJsonOrigin` before every listing fetch,
+ // which reads `page.url()` and navigates when the origin is wrong. A mock
+ // without these reports the page as off-origin, so the fetch short-circuits
+ // to `status: 0` (`reddit_http_0`) and never reaches `evaluate` above.
+ // Same shape as reddit's own `createMockPageForFetch` oracle mock.
+ goto: (): Promise => Promise.resolve(null),
+ url: (): string => `${REDDIT_JSON_ORIGIN}/`,
};
}
@@ -1028,7 +1036,10 @@ export const KNOWN_UNEXERCISED_COVERAGE: ReadonlySet = new Set([
"google_calendar.events",
"google_contacts.people",
"google_contacts.contact_groups",
- "google_maps.timeline_points",
+ // google_maps.timeline_points is no longer listed: it now declares
+ // `required: false` (matching its sibling timeline_segments), so it is not
+ // a required stream and this allowlist — which only tracks UNEXERCISED
+ // REQUIRED streams — must not claim it.
"google_maps_data_portability.archive_jobs",
// Google Takeout (REAL_UNLISTED_CONNECTORS): export-file snapshot-import
// receipts, no driver yet.
@@ -1041,10 +1052,11 @@ export const KNOWN_UNEXERCISED_COVERAGE: ReadonlySet = new Set([
"pocket.items",
"heb.orders",
"heb.order_items",
- // iCal / iMessage (REAL_UNLISTED_CONNECTORS): file-based import receipts, no
- // driver yet.
+ // iCal / iMessage / Signal (REAL_UNLISTED_CONNECTORS): file/subprocess-based
+ // import receipts, no driver yet.
"ical.events",
"imessage.messages",
+ "signal.messages",
"notion.pages",
"notion.databases",
"oura.sleep",
diff --git a/packages/polyfill-connectors/connectors/_conformance/manual-import-coverage-strategy.test.ts b/packages/polyfill-connectors/connectors/_conformance/manual-import-coverage-strategy.test.ts
new file mode 100644
index 000000000..7de5f80f2
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/_conformance/manual-import-coverage-strategy.test.ts
@@ -0,0 +1,99 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+// Every manual-upload connector's data streams must declare
+// `coverage_strategy: "snapshot_import_receipt"`.
+//
+// A manual upload is a one-time snapshot of a file the owner exported: the
+// artifact is parsed once, in full, and nothing will run again. That is what
+// `snapshot_import_receipt` names. `checkpoint_window` names the opposite
+// shape -- a rolling cursor over a source that keeps producing -- and a
+// connector whose freshness strategy is `manual_as_of` has no such window by
+// construction.
+//
+// The two strategies happen to carry the SAME proof obligation today
+// (`strategyBoundsWindowRatherThanCounting` in the shared evidence contract
+// treats both as window-bounding), so this mislabel changes no verdict right
+// now. It is pinned anyway because the label is the manifest's honest
+// self-description of what kind of source this is, and because the two
+// strategies are free to diverge later -- at which point a stale
+// `checkpoint_window` on a finished import would start asking the projection
+// for a window that can never close.
+//
+// Scoped to the manual-upload roster by `setup.modality`, so a newly added
+// manual connector is covered without editing this test.
+
+import assert from "node:assert/strict";
+import { readdirSync, readFileSync } from "node:fs";
+import { test } from "node:test";
+
+const MANIFESTS_DIR = new URL("../../manifests/", import.meta.url);
+
+interface ManifestStream {
+ readonly coverage_strategy?: string;
+ readonly freshness_strategy?: string;
+ readonly name: string;
+}
+
+interface Manifest {
+ readonly connector_key?: string;
+ readonly setup?: { readonly modality?: string };
+ readonly streams?: readonly ManifestStream[];
+}
+
+function readManifest(fileName: string): Manifest {
+ return JSON.parse(readFileSync(new URL(fileName, MANIFESTS_DIR), "utf8")) as Manifest;
+}
+
+/** Every connector whose setup modality is a manual file upload. */
+function manualUploadManifests(): { manifest: Manifest; name: string }[] {
+ return readdirSync(new URL(MANIFESTS_DIR))
+ .filter((file) => file.endsWith(".json"))
+ .map((file) => ({ manifest: readManifest(file), name: file }))
+ .filter(({ manifest }) => manifest.setup?.modality === "manual_or_upload");
+}
+
+test("every manual-upload connector is discoverable by setup modality", () => {
+ // `connector_key` is optional on the manifest type, so a missing key would
+ // otherwise sort as a silent `undefined` hole. Surface it as the literal
+ // string instead: the deepEqual below then fails loudly on the real defect
+ // rather than on an unexplained gap in the roster.
+ const found = manualUploadManifests()
+ .map(({ manifest }) => manifest.connector_key ?? "")
+ .sort((a, b) => a.localeCompare(b));
+ // Guards the filter itself: if `setup.modality` were renamed, the roster
+ // would silently empty and every assertion below would vacuously pass.
+ assert.deepEqual(found, ["google-maps", "netflix-export", "whatsapp"]);
+});
+
+test("manual-upload data streams declare snapshot_import_receipt coverage", () => {
+ for (const { manifest, name } of manualUploadManifests()) {
+ for (const stream of manifest.streams ?? []) {
+ // `parent_detail_accounting` is a stricter per-item obligation (it owes
+ // a numerator that actually satisfies its denominator), so a stream that
+ // declares it is making a stronger claim, not evading this one.
+ if (stream.coverage_strategy === "parent_detail_accounting") {
+ continue;
+ }
+ assert.equal(
+ stream.coverage_strategy,
+ "snapshot_import_receipt",
+ `${name}: stream '${stream.name}' is a finished one-time import, so it must declare ` +
+ `snapshot_import_receipt (got '${String(stream.coverage_strategy)}')`
+ );
+ }
+ }
+});
+
+test("no manual-upload stream claims a rolling checkpoint window", () => {
+ for (const { manifest, name } of manualUploadManifests()) {
+ for (const stream of manifest.streams ?? []) {
+ assert.notEqual(
+ stream.coverage_strategy,
+ "checkpoint_window",
+ `${name}: stream '${stream.name}' declares a rolling checkpoint window, but a manual ` +
+ "upload has no window to roll -- nothing will run again after the import"
+ );
+ }
+ }
+});
diff --git a/packages/polyfill-connectors/connectors/amazon/__fixtures__/orders-list-one-card-missing-order-id.html b/packages/polyfill-connectors/connectors/amazon/__fixtures__/orders-list-one-card-missing-order-id.html
new file mode 100644
index 000000000..ab3592761
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/amazon/__fixtures__/orders-list-one-card-missing-order-id.html
@@ -0,0 +1,51 @@
+
+
+
+
+
+
+
+
+ Delivered January 17
+
+
+
+
+
+
+
diff --git a/packages/polyfill-connectors/connectors/amazon/index.ts b/packages/polyfill-connectors/connectors/amazon/index.ts
index 28de4dd35..a0ff63577 100644
--- a/packages/polyfill-connectors/connectors/amazon/index.ts
+++ b/packages/polyfill-connectors/connectors/amazon/index.ts
@@ -36,6 +36,7 @@ import { type FingerprintCursor, openFingerprintCursor, recordFingerprint } from
import {
buildOrderItemRecord,
buildOrderRecord,
+ countOrderCardsWithoutOrderId,
mergeOrderItems,
parseOrderDate,
parseOrderDetailDom,
@@ -134,6 +135,39 @@ interface EmptyListPageClassification {
reason: string;
}
+/**
+ * What this connection already knows about the order history OF THE YEAR
+ * currently being scraped, threaded explicitly into the otherwise-pure
+ * empty-page classifier.
+ *
+ * `hasPriorOrders` is true when a prior run committed a `years[]`
+ * cursor whose `order_count` is greater than zero — the in-connector proof
+ * that Amazon once listed orders for this account IN THIS YEAR.
+ *
+ * PER-YEAR, not per-account, and that is the whole difference from H-E-B.
+ * H-E-B's list is one globally reverse-chronological feed, so any committed
+ * checkpoint contradicts any empty page. Amazon is year-partitioned: it
+ * renders a genuine, year-scoped "Looks like you didn't place an order in
+ * 2015." for a year the owner simply did not shop, on an account holding
+ * thousands of orders in other years. Account-wide evidence would abort every
+ * such year forever and break the incremental year sweep. Only the SAME
+ * year's own prior `order_count` can contradict that year's empty page.
+ */
+export interface PriorOrdersEvidence {
+ hasPriorOrders: boolean;
+}
+
+/** Owner-facing message for the one classification whose whole point is to be
+ * read by a person. Says exactly what was observed and what was NOT
+ * concluded: neither selector drift nor a bot block is established, so
+ * neither is named. Stored records are untouched — this connector never
+ * deletes, tombstones, or overwrites on an empty page; it only declines to
+ * advance the year cursor. */
+export const AMAZON_EMPTY_AFTER_PRIOR_ORDERS_MESSAGE =
+ "Amazon reported no order history for a year in which PDPP previously collected orders. " +
+ "Your stored orders are retained and untouched. This run was stopped instead of recording " +
+ "an empty history, because a page showing no orders cannot prove those orders are gone.";
+
// Navigation timeouts + pacing knobs
const NAV_TIMEOUT_MS = 30_000;
const DEEP_PROBE_WAIT_MS = 15_000;
@@ -446,12 +480,17 @@ async function fetchOrderDetail(page: Page, orderId: string): Promise {
+interface PageExtractionResult {
+ droppedCardCount: number;
+ orders: ListPageOrder[];
+}
+
+async function extractOrdersOnPage(page: Page): Promise {
try {
const html = await readPageContentWithin(page);
- return parseOrdersListDom(html);
+ return { droppedCardCount: countOrderCardsWithoutOrderId(html), orders: parseOrdersListDom(html) };
} catch {
- return [];
+ return { droppedCardCount: 0, orders: [] };
}
}
@@ -1038,19 +1077,80 @@ export async function emitOrderAndItems(
}
}
if (deps.wantsItems) {
- for (const merged of mergeOrderItems(listOrder, detail)) {
- await deps.emitRecord("order_items", buildOrderItemRecord(listOrder.orderId, orderDate, merged));
+ const merged = mergeOrderItems(listOrder, detail);
+ for (const item of merged) {
+ await deps.emitRecord("order_items", buildOrderItemRecord(listOrder.orderId, orderDate, item));
}
+ await emitItemCountReconciliation(deps, listOrder, detail, merged.length);
}
}
+/**
+ * Reconcile the items we emitted against the count the order page asserted, and
+ * report a shortfall rather than letting it pass.
+ *
+ * The denominator is measured at the parse boundary — `detail.items` is what
+ * the order-detail page listed, counted before the merge and independently of
+ * what `mergeOrderItems` decided to emit. Deduplication across the two surfaces
+ * means the merged list should never be SHORTER than the detail page's own
+ * list; if it is, an item the page showed us did not survive into a record.
+ *
+ * Only runs when a detail page was actually fetched. Without it there is no
+ * assertion to reconcile against — `resolveItemCount` reports null for exactly
+ * that case, and inventing a denominator from the list card would fabricate a
+ * shortfall on every 3+ item order (the list card renders no item titles once
+ * Amazon collapses them behind "+N more items").
+ */
+async function emitItemCountReconciliation(
+ deps: EmitDeps,
+ listOrder: ListPageOrder,
+ detail: OrderDetail | null,
+ emittedItemCount: number
+): Promise {
+ const declared = detail?.items?.length;
+ if (declared === undefined || emittedItemCount >= declared) {
+ return;
+ }
+ await deps.emit({
+ type: "SKIP_RESULT",
+ stream: "order_items",
+ reason: "item_count_shortfall",
+ message: `order ${listOrder.orderId} listed ${declared} items on its detail page but only ${emittedItemCount} became records`,
+ diagnostics: {
+ order_id: listOrder.orderId,
+ declared_item_count: declared,
+ emitted_item_count: emittedItemCount,
+ },
+ });
+}
+
/**
* Run list-page extraction through the zod shape-check. Orders that fail
* the shape-check become SKIP_RESULT events; the successful subset is
* returned in source order.
*/
async function extractAndShapeCheckOrders(page: Page, emit: EmitFn): Promise {
- const rawOrders = await extractOrdersOnPage(page);
+ const { droppedCardCount, orders: rawOrders } = await extractOrdersOnPage(page);
+ // Only report when the page also proved at least one real order: `.order-card`/
+ // `.js-order-card` is reused by Amazon's "Buy it again" recommendation carousel on
+ // a genuinely empty year (see the `orders-list-empty-year-with-carousel` fixture),
+ // so a page with ZERO real orders and only carousel cards is not evidence of loss —
+ // `reportEmptyPageDiagnostics` already owns that case. Gating on `rawOrders.length
+ // > 0` keeps this diagnostic scoped to "some cards parsed, but not all of them."
+ if (droppedCardCount > 0 && rawOrders.length > 0) {
+ // A card matched `.order-card`/`.js-order-card` but `parseOrderCard`
+ // could not find a `.yohtmlc-order-id` on it, so it never became a raw
+ // order and never reached the shape-check below — with no fix here that
+ // card is lost with zero trace (no SKIP_RESULT, no coverage-considered
+ // id, no rejection). This is the only signal that loss ever happened.
+ await emit({
+ type: "SKIP_RESULT",
+ stream: "orders",
+ reason: "list_page_order_id_not_found",
+ message: `${droppedCardCount} order card(s) on this page had no parseable .yohtmlc-order-id and were dropped before the shape-check`,
+ diagnostics: { dropped_card_count: droppedCardCount },
+ });
+ }
const orders: ListPageOrder[] = [];
for (const r of rawOrders) {
const parsed = listPageOrderShape.safeParse(r);
@@ -1077,7 +1177,13 @@ async function extractAndShapeCheckOrders(page: Page, emit: EmitFn): Promise {
+async function reportEmptyPageDiagnostics(
+ page: Page,
+ year: number,
+ startIndex: number,
+ emit: EmitFn,
+ priorOrdersEvidence: PriorOrdersEvidence
+): Promise {
let diag: ListPageDiagnostics;
try {
diag = await page.evaluate((noOrdersTextPattern): ListPageDiagnostics => {
@@ -1108,7 +1214,7 @@ async function reportEmptyPageDiagnostics(page: Page, year: number, startIndex:
});
throw new Error("amazon_empty_list_page_renderer_diagnostics_failed", { cause: error });
}
- const classification = classifyEmptyListPageDiagnostics(diag, startIndex);
+ const classification = classifyEmptyListPageDiagnostics(diag, startIndex, priorOrdersEvidence);
if (classification.action === "terminal") {
return;
}
@@ -1134,8 +1240,13 @@ async function reportEmptyPageDiagnostics(page: Page, year: number, startIndex:
type: "SKIP_RESULT",
stream: "orders",
reason: classification.reason,
- message: `Year ${year} startIndex=${startIndex}: empty Amazon list page is not a proven terminal page; refusing to advance the cursor.`,
- diagnostics: diag ? redactAmazonListPageDiagnostics(diag) : { missing_diagnostics: true },
+ message:
+ classification.reason === "amazon_empty_history_after_prior_orders"
+ ? AMAZON_EMPTY_AFTER_PRIOR_ORDERS_MESSAGE
+ : `Year ${year} startIndex=${startIndex}: empty Amazon list page is not a proven terminal page; refusing to advance the cursor.`,
+ diagnostics: diag
+ ? { ...redactAmazonListPageDiagnostics(diag), has_prior_orders: priorOrdersEvidence.hasPriorOrders, year }
+ : { missing_diagnostics: true },
});
}
throw new Error(`amazon_empty_list_page_${classification.reason}`);
@@ -1149,7 +1260,8 @@ export function redactAmazonListPageDiagnostics(diag: ListPageDiagnostics): List
export function classifyEmptyListPageDiagnostics(
diag: ListPageDiagnostics | null,
- startIndex: number
+ startIndex: number,
+ priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false }
): EmptyListPageClassification {
if (!diag) {
return { action: "abort", reason: "renderer_diagnostics_failed" };
@@ -1161,6 +1273,35 @@ export function classifyEmptyListPageDiagnostics(
if ((diag.any_card > 0 || diag.any_order_header > 0) && diag.order_cards === 0) {
return { action: "abort", reason: "selector_drift" };
}
+ // A year that has already yielded orders can never prove itself empty.
+ // `hasPriorOrders` is this connection's own prior `years[].order_count`
+ // for the SAME year being scraped — durable evidence that Amazon previously
+ // listed orders there — threaded in explicitly by `scrapeListPage` rather
+ // than read from ambient state, so this branch stays pure and unit-testable.
+ //
+ // Without this check, a year holding hundreds of stored orders could finish
+ // as {action:"terminal", reason:"no_orders_text"}, letting the run advance
+ // the year cursor and report covered:0/considered:0 — replacing a measured
+ // coverage claim with a fabricated proven-zero. The two causes are
+ // indistinguishable from the page alone: Amazon may have purged the history
+ // upstream (making our stored copy the only copy), or the page may render
+ // empty for a degraded session. So the run fails loudly and lets a human
+ // decide, rather than guessing.
+ //
+ // Scoped to `startIndex === 0` because only the FIRST page of a year makes
+ // the claim "this year is empty". A later page coming back empty is ordinary
+ // pagination exhaustion on a year that plainly did yield orders — the rows
+ // preceding it were just collected on this very run — and must keep falling
+ // through to `pagination_exhausted` below.
+ //
+ // Ordering: BELOW the auth/challenge and selector-drift checks (an
+ // established block or a real markup change is the more specific and more
+ // actionable diagnosis, and must not be relabelled), and ABOVE the
+ // `no_orders_text` branch, so Amazon's own empty-state copy cannot
+ // short-circuit past it.
+ if (startIndex === 0 && diag.no_orders_text === "true" && priorOrdersEvidence.hasPriorOrders) {
+ return { action: "abort", reason: "amazon_empty_history_after_prior_orders" };
+ }
if (diag.no_orders_text === "true") {
return { action: "terminal", reason: "no_orders_text" };
}
@@ -1180,7 +1321,8 @@ export async function scrapeListPage(
capture: CaptureDep,
year: number,
startIndex: number,
- emit: EmitFn
+ emit: EmitFn,
+ priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false }
): Promise {
const url = `https://www.amazon.com/your-orders/orders?timeFilter=year-${year}&startIndex=${startIndex}`;
try {
@@ -1212,7 +1354,7 @@ export async function scrapeListPage(
}
const orders = await extractAndShapeCheckOrders(page, emit);
if (orders.length === 0) {
- await reportEmptyPageDiagnostics(page, year, startIndex, emit);
+ await reportEmptyPageDiagnostics(page, year, startIndex, emit, priorOrdersEvidence);
}
return orders;
}
@@ -1454,19 +1596,43 @@ export function shouldEmitTrailingOrdersState(
return !ordersStateEmitted && hydratedOrders.size > 0;
}
+/**
+ * Derive the prior-orders evidence for ONE year from this connection's stored
+ * `years` cursor. Exported and pure because `collect()` lives inside the
+ * `isMainModule` block and cannot be driven from a test — without this seam
+ * the year-state-to-evidence link would be the one untested link in the
+ * chain, and a mutation that hardcodes `false` here (silently disarming the
+ * guard for every connection) would go unnoticed.
+ *
+ * The evidence is `order_count > 0`, not the mere existence of a year cursor.
+ * A year that was scraped and legitimately held zero orders commits a cursor
+ * with `order_count: 0`; that year must stay free to report empty again on
+ * every later run. Only a year that actually yielded orders can contradict a
+ * later "no orders" page.
+ */
+export function priorOrdersEvidenceForYear(yearsState: YearsCursor, year: number): PriorOrdersEvidence {
+ return { hasPriorOrders: (yearsState[String(year)]?.order_count ?? 0) > 0 };
+}
+
/**
* Scrape every list page for one year and emit records. Returns both the total
* order count seen for the year (used for freeze-once-stable policy) and the
* count of rows we could not emit because their order date was unparseable.
*/
-async function runYear(page: Page, deps: EmitDeps, flags: RunFlags, year: number): Promise {
+async function runYear(
+ page: Page,
+ deps: EmitDeps,
+ flags: RunFlags,
+ year: number,
+ priorOrdersEvidence: PriorOrdersEvidence = { hasPriorOrders: false }
+): Promise {
let startIndex = 0;
let pageCount = 0;
let yearOrderCount = 0;
let unparseableDateCount = 0;
while (pageCount < PAGE_LIMIT) {
await deps.progress(`Amazon year ${year}: scanning page ${pageCount + 1}`, { stream: "orders" });
- const orders = await scrapeListPage(page, deps.capture, year, startIndex, deps.emit);
+ const orders = await scrapeListPage(page, deps.capture, year, startIndex, deps.emit, priorOrdersEvidence);
if (orders.length === 0) {
await deps.progress(`Amazon year ${year}: no more orders after ${yearOrderCount} seen`, { stream: "orders" });
break;
@@ -1719,7 +1885,18 @@ if (isMainModule(import.meta.url)) {
continue;
}
- const { orderCount: yearOrderCount, unparseableDateCount } = await runYear(page, deps, flags, year);
+ // A prior `years[].order_count > 0` is this connection's own
+ // record that Amazon has listed orders for this account in THIS year
+ // before. It is what makes a later "no orders in " page a
+ // contradiction to escalate rather than a result to trust. Read here,
+ // next to the year cursor it derives from, and passed down explicitly.
+ const { orderCount: yearOrderCount, unparseableDateCount } = await runYear(
+ page,
+ deps,
+ flags,
+ year,
+ priorOrdersEvidenceForYear(yearsState, year)
+ );
await applyYearCompletionState({
newYearsState,
diff --git a/packages/polyfill-connectors/connectors/amazon/integration.test.ts b/packages/polyfill-connectors/connectors/amazon/integration.test.ts
index 7c0bcf6c1..a10edff74 100644
--- a/packages/polyfill-connectors/connectors/amazon/integration.test.ts
+++ b/packages/polyfill-connectors/connectors/amazon/integration.test.ts
@@ -53,6 +53,7 @@ import {
type OrderItemsCoverage,
type OrdersCoverage,
planIncrementalYears,
+ priorOrdersEvidenceForYear,
processListOrder,
type RunFlags,
readPageContentWithin,
@@ -424,13 +425,33 @@ test("collect path does not advance a year cursor after unparseable order-date d
);
});
-test("amazon manifest: successful manual runs have a bounded freshness window", () => {
+test("amazon manifest: successful runs have a bounded freshness window", () => {
+ // The bounded window is what this test is for: without
+ // `maximum_staleness_seconds`, freshness is `unknown` and a successful run
+ // cannot project `current`.
+ //
+ // The mode assertion moved from a hard-coded "manual" to the connector's
+ // own declared facts. Amazon declares `background_safe: true` — the
+ // browser session persists after the owner's first login — and mode is now
+ // DERIVED from that (see reference-implementation/runtime/
+ // refresh-mode-derivation.ts). Pinning "manual" here contradicted the
+ // manifest's own background-safety claim.
const manifest = JSON.parse(readFileSync(AMAZON_MANIFEST_PATH, "utf8")) as {
- capabilities?: { refresh_policy?: { maximum_staleness_seconds?: number; recommended_mode?: string } };
+ capabilities?: {
+ refresh_policy?: {
+ background_safe?: boolean;
+ interaction_posture?: string;
+ maximum_staleness_seconds?: number;
+ recommended_mode?: string;
+ };
+ };
};
const policy = manifest.capabilities?.refresh_policy;
- assert.equal(policy?.recommended_mode, "manual");
assert.equal(policy?.maximum_staleness_seconds, 86_400);
+ // Amazon's first login is owner-present, and the session then persists.
+ assert.equal(policy?.interaction_posture, "otp_likely");
+ assert.equal(policy?.background_safe, true);
+ assert.equal(policy?.recommended_mode, "automatic");
});
// ─── Empty list-page classification ──────────────────────────────────────
@@ -548,6 +569,40 @@ test("scrapeListPage: a failed list navigation cannot reuse the prior page as a
assert.doesNotMatch(JSON.stringify(messages), /orders-list-minimal|B0/);
});
+test("scrapeListPage: a card with no parseable order id is dropped AND reported, alongside a card that survives", async () => {
+ // The "prove the drop happens today, is invisible today" half of the pair.
+ // One card has a normal `.yohtmlc-order-id`; the other (modeling a
+ // never-shipped/cancelled order rendering under a variant Amazon uses for
+ // that order type) has none. Before this fix, the second card vanished
+ // from `orders` with zero SKIP_RESULT anywhere — see
+ // `countOrderCardsWithoutOrderId` in parsers.ts for the mechanism.
+ const html = readFileSync(
+ new URL("./__fixtures__/orders-list-one-card-missing-order-id.html", import.meta.url),
+ "utf8"
+ );
+ const messages: EmittedMessage[] = [];
+ const page = Object.assign({} as Page, {
+ content: (): Promise => Promise.resolve(html),
+ goto: (): Promise => Promise.resolve(null),
+ locator: (): { first: () => { waitFor: () => Promise } } => ({
+ first: () => ({ waitFor: (): Promise => Promise.resolve(null) }),
+ }),
+ });
+
+ const orders = await scrapeListPage(page, null, 2024, 0, (message) => {
+ messages.push(message);
+ return Promise.resolve();
+ });
+
+ assert.equal(orders.length, 1, "only the card with a parseable order id survives");
+ assert.equal(orders[0]?.orderId, "111-2222222-3333333");
+
+ const skip = messages.find((message) => message.type === "SKIP_RESULT");
+ assert.ok(skip, "the drop must be reported, not silent");
+ assert.equal(skip?.reason, "list_page_order_id_not_found");
+ assert.deepEqual(skip?.diagnostics, { dropped_card_count: 1 });
+});
+
test("scrapeListPage: a page-2 renderer diagnostic failure aborts without STATE or coverage", async () => {
const messages: EmittedMessage[] = [];
const page = Object.assign({} as Page, {
@@ -579,6 +634,241 @@ test("scrapeListPage: a page-2 renderer diagnostic failure aborts without STATE
assert.doesNotMatch(JSON.stringify(messages), /private renderer|owner label|amazon\.com/);
});
+// ─── Proven-empty regression guard (prior-orders evidence) ────────────────
+//
+// A YEAR that has already yielded orders must never be able to complete a run
+// as "proven empty". Amazon's own year-scoped empty copy is trustworthy for a
+// year the owner did not shop; for a year we have already measured, the same
+// page is a contradiction, not a result.
+//
+// The evidence is PER-YEAR, unlike H-E-B's account-wide checkpoint, because
+// Amazon is year-partitioned: an account with thousands of orders legitimately
+// renders "Looks like you didn't place an order in 2015." for an unshopped
+// year, and that year must keep reporting empty forever.
+
+const RESOLVED_EMPTY_YEAR_DIAG = (): ListPageDiagnostics => makeEmptyPageDiagnostics({ no_orders_text: "true" });
+
+/**
+ * A Page stub serving Amazon's real captured empty-year list page. `content()`
+ * feeds the DOM parser (which finds no order cards, the trigger for the
+ * empty-page branch) and `evaluate()` returns diagnostics derived from the
+ * SAME fixture text, so `no_orders_text` is decided by Amazon's own copy
+ * rather than asserted by the test.
+ */
+function makeEmptyYearPageStub(): Page {
+ const html = readFileSync(AMAZON_EMPTY_YEAR_FIXTURE, "utf8");
+ const visibleText = html.replace(/<[^>]+>/g, " ");
+ return Object.assign({} as Page, {
+ content: (): Promise => Promise.resolve(html),
+ evaluate: (): Promise =>
+ Promise.resolve(
+ makeEmptyPageDiagnostics({
+ no_orders_text: new RegExp(AMAZON_NO_ORDERS_TEXT_PATTERN, "i").test(visibleText).toString(),
+ })
+ ),
+ goto: (): Promise => Promise.resolve(null),
+ locator: (): { first: () => { waitFor: () => Promise } } => ({
+ first: () => ({ waitFor: (): Promise => Promise.resolve(null) }),
+ }),
+ screenshot: (): Promise => Promise.resolve(null),
+ });
+}
+
+test("classifyEmptyListPageDiagnostics: source-reported empty for a year with NO prior orders stays proven-empty", () => {
+ // Preserves existing behavior. A year the owner did not shop is the case
+ // where zero coverage is an honest measurement.
+ assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0, { hasPriorOrders: false }), {
+ action: "terminal",
+ reason: "no_orders_text",
+ });
+});
+
+test("classifyEmptyListPageDiagnostics: the prior-orders argument defaults to absent, so callers cannot silently opt in", () => {
+ assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0), {
+ action: "terminal",
+ reason: "no_orders_text",
+ });
+});
+
+test("classifyEmptyListPageDiagnostics: source-reported empty for a year WITH prior orders aborts instead of proving zero", () => {
+ // The defect this guard closes: without it, this exact input returned
+ // {action:"terminal", reason:"no_orders_text"}, letting a year holding
+ // hundreds of stored orders advance its cursor on a fabricated proven-zero.
+ assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0, { hasPriorOrders: true }), {
+ action: "abort",
+ reason: "amazon_empty_history_after_prior_orders",
+ });
+});
+
+test("classifyEmptyListPageDiagnostics: a legitimately empty year on an account with orders in OTHER years stays terminal", () => {
+ // THE year-partition requirement, driven through Amazon's real 2015
+ // empty-year capture. `priorOrdersEvidenceForYear` scopes evidence to the
+ // year being scraped, so 2015 carries `hasPriorOrders: false` even though
+ // 2024 and 2026 hold orders. Account-wide evidence would abort this year on
+ // every run forever and break the incremental year sweep.
+ const yearsState = {
+ "2024": { frozen: true, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 312 },
+ "2026": { frozen: false, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 44 },
+ };
+ const html = readFileSync(AMAZON_EMPTY_YEAR_FIXTURE, "utf8");
+ const visibleText = html.replace(/<[^>]+>/g, " ");
+ const noOrdersRe = new RegExp(AMAZON_NO_ORDERS_TEXT_PATTERN, "i");
+ assert.match(visibleText, noOrdersRe, "the fixture must still carry Amazon's year-scoped empty copy");
+
+ assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2015), { hasPriorOrders: false });
+ assert.deepEqual(
+ classifyEmptyListPageDiagnostics(
+ makeEmptyPageDiagnostics({ any_card: 1, no_orders_text: "true", order_cards: 1 }),
+ 0,
+ priorOrdersEvidenceForYear(yearsState, 2015)
+ ),
+ { action: "terminal", reason: "no_orders_text" },
+ "an unshopped year must stay proven-empty on an account with orders elsewhere"
+ );
+
+ // Same account, same run, a year that DID yield orders: opposite verdict.
+ assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2024), { hasPriorOrders: true });
+ assert.deepEqual(
+ classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 0, priorOrdersEvidenceForYear(yearsState, 2024)),
+ { action: "abort", reason: "amazon_empty_history_after_prior_orders" }
+ );
+});
+
+test("classifyEmptyListPageDiagnostics: an auth/challenge page keeps its own reason even when the year has prior orders", () => {
+ // Ordering guard, upper half. The auth check stays ABOVE the new branch:
+ // when a challenge is actually established, that is the more specific and
+ // more actionable diagnosis, and it must not be relabelled.
+ const authOverrides: Partial[] = [{ captcha: "true" }, { sign_in_form: true }];
+ for (const override of authOverrides) {
+ assert.deepEqual(
+ classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics({ no_orders_text: "true", ...override }), 0, {
+ hasPriorOrders: true,
+ }),
+ { action: "abort", reason: "source_auth_or_challenge" }
+ );
+ }
+});
+
+test("classifyEmptyListPageDiagnostics: selector drift keeps its own reason even when the year has prior orders", () => {
+ // Ordering guard, middle. Real markup change stays diagnosable as drift.
+ assert.deepEqual(
+ classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics({ any_order_header: 2, no_orders_text: "true" }), 0, {
+ hasPriorOrders: true,
+ }),
+ { action: "abort", reason: "selector_drift" }
+ );
+});
+
+test("classifyEmptyListPageDiagnostics: prior orders do not relabel a page that never claimed to be empty", () => {
+ // Ordering guard, lower half. The new branch is gated on `no_orders_text`,
+ // so the existing terminal/abort verdicts for pages that make no empty claim
+ // are untouched — the guard adds a failure mode, it does not swallow others.
+ assert.deepEqual(classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics(), 10, { hasPriorOrders: true }), {
+ action: "terminal",
+ reason: "pagination_exhausted",
+ });
+ assert.deepEqual(classifyEmptyListPageDiagnostics(makeEmptyPageDiagnostics(), 0, { hasPriorOrders: true }), {
+ action: "abort",
+ reason: "empty_first_page_without_terminal_signal",
+ });
+});
+
+test("classifyEmptyListPageDiagnostics: a later empty page in a year that just yielded orders is ordinary exhaustion", () => {
+ // The guard is scoped to startIndex === 0 because only the FIRST page of a
+ // year claims the year is empty. Amazon can serve its empty-state copy on a
+ // trailing page; the rows before it were collected on THIS run, so this is
+ // pagination exhaustion, not a contradiction. Without the startIndex scope,
+ // every incremental run over a year with prior orders would abort on its
+ // last page — the guard would break the connector for exactly the accounts
+ // it exists to protect.
+ assert.deepEqual(classifyEmptyListPageDiagnostics(RESOLVED_EMPTY_YEAR_DIAG(), 10, { hasPriorOrders: true }), {
+ action: "terminal",
+ reason: "no_orders_text",
+ });
+});
+
+test("priorOrdersEvidenceForYear: a prior year order_count > 0 is what arms the guard", () => {
+ // Pins the year-state-to-evidence link that collect() depends on. Without
+ // this test, hardcoding `hasPriorOrders: false` in collect() would disarm
+ // the guard for every connection while every other test still passed.
+ const yearsState = {
+ "2019": { frozen: true, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 0 },
+ "2024": { frozen: true, last_scraped: "2026-08-01T00:00:00.000Z", order_count: 312 },
+ };
+ assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2024), { hasPriorOrders: true });
+ // A year scraped and found genuinely empty commits `order_count: 0`. It must
+ // stay free to report empty again — this is the year-partition case that
+ // makes count, not cursor existence, the right evidence.
+ assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2019), { hasPriorOrders: false });
+ // A never-scraped year has no cursor at all.
+ assert.deepEqual(priorOrdersEvidenceForYear(yearsState, 2015), { hasPriorOrders: false });
+});
+
+test("scrapeListPage: Amazon's empty-year page aborts when this year already yielded orders", async () => {
+ // Integration half, through the same live empty-year capture the
+ // proven-empty test uses. Same page, same markup, opposite verdict — the
+ // only difference is that this year has a prior non-zero order count.
+ const messages: EmittedMessage[] = [];
+ const page = makeEmptyYearPageStub();
+
+ await assert.rejects(
+ scrapeListPage(
+ page,
+ null,
+ 2024,
+ 0,
+ (message) => {
+ messages.push(message);
+ return Promise.resolve();
+ },
+ { hasPriorOrders: true }
+ ),
+ /amazon_empty_list_page_amazon_empty_history_after_prior_orders/,
+ "a year with prior orders cannot be proven empty by a page render"
+ );
+
+ // The failure must be legible to the owner, and must not blame anything the
+ // page does not establish.
+ const skip = messages.find(
+ (message) => message.type === "SKIP_RESULT" && message.reason === "amazon_empty_history_after_prior_orders"
+ ) as { diagnostics: Record; message: string } | undefined;
+ assert.ok(skip, "the abort must surface a SKIP_RESULT the owner can read");
+ assert.match(skip.message, /previously collected orders/);
+ assert.match(skip.message, /retained and untouched/);
+ assert.doesNotMatch(skip.message, /selector|drift/i, "selector drift is not established and must not be blamed");
+ assert.doesNotMatch(skip.message, /block|bot|captcha/i, "a bot block is not established and must not be blamed");
+ assert.equal(skip.diagnostics.has_prior_orders, true);
+ assert.equal(skip.diagnostics.year, 2024);
+
+ // Nothing may be recorded, no cursor advanced, no coverage claimed. The
+ // connector protocol has no delete or tombstone message, so proving it
+ // emitted no RECORD and no STATE proves the stored copy and the prior year
+ // cursor are both untouched.
+ assert.deepEqual(
+ [...new Set(messages.map((message) => message.type))].sort(),
+ ["SKIP_RESULT"],
+ "the abort's only durable output is the owner-facing SKIP_RESULT"
+ );
+});
+
+test("scrapeListPage: the same empty-year page still succeeds as proven-empty for a year with no prior orders", async () => {
+ // The preservation half of the pair above: identical page, no prior orders,
+ // no throw, no diagnostic. A first-ever run on a genuinely empty year is
+ // unaffected by the guard.
+ const messages: EmittedMessage[] = [];
+ const orders = await scrapeListPage(makeEmptyYearPageStub(), null, 2015, 0, (message) => {
+ messages.push(message);
+ return Promise.resolve();
+ });
+
+ assert.deepEqual(orders, [], "an empty year yields no orders and does not throw");
+ assert.equal(
+ messages.some((message) => message.type === "SKIP_RESULT"),
+ false,
+ "a proven-empty year emits no skip diagnostic"
+ );
+});
+
// ─── planIncrementalYears ─────────────────────────────────────────────────
test("planIncrementalYears: no prior state → all discovered years planned (initial backfill)", () => {
@@ -2038,3 +2328,109 @@ test("a run with zero considered orders still emits a zero-required DETAIL_COVER
assert.deepEqual(cov.hydrated_keys, []);
assert.equal(findDetailGaps(protocolMessages).length, 0, "zero considered orders produce no gaps");
});
+
+// ─── item_count reconciliation ──────────────────────────────────────────
+
+/**
+ * `item_count` is not a provider assertion — Amazon never states a count in any
+ * markup this connector reads. The number is a count of item elements parsed
+ * from two surfaces, so the only reconcilable claim is the detail page's own
+ * item list: every item that page showed must survive into a record. A merged
+ * list SHORTER than the detail list means an item was seen and then lost.
+ */
+function findItemCountShortfalls(protocolMessages: readonly unknown[]): Record[] {
+ return protocolMessages.filter(
+ (m) =>
+ (m as { type?: string; reason?: string }).type === "SKIP_RESULT" &&
+ (m as { reason?: string }).reason === "item_count_shortfall"
+ ) as Record[];
+}
+
+test("emitOrderAndItems: every detail-page item becoming a record reports no shortfall", async () => {
+ const { deps, protocolMessages, emitted } = makeRecordingDeps();
+ const detail = makeDetail({
+ items: [
+ makeDetailItem({ asin: "B000000001", name: "Widget A" }),
+ makeDetailItem({ asin: "B000000002", name: "Widget B" }),
+ ],
+ });
+ await emitOrderAndItems(deps, makeListOrder({ items: [] }), detail, "2026-01-05");
+
+ assert.equal(emitted.filter((r) => r.stream === "order_items").length, 2);
+ assert.equal(findItemCountShortfalls(protocolMessages).length, 0, "a complete order must not report a gap");
+});
+
+test("emitOrderAndItems: a detail item that never becomes a record is reported as a shortfall", async () => {
+ // Two detail items collapse to one record because they carry the same
+ // identity, so one of the items the page showed us is not in the database.
+ // Before this check that loss was silent.
+ const { deps, protocolMessages, emitted } = makeRecordingDeps();
+ const detail = makeDetail({
+ items: [
+ makeDetailItem({ asin: "B000000001", name: "Widget A" }),
+ makeDetailItem({ asin: "B000000001", name: "Widget A" }),
+ ],
+ });
+ await emitOrderAndItems(deps, makeListOrder({ items: [] }), detail, "2026-01-05");
+
+ const emittedItems = emitted.filter((r) => r.stream === "order_items").length;
+ const shortfalls = findItemCountShortfalls(protocolMessages);
+ assert.equal(emittedItems, 1, "the two detail rows collapsed to one record");
+ assert.equal(shortfalls.length, 1, "the lost item must be surfaced");
+ assert.deepEqual(shortfalls[0]?.diagnostics, {
+ order_id: makeListOrder().orderId,
+ declared_item_count: 2,
+ emitted_item_count: 1,
+ });
+});
+
+test("emitOrderAndItems: no detail page means no denominator and no fabricated shortfall", async () => {
+ // A 3+ item order whose list card collapsed its titles reaches here with an
+ // empty list and no detail. Counting the list card as the denominator would
+ // invent a gap on every such order.
+ const { deps, protocolMessages } = makeRecordingDeps();
+ await emitOrderAndItems(deps, makeListOrder({ items: [] }), null, "2026-01-05");
+
+ assert.equal(
+ findItemCountShortfalls(protocolMessages).length,
+ 0,
+ "an unfetched detail page is an unknown, not a proven shortfall"
+ );
+});
+
+test("emitOrderAndItems: the denominator is the detail page, never the list card", async () => {
+ // Pins WHICH surface supplies the claim. The list card here shows two items
+ // that dedupe to one record — a merge outcome, not a loss, because the list
+ // card is not an authority on how many items an order has. Only the detail
+ // page's own item list is a claim worth holding the connector to; falling
+ // back to the list card manufactures a gap out of ordinary deduplication.
+ const { deps, protocolMessages, emitted } = makeRecordingDeps();
+ const listOrder = makeListOrder({
+ items: [
+ { asin: "B000000001", name: "Widget A", url: null },
+ { asin: "B000000001", name: "Widget A", url: null },
+ ],
+ });
+ await emitOrderAndItems(deps, listOrder, null, "2026-01-05");
+
+ assert.equal(emitted.filter((r) => r.stream === "order_items").length, 1, "the duplicate list rows collapse");
+ assert.equal(
+ findItemCountShortfalls(protocolMessages).length,
+ 0,
+ "the list card is not a provider claim, so its count must not become a denominator"
+ );
+});
+
+test("emitOrderAndItems: items out of scope suppress the reconciliation entirely", async () => {
+ const { deps, protocolMessages } = makeRecordingDeps({ wantsItems: false });
+ const detail = makeDetail({
+ items: [makeDetailItem({ asin: "B000000001" }), makeDetailItem({ asin: "B000000001" })],
+ });
+ await emitOrderAndItems(deps, makeListOrder({ items: [] }), detail, "2026-01-05");
+
+ assert.equal(
+ findItemCountShortfalls(protocolMessages).length,
+ 0,
+ "a stream nobody asked for cannot report a gap against records it never tried to emit"
+ );
+});
diff --git a/packages/polyfill-connectors/connectors/amazon/item-count-honesty.test.ts b/packages/polyfill-connectors/connectors/amazon/item-count-honesty.test.ts
new file mode 100644
index 000000000..0aa59f788
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/amazon/item-count-honesty.test.ts
@@ -0,0 +1,153 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * `item_count` had no independent source. It was
+ * `Math.max(list.items.length, detail.items.length)` against a non-nullable
+ * `min(0)` schema, so an order whose detail page was never fetched asserted a
+ * confident `item_count: 0` — indistinguishable from a genuinely empty order.
+ *
+ * That mattered because the list card only renders item titles for SMALL
+ * orders; Amazon collapses 3+ item orders behind a "+N more items" affordance
+ * that carries no per-item markup. For those orders the list contributes 0 and
+ * the count rests entirely on the detail page, which the connector defers under
+ * its per-run attempt budget, its temporary-failure cap, or a latched session
+ * repair.
+ *
+ * The live signature is unambiguous: across 1,183 collected orders,
+ * `item_count === 0` never occurs on an order holding 1 or 2 item records, and
+ * occurs on 53 orders holding 3-7 — every one of which also has a null
+ * `shipping_address_summary`, proving the detail page (not the parse) was
+ * missing. Those records claimed an empty order while the database held their
+ * items.
+ *
+ * These tests pin the distinction the old shape could not express: a count of
+ * zero must mean "we looked and there were none", never "we did not look".
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { buildOrderRecord } from "./parsers.ts";
+import { validateRecord } from "./schemas.ts";
+import type { DetailItem, ListPageOrder, OrderDetail } from "./types.ts";
+
+function makeListOrder(overrides: Partial = {}): ListPageOrder {
+ return {
+ orderId: "111-2222222-3333333",
+ orderDateRaw: "January 15, 2024",
+ orderTotal: "$10.00",
+ deliveryStatus: "Delivered Jan 17",
+ items: [],
+ ...overrides,
+ };
+}
+
+function makeDetailItem(overrides: Partial = {}): DetailItem {
+ return {
+ asin: null,
+ name: "",
+ url: null,
+ unit_price: null,
+ quantity: 1,
+ seller: null,
+ item_image_url: null,
+ refund_status: null,
+ ...overrides,
+ };
+}
+
+function makeDetail(overrides: Partial = {}): OrderDetail {
+ return {
+ status_detail: null,
+ recipient_name: null,
+ shipping_address_summary: null,
+ payment_method_summary: null,
+ grand_total: null,
+ gift_order: false,
+ digital_order: false,
+ items: [],
+ ...overrides,
+ };
+}
+
+function build(listOrder: ListPageOrder, detail: OrderDetail | null) {
+ return buildOrderRecord(listOrder, detail, "2024-01-15", "2024-01-20T00:00:00Z");
+}
+
+test("amazon item_count: an unfetched detail page reports unknown, not an empty order", () => {
+ // The exact live shape: a 3+ item order whose list card collapsed its titles
+ // and whose detail fetch was deferred. Before this, the record asserted 0.
+ const rec = build(makeListOrder({ items: [] }), null);
+
+ assert.equal(rec.item_count, null, "no surface saw an item and none was fetched — the count is unknown");
+ assert.notEqual(rec.item_count, 0, "0 would claim a proven-empty order we never actually looked at");
+});
+
+test("amazon item_count: a fetched detail page with no items is a proven-empty order", () => {
+ // We DID look at the surface that carries items, and it carried none. That is
+ // a fact, and it must stay distinguishable from the case above.
+ const rec = build(makeListOrder({ items: [] }), makeDetail({ items: [] }));
+
+ assert.equal(rec.item_count, 0, "a successful detail fetch that found nothing proves zero");
+});
+
+test("amazon item_count: the detail page count wins when the list card collapsed its titles", () => {
+ // The 3+ item case with the detail page present: list contributes 0, detail
+ // carries the real count.
+ const rec = build(
+ makeListOrder({ items: [] }),
+ makeDetail({
+ items: [
+ makeDetailItem({ asin: "B01", name: "A" }),
+ makeDetailItem({ asin: "B02", name: "B" }),
+ makeDetailItem({ asin: "B03", name: "C" }),
+ ],
+ })
+ );
+
+ assert.equal(rec.item_count, 3);
+});
+
+test("amazon item_count: the list card alone still counts when no detail was fetched", () => {
+ // A small order whose titles the list card did render. We saw real items, so
+ // the count is known even without a detail page — this must not regress to
+ // null just because `detail` is absent.
+ const rec = build(
+ makeListOrder({
+ items: [
+ { asin: "B01", name: "A", url: null },
+ { asin: "B02", name: "B", url: null },
+ ],
+ }),
+ null
+ );
+
+ assert.equal(rec.item_count, 2, "observed items are observed regardless of which surface showed them");
+});
+
+test("amazon item_count: the larger of the two surfaces is kept", () => {
+ const rec = build(
+ makeListOrder({ items: [{ asin: "B01", name: "A", url: null }] }),
+ makeDetail({ items: [makeDetailItem({ asin: "B01" }), makeDetailItem({ asin: "B02" })] })
+ );
+
+ assert.equal(rec.item_count, 2, "the detail page saw more than the list card");
+});
+
+test("amazon item_count: a null count is accepted by the orders schema", () => {
+ // The schema was non-nullable, which is what forced the unknown to be encoded
+ // as 0 in the first place. If this regresses, the honest value cannot be
+ // emitted at all and the connector silently falls back to the old lie.
+ const rec = build(makeListOrder({ items: [] }), null);
+ const result = validateRecord("orders", rec as unknown as Record);
+
+ assert.equal(result.ok, true, `a null item_count must validate: ${JSON.stringify(result.ok ? [] : result.issues)}`);
+});
+
+test("amazon item_count: a real count still validates", () => {
+ const rec = build(makeListOrder({ items: [{ asin: "B01", name: "A", url: null }] }), makeDetail());
+ const result = validateRecord("orders", rec as unknown as Record);
+
+ assert.equal(result.ok, true);
+ assert.equal(rec.item_count, 1);
+});
diff --git a/packages/polyfill-connectors/connectors/amazon/parsers.test.ts b/packages/polyfill-connectors/connectors/amazon/parsers.test.ts
index e874fc61f..ffe8d9468 100644
--- a/packages/polyfill-connectors/connectors/amazon/parsers.test.ts
+++ b/packages/polyfill-connectors/connectors/amazon/parsers.test.ts
@@ -9,6 +9,7 @@ import { fileURLToPath } from "node:url";
import {
buildOrderItemRecord,
buildOrderRecord,
+ countOrderCardsWithoutOrderId,
itemId,
mergeDetailByKey,
mergeOrderItems,
@@ -152,6 +153,62 @@ test("parseOrdersListDom: empty page returns []", () => {
assert.deepEqual(parseOrdersListDom(""), []);
});
+// A card that matches `.order-card`/`.js-order-card` but has no
+// `.yohtmlc-order-id` (e.g. a never-shipped/cancelled Subscribe & Save order
+// rendering under a variant Amazon uses for that order type) is dropped by
+// `parseOrdersListDom` with no signal anywhere else in the connector: it
+// never reaches the shape-check, so it produces no SKIP_RESULT, no
+// coverage-considered id, and no rejection. `countOrderCardsWithoutOrderId`
+// is the one place that gap becomes visible.
+const NORMAL_CARD_HTML = `
+
+
+
`;
+const CARD_WITHOUT_ORDER_ID_HTML = `
+ `;
+
+test("countOrderCardsWithoutOrderId: 0 when every card has a .yohtmlc-order-id", () => {
+ const html = `${NORMAL_CARD_HTML}
`;
+ assert.equal(countOrderCardsWithoutOrderId(html), 0);
+ assert.equal(parseOrdersListDom(html).length, 1);
+});
+
+test("countOrderCardsWithoutOrderId: counts a card with no .yohtmlc-order-id that parseOrdersListDom silently drops", () => {
+ const html = `${NORMAL_CARD_HTML}${CARD_WITHOUT_ORDER_ID_HTML}
`;
+ // The card without an order id vanishes from parseOrdersListDom's output —
+ // this is the pre-existing, unfixed behavior of parseOrderCard/findOrderId.
+ const orders = parseOrdersListDom(html);
+ assert.equal(orders.length, 1, "only the normal card survives parseOrdersListDom");
+ // countOrderCardsWithoutOrderId is what makes that loss visible.
+ assert.equal(countOrderCardsWithoutOrderId(html), 1);
+});
+
test("parseOrdersListDom: local real fixture parses ≥5 orders with ids + dates", {
skip: !existsSync(LOCAL_RAW_DIR),
}, () => {
@@ -523,7 +580,9 @@ test("buildOrderRecord: both list + detail present — detail wins for enrichmen
assert.equal(rec.delivery_status, "Arriving tomorrow");
assert.equal(rec.recipient_name, "Fictional Person");
assert.equal(rec.payment_method_summary, "Visa ending in 1234");
- // item_count = max(list.items.length, detail.items.length) = max(1, 2) = 2.
+ // Both surfaces saw items, so the larger count wins: max(1, 2) = 2. The
+ // zero case is where the two surfaces stop being interchangeable — see
+ // item-count-honesty.test.ts.
assert.equal(rec.item_count, 2);
assert.equal(rec.fetched_at, "2024-01-20T00:00:00Z");
});
diff --git a/packages/polyfill-connectors/connectors/amazon/parsers.ts b/packages/polyfill-connectors/connectors/amazon/parsers.ts
index 1be526cb5..c50c22b29 100644
--- a/packages/polyfill-connectors/connectors/amazon/parsers.ts
+++ b/packages/polyfill-connectors/connectors/amazon/parsers.ts
@@ -568,6 +568,29 @@ export function parseOrdersListDom(html: string): ListPageOrder[] {
return results;
}
+/**
+ * Count `.order-card`/`.js-order-card` elements that `parseOrdersListDom`
+ * silently drops because `findOrderId` found no `.yohtmlc-order-id` span
+ * matching the canonical order-id shape. `parseOrderCard`'s `if (!orderId)
+ * return null` (and `parseOrdersListDom`'s discard of that `null`) leaves no
+ * other trace: the card never reaches the shape-check, so it never produces
+ * a SKIP_RESULT, a coverage-considered id, or a rejection — a page can lose
+ * cards here with zero connector-visible evidence. Called alongside
+ * `parseOrdersListDom` so a caller can detect and report the gap the parse
+ * step itself cannot see (it only ever sees what survived).
+ */
+export function countOrderCardsWithoutOrderId(html: string): number {
+ const { document } = parseHTML(html);
+ const cards = document.querySelectorAll(".order-card, .js-order-card");
+ let dropped = 0;
+ for (const card of cards) {
+ if (!findOrderId(card)) {
+ dropped += 1;
+ }
+ }
+ return dropped;
+}
+
export function parseOrderDate(raw: string | null | undefined): string | null {
if (!raw) {
return null;
@@ -698,6 +721,43 @@ export function buildOrderItemRecord(orderId: string, orderDate: string, merged:
* with the detail-page fetch. Prefers detail-page grand total (includes tax)
* over the list-page total.
*/
+/**
+ * Resolve the order's item count, or null when neither surface can prove one.
+ *
+ * Amazon asserts no item count anywhere we read; the number is the count of
+ * item elements we managed to parse, from two independent surfaces. That makes
+ * a zero ambiguous, and the ambiguity was being resolved the wrong way.
+ *
+ * The list card only renders item titles for SMALL orders. Live data is
+ * unambiguous about this: across 1,183 collected orders, `item_count === 0`
+ * NEVER occurs on an order holding 1 or 2 items, and occurs on 53 orders
+ * holding 3-7 — Amazon collapses 3+ item orders behind a "+N more items"
+ * affordance that renders no per-item titles. For those orders the list
+ * contributes 0 and the count rests entirely on the order-detail page.
+ *
+ * When that detail fetch is deferred (per-run attempt budget, temporary-failure
+ * cap, or a latched session repair) `detail` is null, and the previous
+ * `Math.max(list, detail ?? 0)` collapsed to a confident `0`. Every one of
+ * those 53 orders also has a null `shipping_address_summary`, confirming the
+ * detail page — not the parse — is what was missing. The record then claimed an
+ * empty order while the database held 3-7 item rows for it.
+ *
+ * A non-nullable count cannot express "not fetched", so a fetch failure was
+ * indistinguishable from a genuinely empty order. Returning null when no
+ * surface saw an item keeps a real empty order (which reaches here with a
+ * successful detail) reporting 0, while an unfetched one reports unknown.
+ */
+function resolveItemCount(listOrder: ListPageOrder, detail: OrderDetail | null): number | null {
+ const counted = Math.max(listOrder.items.length, detail?.items?.length ?? 0);
+ if (counted > 0) {
+ return counted;
+ }
+ // Nothing counted. Only the detail page can distinguish an empty order from
+ // an unobserved one: with it we looked and found nothing, without it we never
+ // looked at the surface that carries items for anything but a tiny order.
+ return detail ? 0 : null;
+}
+
export function buildOrderRecord(
listOrder: ListPageOrder,
detail: OrderDetail | null,
@@ -717,7 +777,7 @@ export function buildOrderRecord(
payment_method_summary: detail?.payment_method_summary || null,
gift_order: detail?.gift_order ?? false,
digital_order: detail?.digital_order ?? false,
- item_count: Math.max(listOrder.items.length, detail?.items?.length ?? 0),
+ item_count: resolveItemCount(listOrder, detail),
fetched_at: emittedAt,
};
}
diff --git a/packages/polyfill-connectors/connectors/amazon/schemas.ts b/packages/polyfill-connectors/connectors/amazon/schemas.ts
index cc8166b77..54de75d92 100644
--- a/packages/polyfill-connectors/connectors/amazon/schemas.ts
+++ b/packages/polyfill-connectors/connectors/amazon/schemas.ts
@@ -96,7 +96,10 @@ export const orderSchema = z.object({
payment_method_summary: paymentMethodSchema,
gift_order: z.boolean(),
digital_order: z.boolean(),
- item_count: z.number().int().min(0),
+ // Nullable because a deferred order-detail fetch leaves the count genuinely
+ // unknown, and a non-nullable count can only express that as `0` — which
+ // reads as a proven-empty order. See `resolveItemCount`.
+ item_count: z.number().int().min(0).nullable(),
fetched_at: z.string(),
});
diff --git a/packages/polyfill-connectors/connectors/amazon/types.ts b/packages/polyfill-connectors/connectors/amazon/types.ts
index 77ea0799e..4593fdaa4 100644
--- a/packages/polyfill-connectors/connectors/amazon/types.ts
+++ b/packages/polyfill-connectors/connectors/amazon/types.ts
@@ -61,7 +61,7 @@ export interface OrdersRecord {
fetched_at: string;
gift_order: boolean;
id: string;
- item_count: number;
+ item_count: number | null;
order_date: string;
order_total: string | null;
order_total_cents: number | null;
diff --git a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts
index fffedffcf..a6ae99be2 100644
--- a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts
+++ b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.test.ts
@@ -4,6 +4,7 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import {
+ addressbookMultiget,
addressbookQueryAll,
CardDavStructuralError,
listAddressBooks,
@@ -475,3 +476,83 @@ test("syncCollectionReport: a genuinely transient HTTP-status failure is a plain
await server.close();
}
});
+
+test("syncCollectionReport: reports enumerated members that carry no inlined address-data", async () => {
+ // Verbatim shape of iCloud's sync-collection multistatus (probe against
+ // p196-contacts.icloud.com, 2026-08-19): getetag only, no address-data, and
+ // the collection's own href (no trailing slash) listed beside its members.
+ const body = `
+
+
+ /18913754167/carddavhome/card
+ mfhpss66HTTP/1.1 200 OK
+
+
+ /18913754167/carddavhome/card/3FF77740-4879-41BB-8267-0429C3EB15A8.vcf
+ "msndeisq"HTTP/1.1 200 OK
+
+HwoQEgwAABJ++QlJcAACAAEYAhgAIhYI
+`;
+
+ const result = await syncCollectionReport({
+ bookUrl: "https://p196-contacts.icloud.com/18913754167/carddavhome/card/",
+ authHeader: AUTH_HEADER,
+ fetchImpl: async () => syntheticResponse(207, {}, body),
+ trustedOrigins: ["https://p196-contacts.icloud.com"],
+ priorSyncToken: "",
+ });
+
+ assert.equal(result.supportsSyncCollection, true);
+ // The member must be surfaced for a multiget follow-up, not silently dropped.
+ assert.deepEqual(result.hrefsMissingBodies, [
+ "https://p196-contacts.icloud.com/18913754167/carddavhome/card/3FF77740-4879-41BB-8267-0429C3EB15A8.vcf",
+ ]);
+ // The collection's own href is not a member resource.
+ assert.equal(result.resources.length, 0);
+ assert.equal(result.deletedHrefs.length, 0);
+});
+
+test("addressbookMultiget: fetches vCard bodies for explicitly requested hrefs", async () => {
+ const server = await startFakeCardDavServer({
+ username: USERNAME,
+ password: PASSWORD,
+ syncCollectionOmitsAddressData: true,
+ });
+ try {
+ server.contacts.set("gina", {
+ uid: "gina",
+ href: "/addressbooks/owner/card/gina.vcf",
+ vcard: buildVCard({ uid: "gina", fn: "Gina Example" }),
+ });
+ const discovery = await discover(server.origin);
+ const books = await listAddressBooks({
+ homeUrl: discovery.addressBookHomeUrl,
+ authHeader: AUTH_HEADER,
+ fetchImpl,
+ trustedOrigins: discovery.visitedOrigins,
+ });
+ const bookUrl = books[0]?.url ?? "";
+
+ const sync = await syncCollectionReport({
+ bookUrl,
+ authHeader: AUTH_HEADER,
+ fetchImpl,
+ trustedOrigins: discovery.visitedOrigins,
+ priorSyncToken: "",
+ });
+ assert.equal(sync.resources.length, 0, "server inlines nothing");
+ assert.equal(sync.hrefsMissingBodies.length, 1);
+
+ const fetched = await addressbookMultiget({
+ bookUrl,
+ authHeader: AUTH_HEADER,
+ fetchImpl,
+ trustedOrigins: discovery.visitedOrigins,
+ hrefs: sync.hrefsMissingBodies,
+ });
+ assert.equal(fetched.length, 1);
+ assert.equal(parseVCards(fetched[0]?.vcardText ?? "")[0]?.fn, "Gina Example");
+ } finally {
+ await server.close();
+ }
+});
diff --git a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts
index e3decb917..9c6597c92 100644
--- a/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts
+++ b/packages/polyfill-connectors/connectors/apple_contacts/carddav-client.ts
@@ -34,6 +34,17 @@ export interface VCardResource {
export interface SyncCollectionResult {
deletedHrefs: string[];
+ /**
+ * Hrefs the REPORT enumerated as present-and-current but for which the
+ * server returned NO `address-data` body, even though the request asked for
+ * it. RFC 6578 §3.2 does not oblige a server to inline arbitrary properties
+ * in a sync-collection response, and iCloud in fact does not: its
+ * sync-collection multistatus carries `getetag` only. These hrefs are real
+ * members whose bodies must be fetched in a follow-up
+ * `addressbook-multiget` (RFC 6352 §8.7) — dropping them silently is how a
+ * populated address book reports as empty.
+ */
+ hrefsMissingBodies: string[];
resources: VCardResource[];
supportsSyncCollection: boolean;
syncToken?: string;
@@ -207,6 +218,48 @@ const ADDRESSBOOK_QUERY_ALL_BODY = `
`;
+/** Hrefs per addressbook-multiget request. Bounds both the request body and
+ * the response size so one large change set can't produce an unbounded
+ * round-trip (davRequest's MAX_RESPONSE_BYTES would reject it outright). */
+const MULTIGET_CHUNK_SIZE = 50;
+
+function xmlEscapeText(value: string): string {
+ return value
+ .replace(/&/g, "&")
+ .replace(//g, ">")
+ .replace(/"/g, """)
+ .replace(/'/g, "'");
+}
+
+/**
+ * Reduce an href to the path form (`/path/to/card.vcf`) a multiget body must
+ * carry. `syncCollectionReport` resolves member hrefs to absolute URLs so the
+ * rest of the connector can key records by a stable absolute id, but iCloud
+ * answers `400 Bad Request` to an absolute `` inside an
+ * addressbook-multiget and `207` to the path form (probe against
+ * p196-contacts.icloud.com, 2026-08-19). RFC 6352 §8.7's own examples use the
+ * path form, so this is the interoperable shape, not an Apple workaround.
+ * A value that does not parse as a URL is passed through unchanged.
+ */
+function hrefPathOnly(href: string): string {
+ try {
+ const url = new URL(href);
+ return `${url.pathname}${url.search}`;
+ } catch {
+ return href;
+ }
+}
+
+const MULTIGET_BODY = (hrefs: readonly string[]): string => `
+
+
+
+
+
+${hrefs.map((href) => ` ${xmlEscapeText(hrefPathOnly(href))}`).join("\n")}
+`;
+
/**
* Attempt RFC 6578 `sync-collection` REPORT. `priorSyncToken` empty string
* means "initial sync" per RFC 6578 §3.2. Returns
@@ -232,22 +285,42 @@ export async function syncCollectionReport(args: {
trustedOrigins
);
if (res.status === 405 || res.status === 501 || res.status === 415) {
- return { resources: [], deletedHrefs: [], supportsSyncCollection: false, truncated: false };
+ return {
+ resources: [],
+ deletedHrefs: [],
+ hrefsMissingBodies: [],
+ supportsSyncCollection: false,
+ truncated: false,
+ };
}
if (res.status === 507) {
// Insufficient storage / token too old (RFC 6578 §3.6): server wants a
// full resync. Signal via empty sync token so the caller re-derives.
- return { resources: [], deletedHrefs: [], supportsSyncCollection: true, truncated: false, syncToken: "" };
+ return {
+ resources: [],
+ deletedHrefs: [],
+ hrefsMissingBodies: [],
+ supportsSyncCollection: true,
+ truncated: false,
+ syncToken: "",
+ };
}
if (res.status < 200 || res.status >= 300) {
throw new Error(`carddav_sync_collection_failed: status=${String(res.status)}`);
}
const newSyncToken = extractTag(res.text, "sync-token");
if (!newSyncToken) {
- return { resources: [], deletedHrefs: [], supportsSyncCollection: false, truncated: false };
+ return {
+ resources: [],
+ deletedHrefs: [],
+ hrefsMissingBodies: [],
+ supportsSyncCollection: false,
+ truncated: false,
+ };
}
const resources: VCardResource[] = [];
const deletedHrefs: string[] = [];
+ const hrefsMissingBodies: string[] = [];
for (const block of extractAllHrefBlocks(res.text)) {
const href = extractTag(block, "href");
if (!href) {
@@ -258,18 +331,97 @@ export async function syncCollectionReport(args: {
deletedHrefs.push(new URL(href, res.finalUrl).toString());
continue;
}
+ const absoluteHref = new URL(href, res.finalUrl).toString();
+ // The collection's own href appears in the multistatus alongside its
+ // members (iCloud reports the collection with its own getetag). It is not
+ // a contact resource, so it must not be queued for a body fetch.
+ if (isSameCollection(absoluteHref, res.finalUrl)) {
+ continue;
+ }
const vcardText = extractTag(block, "address-data");
if (!vcardText) {
+ // Enumerated member with no inlined body: record it for the multiget
+ // follow-up rather than dropping it.
+ hrefsMissingBodies.push(absoluteHref);
continue;
}
const etag = extractTag(block, "getetag");
resources.push({
- href: new URL(href, res.finalUrl).toString(),
+ href: absoluteHref,
...(etag ? { etag } : {}),
vcardText: decodeXmlEntities(vcardText),
});
}
- return { resources, deletedHrefs, supportsSyncCollection: true, syncToken: newSyncToken, truncated: false };
+ return {
+ resources,
+ deletedHrefs,
+ hrefsMissingBodies,
+ supportsSyncCollection: true,
+ syncToken: newSyncToken,
+ truncated: false,
+ };
+}
+
+/** True when two URLs name the same collection, ignoring a trailing slash.
+ * iCloud reports the collection itself as `.../card` while the request URL is
+ * `.../card/`, so a bare string compare would miss it. */
+function isSameCollection(candidate: string, collectionUrl: string): boolean {
+ const strip = (u: string): string => u.replace(TRAILING_SLASH_RE, "");
+ return strip(candidate) === strip(collectionUrl);
+}
+
+const TRAILING_SLASH_RE = /\/+$/;
+
+/**
+ * Fetch vCard bodies for an explicit set of member hrefs
+ * (RFC 6352 §8.7 `addressbook-multiget`). This is the companion to
+ * `syncCollectionReport` for servers — iCloud among them — that enumerate
+ * members in a sync-collection response without inlining `address-data`.
+ *
+ * Requests are chunked so a large change set cannot produce a single
+ * unbounded request or response body.
+ */
+export async function addressbookMultiget(args: {
+ authHeader: string;
+ bookUrl: string;
+ fetchImpl: DiscoveryFetch;
+ hrefs: readonly string[];
+ trustedOrigins: string[];
+}): Promise {
+ const { authHeader, bookUrl, fetchImpl, hrefs, trustedOrigins } = args;
+ if (hrefs.length === 0) {
+ return [];
+ }
+ const resources: VCardResource[] = [];
+ for (let start = 0; start < hrefs.length; start += MULTIGET_CHUNK_SIZE) {
+ const chunk = hrefs.slice(start, start + MULTIGET_CHUNK_SIZE);
+ const res = await davRequest(
+ fetchImpl,
+ bookUrl,
+ "REPORT",
+ authHeader,
+ { Depth: "1" },
+ MULTIGET_BODY(chunk),
+ trustedOrigins
+ );
+ if (res.status < 200 || res.status >= 300) {
+ throw new Error(`carddav_addressbook_multiget_failed: status=${String(res.status)}`);
+ }
+ for (const block of extractAllHrefBlocks(res.text)) {
+ const href = extractTag(block, "href");
+ const vcardText = extractTag(block, "address-data");
+ if (!(href && vcardText)) {
+ continue;
+ }
+ const etag = extractTag(block, "getetag");
+ resources.push({
+ href: new URL(href, res.finalUrl).toString(),
+ ...(etag ? { etag } : {}),
+ vcardText: decodeXmlEntities(vcardText),
+ });
+ }
+ }
+ return resources;
}
/** Bounded full snapshot via `addressbook-query` (RFC 6352 §8.6) — the
diff --git a/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.test.ts b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.test.ts
new file mode 100644
index 000000000..1477c5be3
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.test.ts
@@ -0,0 +1,207 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Completeness-anchor tests for the `contact_groups` stream.
+ *
+ * `contact_groups` is manifest-REQUIRED and had emitted zero records for
+ * this owner, ever — including zero tombstones. The connector read only the
+ * vCard-standard `CATEGORIES` property, but iCloud stores each group as its
+ * own vCard resource marked `X-ADDRESSBOOKSERVER-KIND:group`. So for an
+ * iCloud account the stream could not emit a record no matter what the
+ * account contained, and the resulting zero was indistinguishable from a
+ * genuinely empty address book.
+ *
+ * The vCard bodies below use Apple's real wire shape.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import {
+ type GroupAnchor,
+ groupAnchorVerdict,
+ groupMemberUids,
+ isGroupVCard,
+ partitionVCards,
+} from "./group-vcards.ts";
+import { parseVCards } from "./vcard.ts";
+
+/** Apple's real group-vCard wire shape, as iCloud serves it. */
+const APPLE_GROUP_VCARD = [
+ "BEGIN:VCARD",
+ "VERSION:3.0",
+ "N:Family;;;;",
+ "FN:Family",
+ "UID:11111111-2222-3333-4444-555555555555",
+ "X-ADDRESSBOOKSERVER-KIND:group",
+ "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:AAAAAAAA-0000-0000-0000-000000000001",
+ "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:BBBBBBBB-0000-0000-0000-000000000002",
+ "END:VCARD",
+].join("\r\n");
+
+const PERSON_VCARD = [
+ "BEGIN:VCARD",
+ "VERSION:3.0",
+ "FN:Tim Nunamaker",
+ "UID:AAAAAAAA-0000-0000-0000-000000000001",
+ "END:VCARD",
+].join("\r\n");
+
+const PERSON_WITH_CATEGORIES = [
+ "BEGIN:VCARD",
+ "VERSION:3.0",
+ "FN:Someone Else",
+ "UID:BBBBBBBB-0000-0000-0000-000000000002",
+ "CATEGORIES:Work,Friends",
+ "END:VCARD",
+].join("\r\n");
+
+function card(text: string) {
+ const [parsed] = parseVCards(text);
+ assert.ok(parsed, "fixture failed to parse");
+ return parsed;
+}
+
+// ─── isGroupVCard: see the groups iCloud actually ships ──────────────────
+
+test("isGroupVCard recognises Apple's real group vCard", () => {
+ // Before this, the connector was blind to exactly this resource.
+ assert.equal(isGroupVCard(card(APPLE_GROUP_VCARD)), true);
+});
+
+test("isGroupVCard recognises the RFC 6350 standard KIND:group", () => {
+ const standard = ["BEGIN:VCARD", "VERSION:4.0", "FN:Team", "KIND:group", "END:VCARD"].join("\r\n");
+ assert.equal(isGroupVCard(card(standard)), true);
+});
+
+test("isGroupVCard is case-insensitive on the value", () => {
+ const upper = ["BEGIN:VCARD", "VERSION:3.0", "FN:Team", "X-ADDRESSBOOKSERVER-KIND:GROUP", "END:VCARD"].join("\r\n");
+ assert.equal(isGroupVCard(card(upper)), true);
+});
+
+test("isGroupVCard treats an ordinary person as a contact", () => {
+ assert.equal(isGroupVCard(card(PERSON_VCARD)), false);
+});
+
+test("isGroupVCard fails safe toward contact on an unknown KIND", () => {
+ // A resource is a group only when the server explicitly says so, so this
+ // predicate can never silently drop a real person from `contacts`.
+ const org = ["BEGIN:VCARD", "VERSION:4.0", "FN:ACME", "KIND:org", "END:VCARD"].join("\r\n");
+ assert.equal(isGroupVCard(card(org)), false);
+});
+
+// ─── groupMemberUids ─────────────────────────────────────────────────────
+
+test("groupMemberUids strips the urn:uuid: prefix Apple ships", () => {
+ assert.deepEqual(groupMemberUids(card(APPLE_GROUP_VCARD)), [
+ "AAAAAAAA-0000-0000-0000-000000000001",
+ "BBBBBBBB-0000-0000-0000-000000000002",
+ ]);
+});
+
+test("groupMemberUids returns empty for a group with no members", () => {
+ const empty = ["BEGIN:VCARD", "VERSION:3.0", "FN:Empty", "X-ADDRESSBOOKSERVER-KIND:group", "END:VCARD"].join("\r\n");
+ assert.deepEqual(groupMemberUids(card(empty)), []);
+});
+
+test("groupMemberUids de-duplicates while preserving order", () => {
+ const dupes = [
+ "BEGIN:VCARD",
+ "VERSION:3.0",
+ "FN:Dupes",
+ "X-ADDRESSBOOKSERVER-KIND:group",
+ "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:B",
+ "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:A",
+ "X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:B",
+ "END:VCARD",
+ ].join("\r\n");
+ assert.deepEqual(groupMemberUids(card(dupes)), ["B", "A"]);
+});
+
+// ─── partitionVCards: the phantom-contact fix ────────────────────────────
+
+test("partitionVCards keeps a group vCard out of the contact set", () => {
+ // The phantom-contact defect: a group emitted as a contact whose
+ // display_name is the group's name, counted as a covered contact.
+ const resources = [
+ { card: card(PERSON_VCARD) },
+ { card: card(APPLE_GROUP_VCARD) },
+ { card: card(PERSON_WITH_CATEGORIES) },
+ ];
+ const { contacts, groups } = partitionVCards(resources);
+ assert.equal(contacts.length, 2);
+ assert.equal(groups.length, 1);
+ assert.equal(groups[0]?.card.fn, "Family");
+ assert.equal(
+ contacts.some((c) => c.card.fn === "Family"),
+ false,
+ "group leaked into the contact set"
+ );
+});
+
+test("partitionVCards handles a collection with no groups", () => {
+ const { contacts, groups } = partitionVCards([{ card: card(PERSON_VCARD) }]);
+ assert.equal(contacts.length, 1);
+ assert.equal(groups.length, 0);
+});
+
+// ─── groupAnchorVerdict: the anchor ──────────────────────────────────────
+
+function anchor(overrides: Partial): GroupAnchor {
+ return {
+ serverGroupVCards: 0,
+ derivedCategoryGroups: 0,
+ emitted: 0,
+ boundaryEstablished: true,
+ ...overrides,
+ };
+}
+
+test("groupAnchorVerdict refuses to claim anything without a boundary", () => {
+ // An incomplete enumeration cannot prove or disprove completeness.
+ const verdict = groupAnchorVerdict(anchor({ boundaryEstablished: false, serverGroupVCards: 3 }));
+ assert.equal(verdict.status, "unproven");
+});
+
+test("groupAnchorVerdict turns the live zero into a CHECKED zero", () => {
+ // This is the outcome that resolves the original question: the server
+ // enumerated the whole collection and it genuinely holds no groups.
+ assert.equal(groupAnchorVerdict(anchor({})).status, "empty_confirmed");
+});
+
+test("groupAnchorVerdict reports SHORT when the server holds groups we did not emit", () => {
+ // The blindness case: iCloud has 3 group vCards, the connector emitted none.
+ const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 3, emitted: 0 }));
+ assert.equal(verdict.status, "short");
+ assert.equal(verdict.status === "short" && verdict.missing, 3);
+ assert.equal(verdict.status === "short" && verdict.considered, 3);
+});
+
+test("groupAnchorVerdict reports COMPLETE when every server group was emitted", () => {
+ const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 2, emitted: 2 }));
+ assert.equal(verdict.status, "complete");
+ assert.equal(verdict.status === "complete" && verdict.covered, 2);
+});
+
+test("groupAnchorVerdict does NOT flag CATEGORIES groups as an overage", () => {
+ // CATEGORIES groups have no server-side resource, so emitting more than
+ // the measured denominator is correct behaviour, not a defect. A two-way
+ // equality here would flag correct behaviour as failure.
+ const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 1, derivedCategoryGroups: 2, emitted: 3 }));
+ assert.equal(verdict.status, "complete");
+});
+
+test("groupAnchorVerdict keeps a partial shortfall visible", () => {
+ // 4 server groups, only 1 emitted: still short by 3 even though something
+ // was emitted.
+ const verdict = groupAnchorVerdict(anchor({ serverGroupVCards: 4, emitted: 1 }));
+ assert.equal(verdict.status, "short");
+ assert.equal(verdict.status === "short" && verdict.missing, 3);
+});
+
+test("groupAnchorVerdict does not confirm empty when CATEGORIES groups exist", () => {
+ // A CATEGORIES-only account is not an empty one; claiming
+ // `empty_confirmed` there would be a false clean bill.
+ const verdict = groupAnchorVerdict(anchor({ derivedCategoryGroups: 2, emitted: 2 }));
+ assert.notEqual(verdict.status, "empty_confirmed");
+});
diff --git a/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.ts b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.ts
new file mode 100644
index 000000000..dbd65da7f
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/apple_contacts/group-vcards.ts
@@ -0,0 +1,190 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+// Recognition of Apple's group vCards, and the `contact_groups` completeness
+// anchor built on it.
+//
+// THE PROBLEM THIS SOLVES
+// -----------------------
+// `contact_groups` is a manifest-REQUIRED stream that has emitted zero
+// records for this owner, ever — including zero tombstones. Two very
+// different worlds produce that same zero:
+//
+// (a) the address book genuinely has no groups, or
+// (b) the connector cannot see the groups it has.
+//
+// Until now the connector could not tell them apart, and a required stream
+// sitting at zero for an unfalsifiable reason is exactly the defect this
+// work exists to eliminate.
+//
+// The connector derives groups from the standards-based vCard `CATEGORIES`
+// property (RFC 6350 §6.7.1). That derivation is sound as far as it goes.
+// But Apple/iCloud does not represent groups with CATEGORIES: it stores each
+// group as its OWN vCard resource in the same collection, marked
+// `X-ADDRESSBOOKSERVER-KIND:group`, whose members are listed as
+// `X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:` lines.
+//
+// So for an iCloud account, (b) was the live behaviour and the connector
+// reported it as (a).
+//
+// THE SECOND, WORSE CONSEQUENCE
+// -----------------------------
+// A group vCard is a real resource in the collection, so the enumeration
+// fetches it and the contact emitter emits it AS A CONTACT — a phantom whose
+// `display_name` is the group's name, counted as a covered contact. Nothing
+// in the emit path checked `KIND`. `isGroupVCard` is what lets the caller
+// keep groups out of `contacts`.
+//
+// THE ANCHOR
+// ----------
+// CardDAV enumerates the collection fully, so the set of resources is
+// measured at the SOURCE boundary. Partitioning that enumerated set by KIND
+// yields a real denominator for `contact_groups`: the number of group vCards
+// the server actually holds. Comparing it against what was emitted turns the
+// unfalsifiable zero into a checkable claim — and a zero that survives the
+// check is now genuine evidence of an empty account rather than an absence
+// of evidence.
+//
+// CEILING, stated honestly
+// ------------------------
+// This anchors groups as iCloud REPRESENTS them. Two group sources are
+// unified here — Apple group vCards and CATEGORIES — and a group expressed
+// only through CATEGORIES on contacts still has no independent server-side
+// resource to count, so its denominator contribution is derived, not
+// measured. `groupAnchor` reports those two populations separately for that
+// reason; it never merges a derived count into the measured one.
+
+import type { ParsedVCard } from "./vcard.ts";
+
+/** RFC 6350 §6.1.4 defines `KIND`; Apple ships the pre-standard
+ * `X-ADDRESSBOOKSERVER-KIND` on iCloud. Both are accepted so a standards
+ * -compliant server and iCloud are read the same way. */
+const KIND_PROPERTY_NAMES = ["X-ADDRESSBOOKSERVER-KIND", "KIND"];
+
+/** Apple's member property. RFC 6350 §6.6.5 standardises `MEMBER`; iCloud
+ * ships the `X-ADDRESSBOOKSERVER-` prefixed form. */
+const MEMBER_PROPERTY_NAMES = ["X-ADDRESSBOOKSERVER-MEMBER", "MEMBER"];
+
+/** Members are listed as `urn:uuid:`. The prefix is stripped so a
+ * member uid compares equal to the `UID` the same server reports on the
+ * member's own vCard, which `vcard.ts` already strips identically. */
+const URN_UUID_PREFIX_RE = /^urn:uuid:/i;
+
+function rawValues(card: ParsedVCard, names: readonly string[]): string[] {
+ const wanted = new Set(names.map((n) => n.toUpperCase()));
+ return card.rawProperties.filter((p) => wanted.has(p.name.toUpperCase())).map((p) => p.value.trim());
+}
+
+/**
+ * True when this vCard is a GROUP rather than a person.
+ *
+ * Fails SAFE toward "contact": a resource is only treated as a group when
+ * the server explicitly said so. An unrecognised or absent KIND means the
+ * resource keeps its existing treatment as a contact, so this predicate can
+ * never silently remove a real person from `contacts`.
+ */
+export function isGroupVCard(card: ParsedVCard): boolean {
+ return rawValues(card, KIND_PROPERTY_NAMES).some((v) => v.toLowerCase() === "group");
+}
+
+/**
+ * The member UIDs a group vCard lists, `urn:uuid:` prefix stripped and
+ * blanks dropped. Order is preserved and duplicates are removed, so the
+ * result is a stable set-like list suitable for a record body.
+ */
+export function groupMemberUids(card: ParsedVCard): string[] {
+ const out: string[] = [];
+ const seen = new Set();
+ for (const value of rawValues(card, MEMBER_PROPERTY_NAMES)) {
+ const uid = value.replace(URN_UUID_PREFIX_RE, "").trim();
+ if (uid && !seen.has(uid)) {
+ seen.add(uid);
+ out.push(uid);
+ }
+ }
+ return out;
+}
+
+/** The enumerated collection, partitioned by what each resource actually is. */
+export interface VCardPartition {
+ contacts: T[];
+ groups: T[];
+}
+
+/**
+ * Split an enumerated collection into person vCards and group vCards.
+ *
+ * This is the single place that decides what a resource IS, so the contact
+ * emitter and the group anchor cannot disagree about it — the phantom-contact
+ * defect was precisely such a disagreement, with no partition at all.
+ */
+export function partitionVCards(resources: readonly T[]): VCardPartition {
+ const contacts: T[] = [];
+ const groups: T[] = [];
+ for (const resource of resources) {
+ if (isGroupVCard(resource.card)) {
+ groups.push(resource);
+ } else {
+ contacts.push(resource);
+ }
+ }
+ return { contacts, groups };
+}
+
+/**
+ * The `contact_groups` completeness anchor for one address book.
+ *
+ * `serverGroupVCards` is the MEASURED denominator: group resources the
+ * server itself enumerated. `derivedCategoryGroups` is reported alongside it
+ * but deliberately kept separate — it is derived from contact bodies, not
+ * measured at the boundary, and merging the two would manufacture a
+ * denominator partly out of the data it is meant to verify.
+ */
+export interface GroupAnchor {
+ /** True only when the enumeration reached a complete boundary. */
+ boundaryEstablished: boolean;
+ /** Distinct group names derived from contacts' CATEGORIES. Derived, not measured. */
+ derivedCategoryGroups: number;
+ /** Group records actually emitted this run. */
+ emitted: number;
+ /** Group vCards the server enumerated. Measured at the source boundary. */
+ serverGroupVCards: number;
+}
+
+export type GroupAnchorVerdict =
+ /** Enumeration was incomplete; no claim about completeness is possible. */
+ | { status: "unproven"; reason: "boundary_not_established" }
+ /** The server enumerated no groups and no CATEGORIES groups were derived.
+ * This is a CHECKED zero — genuine evidence of an empty account. */
+ | { status: "empty_confirmed" }
+ /** Every group the server holds is accounted for in what was emitted. */
+ | { status: "complete"; considered: number; covered: number }
+ /** The server holds groups that were not emitted. */
+ | { status: "short"; considered: number; covered: number; missing: number };
+
+/**
+ * Turn the measured anchor into a verdict.
+ *
+ * The comparison is one-directional on purpose. Emitting MORE groups than
+ * the server enumerated as group vCards is normal and correct: CATEGORIES
+ * groups have no server-side resource, so they legitimately add to the
+ * emitted count without adding to the measured denominator. Only "the server
+ * holds groups we did not emit" is a gap.
+ *
+ * This mirrors the deletion-safe reasoning used elsewhere in the fleet: a
+ * two-way equality would flag correct behaviour as failure.
+ */
+export function groupAnchorVerdict(anchor: GroupAnchor): GroupAnchorVerdict {
+ if (!anchor.boundaryEstablished) {
+ return { status: "unproven", reason: "boundary_not_established" };
+ }
+ if (anchor.serverGroupVCards === 0 && anchor.derivedCategoryGroups === 0 && anchor.emitted === 0) {
+ return { status: "empty_confirmed" };
+ }
+ const considered = anchor.serverGroupVCards;
+ const covered = Math.min(anchor.emitted, considered);
+ if (covered < considered) {
+ return { status: "short", considered, covered, missing: considered - covered };
+ }
+ return { status: "complete", considered, covered };
+}
diff --git a/packages/polyfill-connectors/connectors/apple_contacts/index.ts b/packages/polyfill-connectors/connectors/apple_contacts/index.ts
index 676c5c6c8..83b541978 100644
--- a/packages/polyfill-connectors/connectors/apple_contacts/index.ts
+++ b/packages/polyfill-connectors/connectors/apple_contacts/index.ts
@@ -50,6 +50,7 @@ import {
} from "../../src/connector-runtime.ts";
import { type FingerprintCursor, openFingerprintCursor } from "../../src/fingerprint-cursor.ts";
import {
+ addressbookMultiget,
addressbookQueryAll,
CardDavStructuralError,
listAddressBooks,
@@ -63,6 +64,13 @@ import {
discoverCardDav,
nativeFetchAdapter,
} from "./discovery.ts";
+import {
+ type GroupAnchor,
+ groupAnchorVerdict,
+ groupMemberUids,
+ isGroupVCard,
+ partitionVCards,
+} from "./group-vcards.ts";
import { validateRecord } from "./schemas.ts";
import { categoriesOf, type ParsedVCard, parseVCards } from "./vcard.ts";
@@ -234,18 +242,47 @@ export function groupRecord(bookUrl: string, name: string, memberUids: string[])
};
}
-/** Derive group-membership records from every contact's CATEGORIES field.
- * This is CardDAV/vCard-standard (RFC 6350 §6.7.1), unlike Apple's
- * proprietary group-vCard mechanism, which is unconfirmed for iCloud. */
+/**
+ * Derive group-membership records from BOTH mechanisms a CardDAV server may
+ * use.
+ *
+ * 1. Apple's group vCards — a resource in the collection carrying
+ * `X-ADDRESSBOOKSERVER-KIND:group` and `X-ADDRESSBOOKSERVER-MEMBER`
+ * lines. This is how iCloud actually stores groups.
+ * 2. The vCard-standard `CATEGORIES` property on each contact
+ * (RFC 6350 §6.7.1).
+ *
+ * Only (2) was previously read. For an iCloud account that meant
+ * `contact_groups` — a manifest-REQUIRED stream — could never emit a record
+ * no matter how many groups the account had, and the resulting zero was
+ * indistinguishable from a genuinely empty address book.
+ *
+ * A group vCard wins over a same-named CATEGORIES group: the server's own
+ * membership list is authoritative over one inferred from contact bodies.
+ */
export function deriveGroups(bookUrl: string, cards: ReadonlyArray<{ card: ParsedVCard; uid: string }>): RecordData[] {
+ const { contacts, groups } = partitionVCards(cards);
+
const membersByGroup = new Map();
- for (const { card, uid } of cards) {
+ for (const { card, uid } of contacts) {
for (const category of categoriesOf(card)) {
const members = membersByGroup.get(category) ?? [];
members.push(uid);
membersByGroup.set(category, members);
}
}
+
+ // Apple group vCards are authoritative; they overwrite a CATEGORIES-derived
+ // entry of the same name rather than merging into it, so a group's
+ // membership is never half server-stated and half inferred.
+ for (const { card } of groups) {
+ const name = card.fn?.trim();
+ if (!name) {
+ continue;
+ }
+ membersByGroup.set(name, groupMemberUids(card));
+ }
+
return [...membersByGroup.entries()].map(([name, members]) => groupRecord(bookUrl, name, members));
}
@@ -253,6 +290,10 @@ interface AddressBookCollectionCtx {
authHeader: string;
book: { url: string; displayName?: string };
bookCursor: FingerprintCursor;
+ /** See `BaseCollectContext.collectionMode`. `"full_refresh"` makes this book
+ * ignore its stored `sync_token` for this run. `undefined` means
+ * `"incremental"`, matching that contract. */
+ collectionMode: "full_refresh" | "incremental" | undefined;
emit: (msg: { type: "STATE"; stream: string; cursor: unknown }) => Promise;
emitRecord: (stream: string, data: RecordData) => Promise;
fetchImpl: DiscoveryFetch;
@@ -319,10 +360,18 @@ async function emitContactGroupsIfRequested(args: {
emitRecord: (stream: string, data: RecordData) => Promise;
requested: Map;
seenCards: Array<{ card: ParsedVCard; uid: string }>;
-}): Promise {
+}): Promise<{ emitted: number; anchor: GroupAnchor }> {
const { bookUrl, boundaryEstablished, emit, emitRecord, requested, seenCards } = args;
+ const { contacts, groups } = partitionVCards(seenCards);
+ const derivedCategoryGroups = new Set(contacts.flatMap(({ card }) => categoriesOf(card))).size;
+ const anchor: GroupAnchor = {
+ serverGroupVCards: groups.length,
+ derivedCategoryGroups,
+ emitted: 0,
+ boundaryEstablished,
+ };
if (!(requested.has("contact_groups") && boundaryEstablished)) {
- return 0;
+ return { emitted: 0, anchor };
}
let groupsEmitted = 0;
@@ -331,7 +380,50 @@ async function emitContactGroupsIfRequested(args: {
groupsEmitted += 1;
}
await emit({ type: "STATE", stream: "contact_groups", cursor: { fetched_at: nowIso() } });
- return groupsEmitted;
+ return { emitted: groupsEmitted, anchor: { ...anchor, emitted: groupsEmitted } };
+}
+
+/**
+ * Fetch and emit vCard bodies for members a sync-collection response
+ * enumerated without inlining `address-data` (RFC 6352 §8.7
+ * addressbook-multiget). Returns the count of enumerated members whose body
+ * the server never handed over, so the caller can keep them in the coverage
+ * denominator instead of dropping them.
+ */
+async function hydrateMissingBodies(args: {
+ authHeader: string;
+ bookUrl: string;
+ emitContactRecord: (resource: VCardResource) => Promise;
+ fetchImpl: DiscoveryFetch;
+ hrefs: readonly string[];
+ progress: (message: string, extra?: { count?: number; stream?: string; total?: number }) => Promise;
+ trustedOrigins: string[];
+}): Promise {
+ const { authHeader, bookUrl, emitContactRecord, fetchImpl, hrefs, progress, trustedOrigins } = args;
+ if (hrefs.length === 0) {
+ return 0;
+ }
+ await progress("Fetching contact bodies the sync response did not inline", {
+ stream: "contacts",
+ count: 0,
+ total: hrefs.length,
+ });
+ const fetched = await addressbookMultiget({ bookUrl, authHeader, fetchImpl, trustedOrigins, hrefs }).catch(
+ (err: unknown) => {
+ throw classifyCardDavRequestFailure(err, { retryableByDefault: true });
+ }
+ );
+ const fetchedByHref = new Map(fetched.map((resource) => [resource.href, resource]));
+ let unfetched = 0;
+ for (const href of hrefs) {
+ const resource = fetchedByHref.get(href);
+ if (resource) {
+ await emitContactRecord(resource);
+ continue;
+ }
+ unfetched += 1;
+ }
+ return unfetched;
}
async function loadFullGroupSnapshot(args: {
@@ -359,6 +451,44 @@ async function loadFullGroupSnapshot(args: {
return { cards, complete };
}
+/**
+ * No token to resume from: `resolveSyncResult` treats `undefined` as "initial
+ * sync". Named so the full_refresh branch below states what it is doing rather
+ * than falling off the end of the function.
+ */
+const WITHHOLD_SYNC_TOKEN = undefined;
+
+/**
+ * Decide which stored sync token (if any) this run may resume from.
+ *
+ * `full_refresh` is the owner/operator bypass BaseCollectContext.collectionMode
+ * defines: a connector with its own incremental bookkeeping MUST ignore that
+ * bookkeeping and walk each stream to its natural end. Withholding the stored
+ * token here (rather than deleting it) sends `resolveSyncResult` down its
+ * initial-sync path, which sets `fullBoundary` and so re-establishes the
+ * contacts enumeration boundary this run — the ONLY way the contacts coverage
+ * claim can be emitted for a book that already has a token. The stored token is
+ * left untouched on disk: the run rewrites it from its own sync-collection
+ * response, and a failed run persists nothing, so an incremental resume stays
+ * available either way. `undefined` collectionMode means `"incremental"`, per
+ * that same contract.
+ */
+async function resolveResumableSyncToken(
+ collectionMode: "full_refresh" | "incremental" | undefined,
+ storedSyncToken: string | undefined,
+ progress: AddressBookCollectionCtx["progress"]
+): Promise {
+ if (collectionMode !== "full_refresh") {
+ return storedSyncToken;
+ }
+ if (storedSyncToken) {
+ await progress("Full refresh requested: re-enumerating this address book", { stream: "contacts" });
+ }
+ // Withhold the stored token so `resolveSyncResult` takes its initial-sync
+ // path and re-establishes the enumeration boundary. The token stays on disk.
+ return WITHHOLD_SYNC_TOKEN;
+}
+
/**
* Collect one address book: probe sync capability, fetch (sync-collection or
* bounded full snapshot), emit the address-book entity record plus contact +
@@ -367,9 +497,11 @@ async function loadFullGroupSnapshot(args: {
* is the whole per-book unit of work in one place.
*/
async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
+ contactsBoundaryEstablished: boolean;
contactsConsidered: number;
contactsCovered: number;
covered: boolean;
+ groupAnchor: GroupAnchor;
groupsEmitted: number;
groupsBoundaryEstablished: boolean;
hadUnparseableResource: boolean;
@@ -378,6 +510,7 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
book,
bookCursor,
authHeader,
+ collectionMode,
fetchImpl,
trustedOrigins,
state,
@@ -392,13 +525,17 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
state.contacts as Record }>
)?.[bookKey];
+ // See `resolveResumableSyncToken`: a full_refresh run withholds the stored
+ // token so this book re-establishes its enumeration boundary.
+ const priorSyncToken = await resolveResumableSyncToken(collectionMode, priorSync?.sync_token, progress);
+
await progress("Probing sync capability", { stream: "contacts" });
const syncResult = await resolveSyncResult({
bookUrl: book.url,
authHeader,
fetchImpl,
trustedOrigins,
- priorSyncToken: priorSync?.sync_token,
+ priorSyncToken,
}).catch((err: unknown) => {
// resolveSyncResult throws either the HTTP-status-shaped
// carddav_sync_collection_failed (transient — retryable, matching the
@@ -414,7 +551,18 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
// An incremental sync reports only changes, so its empty resource list is
// not an empty inventory. Only the initial sync (no prior token) or the
// non-incremental fallback establishes the full contact boundary.
- let groupsBoundaryEstablished = !supportsSync || fullBoundary;
+ //
+ // This is a CONTACTS boundary fact as much as a groups one: on an
+ // incremental run `resourcesEnumerated` counts changed resources, not the
+ // address book's inventory, so a no-change run measures 0 without having
+ // enumerated anything. Emitting that as `considered: 0` would hand the
+ // coherence contract a fabricated `enumeration_boundary` proof
+ // (`packages/reference-contract/src/evidence/coherence.ts` rule 2 reads a
+ // measured `considered: 0` as "I enumerated the boundary and it held
+ // nothing"). Track the boundary for contacts explicitly and let the caller
+ // withhold the claim rather than overstate it.
+ const contactsBoundaryEstablished = !supportsSync || fullBoundary;
+ let groupsBoundaryEstablished = contactsBoundaryEstablished;
const bookCovered = await emitAddressBookRecordIfRequested({ book, bookCursor, requested, emitRecord, supportsSync });
const fingerprintState =
@@ -439,10 +587,20 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
return;
}
const record = contactRecord(book.url, resource, card);
- if (requested.has("contacts") && entityCursor.shouldEmit(record)) {
+ // A group vCard is a real resource in this collection, so the
+ // enumeration returns it alongside people. Emitting it as a contact
+ // creates a phantom whose `display_name` is the group's name and which
+ // counts as a covered contact. It is still SEEN (it belongs in
+ // `seenCards`, where the group derivation and the group anchor both
+ // read it), and it still counts as enumerated — it simply is not a
+ // person, so it must not enter the `contacts` stream.
+ const isGroup = isGroupVCard(card);
+ if (!isGroup && requested.has("contacts") && entityCursor.shouldEmit(record)) {
await emitRecord("contacts", record);
}
- contactCount += 1;
+ if (!isGroup) {
+ contactCount += 1;
+ }
seenCards.push({ card, uid: String(record.id) });
};
@@ -450,6 +608,26 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
for (const resource of resolvedSyncResult.resources) {
await emitContactRecord(resource);
}
+ // A sync-collection response is only obliged to enumerate members; RFC
+ // 6578 §3.2 does not require the server to inline the `address-data` the
+ // request asked for, and iCloud returns `getetag` only. Fetch those
+ // bodies explicitly (RFC 6352 §8.7 addressbook-multiget) — treating an
+ // un-inlined member as absent is what made a populated address book
+ // report zero contacts.
+ const unfetched = await hydrateMissingBodies({
+ authHeader,
+ bookUrl: book.url,
+ emitContactRecord,
+ fetchImpl,
+ hrefs: resolvedSyncResult.hrefsMissingBodies,
+ progress,
+ trustedOrigins,
+ });
+ // A member the server enumerated but whose body it never returned must
+ // stay in the denominator: considered-but-not-covered is an honest
+ // partial, where dropping it would fabricate a complete claim.
+ resourcesEnumerated += unfetched;
+ unparseableResources += unfetched;
for (const deletedHref of resolvedSyncResult.deletedHrefs) {
if (requested.has("contacts")) {
await emitRecord("contacts", contactTombstone(book.url, deletedHref));
@@ -506,7 +684,7 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
// after the source enumeration and derivation complete successfully;
// this lets a genuine zero-group result prove coverage without turning a
// failed or unattempted scan into proof.
- const groupsEmitted = await emitContactGroupsIfRequested({
+ const { emitted: groupsEmitted, anchor: groupAnchor } = await emitContactGroupsIfRequested({
bookUrl: book.url,
boundaryEstablished: groupsBoundaryEstablished,
emit,
@@ -515,6 +693,16 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
seenCards: groupCards,
});
+ // The `contact_groups` completeness anchor. The verdict is RETURNED rather
+ // than emitted here: this scope's `emit` is deliberately narrowed to STATE
+ // messages, and widening it just to report a finding would erode a
+ // boundary that is doing useful work. The caller owns the full emit.
+ await progress("Group inventory checked against the enumerated collection", {
+ stream: "contact_groups",
+ count: groupsEmitted,
+ total: Math.max(groupAnchor.serverGroupVCards, groupsEmitted),
+ });
+
const contactsState =
(newState.contacts as Record }>) ?? {};
contactsState[bookKey] = {
@@ -537,13 +725,21 @@ async function collectAddressBook(ctx: AddressBookCollectionCtx): Promise<{
// address book still proves considered === covered === 0). When one or
// more resources failed to parse, considered > covered, so the caller
// reads a real partial instead of a fabricated complete.
+ //
+ // Group vCards are enumerated resources that are deliberately NOT
+ // contacts, so they must leave the contacts denominator too — otherwise
+ // excluding them from `contactsCovered` alone would manufacture a
+ // permanent considered > covered shortfall out of correct behaviour. They
+ // are accounted for by the `contact_groups` anchor above instead.
return {
- contactsConsidered: resourcesEnumerated,
+ contactsBoundaryEstablished,
+ contactsConsidered: resourcesEnumerated - groupAnchor.serverGroupVCards,
contactsCovered: contactCount,
hadUnparseableResource: unparseableResources > 0,
covered: bookCovered,
groupsEmitted,
groupsBoundaryEstablished,
+ groupAnchor,
};
}
@@ -564,7 +760,7 @@ if (isMainModule(import.meta.url)) {
kind: "env",
required: [["APPLE_ID", "APPLE_ID_EMAIL"], "APPLE_APP_SPECIFIC_PASSWORD"],
},
- async collect({ state, requested, credentials, emit, emitRecord, progress }) {
+ async collect({ state, requested, credentials, emit, emitRecord, progress, collectionMode }) {
const accountEmail = credentials.APPLE_ID || credentials.APPLE_ID_EMAIL;
const appPassword = credentials.APPLE_APP_SPECIFIC_PASSWORD;
if (!(accountEmail && appPassword)) {
@@ -614,14 +810,17 @@ if (isMainModule(import.meta.url)) {
let groupsBoundaryEstablished = true;
let contactsConsidered = 0;
let contactsCovered = 0;
+ let contactsBoundaryEstablished = true;
let anyUnparseableResource = false;
for (const book of books) {
considered += 1;
const {
+ contactsBoundaryEstablished: bookContactsBoundaryEstablished,
contactsConsidered: bookContactsConsidered,
contactsCovered: bookContactsCovered,
covered: bookCovered,
+ groupAnchor,
groupsEmitted,
groupsBoundaryEstablished: bookGroupsBoundaryEstablished,
hadUnparseableResource,
@@ -629,6 +828,7 @@ if (isMainModule(import.meta.url)) {
book,
bookCursor,
authHeader,
+ collectionMode,
fetchImpl,
trustedOrigins,
state,
@@ -641,17 +841,48 @@ if (isMainModule(import.meta.url)) {
if (bookCovered) {
covered += 1;
}
+ // The completeness anchor for this book's groups. `short` is the
+ // case the connector was previously blind to: the server enumerated
+ // group vCards that never became records. It is reported as a
+ // SKIP_RESULT here, where the full `emit` is in scope.
+ const groupVerdict = groupAnchorVerdict(groupAnchor);
+ if (groupVerdict.status === "short") {
+ await emit({
+ type: "SKIP_RESULT",
+ stream: "contact_groups",
+ reason: "group_inventory_short",
+ message: "The address book holds groups this run did not record",
+ diagnostics: {
+ considered: groupVerdict.considered,
+ covered: groupVerdict.covered,
+ missing: groupVerdict.missing,
+ },
+ });
+ }
// deriveGroups has no drop/filter path: every derived group is
// unconditionally emitted, so considered === covered === the exact
// count emitted for this book (including a genuine zero-group book).
+ // The anchor above is what checks that claim against the server's
+ // own enumeration rather than trusting it.
groupsConsidered += groupsEmitted;
groupsBoundaryEstablished = groupsBoundaryEstablished && bookGroupsBoundaryEstablished;
+ contactsBoundaryEstablished = contactsBoundaryEstablished && bookContactsBoundaryEstablished;
contactsConsidered += bookContactsConsidered;
contactsCovered += bookContactsCovered;
anyUnparseableResource = anyUnparseableResource || hadUnparseableResource;
}
- if (requested.has("contacts")) {
+ // Withhold the contacts coverage claim entirely when no book established
+ // a full boundary this run. An incremental sync-collection delta is a
+ // change feed, not an inventory: its `considered` is the number of
+ // CHANGED resources, so a quiet run would otherwise emit
+ // `considered: 0, covered: 0` and be read as a proven-empty address
+ // book. Emitting nothing leaves the stream honestly unproven (the
+ // coherence contract's `checkpoint_only`/`no_proof_strategy` -> axis
+ // `unknown`) instead of falsely complete. A run that DOES establish the
+ // boundary — initial sync, stale-token resync, or the non-incremental
+ // fallback — still proves a genuine zero exactly as before.
+ if (requested.has("contacts") && contactsBoundaryEstablished) {
// `considered` counts every resource the server enumerated;
// `covered` counts only the ones this run successfully parsed and
// accounted for (emitted, or suppressed as unchanged by the
diff --git a/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts b/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts
index 2d30d045a..807f8904f 100644
--- a/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts
+++ b/packages/polyfill-connectors/connectors/apple_contacts/integration.test.ts
@@ -231,6 +231,38 @@ test("apple_contacts integration: a second run emits a tombstone for a server-si
}
});
+test("apple_contacts integration: an initial sync of a genuinely empty address book proves verified-empty coverage", async () => {
+ const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD });
+ try {
+ // No contacts at all, and no prior cursor: this run establishes the full
+ // boundary, so `considered: 0` is a real measurement ("I enumerated the
+ // address book and it held nothing") rather than a quiet change feed.
+ // Withholding the claim here would turn a legitimately-empty required
+ // stream into a permanent `unknown`, which is its own failure.
+ const only = await runConnectorProtocolSubprocess({
+ cwd: CWD,
+ entrypoint: ENTRYPOINT,
+ start: startMessage(),
+ env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin },
+ });
+
+ const done = only.messages.findLast((m) => m.type === "DONE");
+ assert.ok(done && done.type === "DONE");
+ assert.equal(done.status, "succeeded");
+ assert.equal(recordsOf(only.messages, "contacts").length, 0);
+
+ const contactsCoverage = only.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts");
+ assert.ok(
+ contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE",
+ "a boundary-establishing run must still emit contacts coverage for a genuine zero"
+ );
+ assert.equal(contactsCoverage.considered, 0);
+ assert.equal(contactsCoverage.covered, 0);
+ } finally {
+ await server.close();
+ }
+});
+
test("apple_contacts integration: an unchanged incremental sync re-enumerates the group inventory", async () => {
const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD });
try {
@@ -268,6 +300,76 @@ test("apple_contacts integration: an unchanged incremental sync re-enumerates th
assert.ok(groupsCoverage && groupsCoverage.type === "DETAIL_COVERAGE");
assert.equal(groupsCoverage.considered, 1);
assert.equal(groupsCoverage.covered, 1);
+
+ // The quiet incremental run enumerated nothing, so it must not claim a
+ // contacts boundary. This is the shape observed in production on
+ // cin_d344ba53d6d95c7dd343393d: a stored sync_token with an empty
+ // fingerprint map, replaying as `considered: 0` forever.
+ const contactsCoverage = second.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts");
+ assert.equal(
+ contactsCoverage,
+ undefined,
+ "an unchanged incremental sync must not claim a proven contacts boundary"
+ );
+ } finally {
+ await server.close();
+ }
+});
+
+test("apple_contacts integration: a full_refresh run re-enumerates and proves contacts coverage despite a stored sync token", async () => {
+ const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD });
+ try {
+ server.contacts.set("erin", {
+ uid: "erin",
+ href: "/addressbooks/owner/card/erin.vcf",
+ vcard: buildVCard({ uid: "erin", fn: "Erin Example", categories: ["Friends"] }),
+ });
+ server.contacts.set("frank", {
+ uid: "frank",
+ href: "/addressbooks/owner/card/frank.vcf",
+ vcard: buildVCard({ uid: "frank", fn: "Frank Example" }),
+ });
+
+ const first = await runConnectorProtocolSubprocess({
+ cwd: CWD,
+ entrypoint: ENTRYPOINT,
+ start: startMessage(),
+ env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin },
+ });
+ const firstState = first.messages.findLast(
+ (m): m is Extract => m.type === "STATE" && m.stream === "contacts"
+ );
+ assert.ok(firstState);
+
+ // Same stored-token starting point as the quiet-incremental test above —
+ // which asserts this run emits NO contacts coverage. The only difference
+ // here is the owner-requested `collection_mode: "full_refresh"`, so this
+ // pins the bypass itself rather than any change in the source data.
+ const refreshed = await runConnectorProtocolSubprocess({
+ cwd: CWD,
+ entrypoint: ENTRYPOINT,
+ start: { ...startMessage({ contacts: firstState.cursor }), collection_mode: "full_refresh" },
+ env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin },
+ });
+
+ const contactsCoverage = refreshed.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts");
+ assert.ok(
+ contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE",
+ "a full_refresh run must re-establish the contacts boundary and claim coverage"
+ );
+ // The claim must be the real inventory (both contacts), not a `considered: 0`
+ // artifact of a change feed — that distinction is the whole point.
+ assert.equal(contactsCoverage.considered, 2);
+ assert.equal(contactsCoverage.covered, 2);
+
+ // The refreshed run must still write a usable token forward, so the next
+ // ordinary run can resume incrementally rather than re-walking forever.
+ const refreshedState = refreshed.messages.findLast(
+ (m): m is Extract => m.type === "STATE" && m.stream === "contacts"
+ );
+ assert.ok(refreshedState);
+ const [bookState] = Object.values(refreshedState.cursor as Record);
+ assert.ok(bookState?.sync_token, "a full_refresh run must still persist a sync token for the next run");
} finally {
await server.close();
}
@@ -315,10 +417,20 @@ test("apple_contacts integration: incremental group requests re-enumerate a genu
assert.equal(contactRecords.filter((message) => message.op !== "delete").length, 0);
assert.equal(recordsOf(second.messages, "contact_groups").length, 0);
+ // The incremental contact delta is not a CONTACT boundary either. This run
+ // saw one deletion and zero live resources, which is a change feed, not an
+ // enumeration of the address book. Emitting `considered: 0, covered: 0`
+ // here would hand the coherence contract a fabricated
+ // `enumeration_boundary` proof and paint a required stream green off a
+ // quiet sync. The honest move is to emit no contacts coverage at all and
+ // let the stream read `unknown` until a run actually establishes the
+ // boundary.
const contactsCoverage = second.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts");
- assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE");
- assert.equal(contactsCoverage.considered, 0);
- assert.equal(contactsCoverage.covered, 0);
+ assert.equal(
+ contactsCoverage,
+ undefined,
+ "an incremental contact delta must not claim a proven-empty contacts inventory"
+ );
// The incremental contact delta is not a group boundary. The connector
// must obtain a full snapshot before proving that the now-empty group
@@ -508,3 +620,183 @@ test("apple_contacts integration: never logs credentials or vCard field values i
await server.close();
}
});
+
+/**
+ * Live-shape regression (probe against p196-contacts.icloud.com, 2026-08-19).
+ *
+ * iCloud answers a `sync-collection` REPORT by enumerating each member with
+ * `getetag` ONLY — it does not inline the `address-data` the request asked
+ * for — and it also lists the collection's own href beside its members. That
+ * is RFC-legal (RFC 6578 §3.2 requires enumeration, not inlining), so the
+ * client must fetch bodies via `addressbook-multiget` (RFC 6352 §8.7).
+ *
+ * Before this was handled, the connector dropped every member that arrived
+ * without an inlined body, and a populated address book reported a *measured*
+ * zero — the most dangerous possible failure, because a proven zero is
+ * indistinguishable downstream from a genuinely empty account.
+ */
+test("apple_contacts integration: collects contacts when sync-collection enumerates members without inlining address-data", async () => {
+ const server = await startFakeCardDavServer({
+ username: USERNAME,
+ password: PASSWORD,
+ syncCollectionOmitsAddressData: true,
+ });
+ try {
+ server.contacts.set("frank", {
+ uid: "frank",
+ href: "/addressbooks/owner/card/frank.vcf",
+ vcard: buildVCard({ uid: "frank", fn: "Frank Example", email: "frank@example.com", categories: ["Family"] }),
+ });
+
+ const result = await runConnectorProtocolSubprocess({
+ cwd: CWD,
+ entrypoint: ENTRYPOINT,
+ start: startMessage(),
+ env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin },
+ });
+
+ const done = result.messages.findLast((m) => m.type === "DONE");
+ assert.ok(done && done.type === "DONE");
+ assert.equal(done.status, "succeeded");
+
+ const contacts = recordsOf(result.messages, "contacts");
+ assert.equal(contacts.length, 1, "the enumerated member's body must be fetched, not dropped");
+ assert.equal(contacts[0]?.display_name, "Frank Example");
+
+ // The collection's own href is enumerated alongside its members; it must
+ // not be mistaken for a contact resource.
+ assert.equal(
+ contacts.some((c) => String(c.id).endsWith("/addressbooks/owner/card")),
+ false,
+ "the collection itself must not be emitted as a contact"
+ );
+
+ const contactsCoverage = result.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts");
+ assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE");
+ assert.equal(contactsCoverage.considered, 1);
+ assert.equal(contactsCoverage.covered, 1);
+
+ // Groups derive from CATEGORIES, which only exist once bodies are hydrated.
+ const groups = recordsOf(result.messages, "contact_groups");
+ assert.deepEqual(
+ groups.map((g) => g.name),
+ ["Family"]
+ );
+ } finally {
+ await server.close();
+ }
+});
+
+test("apple_contacts integration: an enumerated member whose body never arrives is an honest partial, not a proven zero", async () => {
+ const server = await startFakeCardDavServer({
+ username: USERNAME,
+ password: PASSWORD,
+ syncCollectionOmitsAddressData: true,
+ multigetReturnsNoBodies: true,
+ });
+ try {
+ server.contacts.set("ghost", {
+ uid: "ghost",
+ href: "/addressbooks/owner/card/ghost.vcf",
+ vcard: buildVCard({ uid: "ghost", fn: "Ghost Example" }),
+ });
+ const result = await runConnectorProtocolSubprocess({
+ cwd: CWD,
+ entrypoint: ENTRYPOINT,
+ start: startMessage(),
+ env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin },
+ });
+
+ const done = result.messages.findLast((m) => m.type === "DONE");
+ assert.ok(done && done.type === "DONE");
+ assert.equal(done.status, "succeeded");
+
+ assert.equal(recordsOf(result.messages, "contacts").length, 0);
+
+ const contactsCoverage = result.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts");
+ assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE");
+ assert.equal(contactsCoverage.considered, 1, "the member the server enumerated must stay in the denominator");
+ assert.equal(contactsCoverage.covered, 0);
+ assert.equal(
+ contactsCoverage.considered > contactsCoverage.covered,
+ true,
+ "an unfetchable member must read as an honest partial, never a proven zero"
+ );
+ } finally {
+ await server.close();
+ }
+});
+
+test("apple_contacts integration: an iCloud group vCard becomes a group, not a phantom contact", async () => {
+ // The defect this closes: iCloud stores each group as its own vCard
+ // resource marked `X-ADDRESSBOOKSERVER-KIND:group`, but the connector read
+ // only the vCard-standard CATEGORIES property. So the group's own resource
+ // was emitted AS A CONTACT — a phantom whose display_name is the group's
+ // name, counted as a covered contact — while `contact_groups`, a REQUIRED
+ // stream, stayed empty. The fake server could not synthesize this shape
+ // until now, which is why the whole suite passed while the connector was
+ // blind.
+ const server = await startFakeCardDavServer({ username: USERNAME, password: PASSWORD });
+ try {
+ server.contacts.set("alice", {
+ uid: "alice",
+ href: "/addressbooks/owner/card/alice.vcf",
+ vcard: buildVCard({ uid: "alice", fn: "Alice Example" }),
+ });
+ server.contacts.set("bob", {
+ uid: "bob",
+ href: "/addressbooks/owner/card/bob.vcf",
+ vcard: buildVCard({ uid: "bob", fn: "Bob Example" }),
+ });
+ server.contacts.set("family-group", {
+ uid: "family-group",
+ href: "/addressbooks/owner/card/family-group.vcf",
+ vcard: buildVCard({ uid: "family-group", fn: "Family", groupMemberUids: ["alice", "bob"] }),
+ });
+
+ const result = await runConnectorProtocolSubprocess({
+ cwd: CWD,
+ entrypoint: ENTRYPOINT,
+ start: startMessage(),
+ env: { APPLE_ID: USERNAME, APPLE_APP_SPECIFIC_PASSWORD: PASSWORD, APPLE_CARDDAV_ORIGIN: server.origin },
+ });
+
+ const done = result.messages.findLast((m) => m.type === "DONE");
+ assert.ok(done && done.type === "DONE");
+ assert.equal(done.status, "succeeded");
+
+ const contacts = recordsOf(result.messages, "contacts");
+ const contactNames = contacts.map((c) => String(c.display_name)).sort((a, b) => a.localeCompare(b));
+ assert.deepEqual(contactNames, ["Alice Example", "Bob Example"]);
+ assert.equal(
+ contacts.some((c) => c.display_name === "Family"),
+ false,
+ "the group vCard leaked into the contacts stream as a phantom contact"
+ );
+
+ // The group the connector was previously blind to must now be a record,
+ // with the membership the SERVER stated rather than one inferred from
+ // contact bodies.
+ const groups = recordsOf(result.messages, "contact_groups");
+ assert.deepEqual(
+ groups.map((g) => g.name),
+ ["Family"]
+ );
+ const memberUids = groups[0]?.member_uids as string[] | undefined;
+ assert.ok(memberUids);
+ assert.deepEqual(
+ [...memberUids].sort((a, b) => a.localeCompare(b)),
+ ["alice", "bob"]
+ );
+
+ // The group resource must leave the contacts denominator too: counting
+ // it as considered-but-not-covered would manufacture a permanent
+ // shortfall out of correct behaviour.
+ const contactsCoverage = result.messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "contacts");
+ assert.ok(contactsCoverage && contactsCoverage.type === "DETAIL_COVERAGE");
+ assert.equal(contactsCoverage.considered, 2);
+ assert.equal(contactsCoverage.covered, 2);
+ } finally {
+ await server.close();
+ }
+});
diff --git a/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts b/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts
index 94bbb97be..afda47a1f 100644
--- a/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts
+++ b/packages/polyfill-connectors/connectors/apple_contacts/test-carddav-server.ts
@@ -23,11 +23,42 @@ export interface FakeServerOptions {
/** When true, REPORT sync-collection returns 501 (unsupported); the
* server still answers addressbook-query so the fallback path works. */
disableSyncCollection?: boolean;
+ /**
+ * When true, sync-collection obeys RFC 6578 §3.2 faithfully: a request
+ * carrying a NON-EMPTY `` returns only the members that
+ * changed since that token — for a quiet collection, an EMPTY response with
+ * a fresh token — while a request with an empty token still returns every
+ * member. The default (false) returns all contacts for any token, which is
+ * a permissive fiction convenient for most tests.
+ *
+ * Set this to reproduce the class of defect where an empty CHANGE FEED is
+ * mistaken for an empty INVENTORY.
+ */
+ enforceRfc6578IncrementalSemantics?: boolean;
+ /** When true, `addressbook-multiget` answers 404 for every requested href.
+ * Models a member the server enumerated but whose body it will not hand
+ * over, so the client must report considered-but-not-covered rather than a
+ * proven zero. */
+ multigetReturnsNoBodies?: boolean;
password: string;
/** When true, /.well-known/carddav redirects to a second listener
* simulating iCloud's regional-host resolution, instead of a
* same-origin redirect. */
regionalHost?: boolean;
+ /**
+ * When true, the sync-collection REPORT enumerates members with `getetag`
+ * ONLY — it does not inline the requested `address-data` — and additionally
+ * reports the collection's own href alongside its members. This is the
+ * behavior observed live against iCloud (p196-contacts.icloud.com, probe
+ * 2026-08-19) and it is RFC-legal: RFC 6578 §3.2 obliges the server to
+ * enumerate changed members, not to inline arbitrary properties. Bodies are
+ * then only obtainable via `addressbook-multiget` (RFC 6352 §8.7).
+ *
+ * Set this to reproduce the class of defect where a client drops every
+ * enumerated member that arrived without an inlined body, turning a
+ * populated address book into a measured zero.
+ */
+ syncCollectionOmitsAddressData?: boolean;
/** When true, the sync-collection REPORT (and addressbook-query REPORT)
* respond with a 302 to an untrusted, unrelated origin instead of the
* normal multistatus body — models a compromised/misconfigured server
@@ -57,6 +88,8 @@ export interface FakeCardDavServer {
url: (path: string) => string;
}
+const TRAILING_SLASH_RE = /\/$/;
+const ABSOLUTE_HREF_RE = /^https?:\/\//i;
const NS_D = "DAV:";
const NS_CS = "http://calendarserver.org/ns/";
const PRINCIPAL_PATH = "/principals/owner/";
@@ -71,6 +104,19 @@ function multistatus(inner: string): string {
return `${inner}`;
}
+/**
+ * Read the `` value out of a sync-collection REPORT body.
+ * RFC 6578 §3.2 distinguishes an EMPTY token (send me the whole collection)
+ * from a non-empty one (send me only what changed since then), so the fake
+ * server needs the actual value, not merely its presence.
+ */
+const SYNC_TOKEN_RE = /<(?:[A-Za-z0-9]+:)?sync-token>([\s\S]*?)<\/(?:[A-Za-z0-9]+:)?sync-token>/;
+
+function extractSyncTokenFromRequest(body: string): string {
+ const match = SYNC_TOKEN_RE.exec(body);
+ return match?.[1]?.trim() ?? "";
+}
+
function readBody(req: IncomingMessage): Promise {
return new Promise((resolve, reject) => {
let data = "";
@@ -101,9 +147,12 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis
username,
password,
disableSyncCollection = false,
+ enforceRfc6578IncrementalSemantics = false,
regionalHost = false,
wellKnownAnswersInlineWithoutPrincipal = false,
syncReportRedirectsToUnsafeOrigin = false,
+ syncCollectionOmitsAddressData = false,
+ multigetReturnsNoBodies = false,
} = options;
const contacts = new Map();
const deletedHrefs = new Set();
@@ -165,7 +214,51 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis
)
.join("");
- const respondSyncCollection = (res: ServerResponse): void => {
+ /** Members enumerated with `getetag` only — no inlined `address-data`. */
+ const etagOnlyResponseBlocks = (): string =>
+ [...contacts.values()]
+ .map(
+ (c) =>
+ `${c.href}"${c.uid}-${String(changeCounter)}"HTTP/1.1 200 OK`
+ )
+ .join("");
+
+ const HREF_RE = /<[^:>]*:?href[^>]*>([\s\S]*?)<\/[^:>]*:?href>/gi;
+
+ /** RFC 6352 §8.7 addressbook-multiget: return bodies for exactly the
+ * requested hrefs, and 404 any href the collection does not hold. */
+ const respondAddressbookMultiget = (res: ServerResponse, body: string): void => {
+ const requestedHrefs = [...body.matchAll(HREF_RE)].map((m) => (m[1] ?? "").trim());
+ // iCloud answers 400 to an absolute inside a multiget and 207 to
+ // the path form (probe against p196-contacts.icloud.com, 2026-08-19).
+ // Enforce that here so a regression to absolute hrefs fails in tests
+ // instead of only against the real provider.
+ if (requestedHrefs.some((href) => ABSOLUTE_HREF_RE.test(href))) {
+ res.writeHead(400, { "Content-Type": "text/plain" });
+ res.end("Bad Request");
+ return;
+ }
+ const blocks = requestedHrefs
+ .map((href) => {
+ const path = (() => {
+ try {
+ return new URL(href, "http://placeholder.invalid").pathname;
+ } catch {
+ return href;
+ }
+ })();
+ const contact = multigetReturnsNoBodies ? undefined : [...contacts.values()].find((c) => c.href === path);
+ if (!contact) {
+ return `${href}HTTP/1.1 404 Not Found`;
+ }
+ return `${contact.href}"${contact.uid}-${String(changeCounter)}"${xmlEscape(contact.vcard)}HTTP/1.1 200 OK`;
+ })
+ .join("");
+ res.writeHead(207, { "Content-Type": "application/xml" });
+ res.end(multistatus(blocks));
+ };
+
+ const respondSyncCollection = (res: ServerResponse, requestedSyncToken: string): void => {
if (syncReportRedirectsToUnsafeOrigin) {
res.writeHead(302, { Location: "https://attacker.example/steal-carddav-creds" });
res.end();
@@ -180,7 +273,19 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis
const deleted = [...deletedHrefs]
.map((href) => `${href}HTTP/1.1 404 Not Found`)
.join("");
- const responseBody = multistatus(`${contactResponseBlocks()}${deleted}${newToken}`);
+ // RFC 6578 §3.2: a non-empty sync-token requests only what changed since
+ // that token. Nothing has changed in this fixture, so the change feed is
+ // empty — while the collection itself still holds every contact.
+ const quiet = enforceRfc6578IncrementalSemantics && requestedSyncToken.length > 0;
+ const populatedMembers = syncCollectionOmitsAddressData ? etagOnlyResponseBlocks() : contactResponseBlocks();
+ const members = quiet ? "" : populatedMembers;
+ // iCloud reports the collection's own href (without a trailing slash)
+ // beside its members. A client must not mistake it for a contact.
+ const collectionBlock =
+ syncCollectionOmitsAddressData && !quiet
+ ? `${BOOK_PATH.replace(TRAILING_SLASH_RE, "")}collection-${String(changeCounter)}HTTP/1.1 200 OK`
+ : "";
+ const responseBody = multistatus(`${collectionBlock}${members}${deleted}${newToken}`);
res.writeHead(207, { "Content-Type": "application/xml" });
res.end(responseBody);
};
@@ -197,7 +302,7 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis
interface Route {
match: (req: IncomingMessage, url: string, body: string) => boolean;
- respond: (req: IncomingMessage, res: ServerResponse, thisOrigin: () => string) => void;
+ respond: (req: IncomingMessage, res: ServerResponse, thisOrigin: () => string, body: string) => void;
}
const routes: Route[] = [
@@ -228,7 +333,11 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis
},
{
match: (req, url, body) => req.method === "REPORT" && url === BOOK_PATH && body.includes("sync-collection"),
- respond: (_req, res) => respondSyncCollection(res),
+ respond: (_req, res, _origin, body) => respondSyncCollection(res, extractSyncTokenFromRequest(body ?? "")),
+ },
+ {
+ match: (req, url, body) => req.method === "REPORT" && url === BOOK_PATH && body.includes("addressbook-multiget"),
+ respond: (_req, res, _origin, body) => respondAddressbookMultiget(res, body ?? ""),
},
{
match: (req, url, body) => req.method === "REPORT" && url === BOOK_PATH && body.includes("addressbook-query"),
@@ -252,7 +361,7 @@ export async function startFakeCardDavServer(options: FakeServerOptions): Promis
const body = await readBody(req);
const route = routes.find((r) => r.match(req, url, body));
if (route) {
- route.respond(req, res, thisOrigin);
+ route.respond(req, res, thisOrigin, body);
return;
}
@@ -326,6 +435,12 @@ export function buildVCard(fields: {
categories?: string[];
email?: string;
fn: string;
+ /** Member UIDs. Presence makes this a group vCard in Apple's wire shape
+ * (`X-ADDRESSBOOKSERVER-KIND:group`), which is how iCloud actually stores
+ * groups. Without this the fixture could only synthesize CATEGORIES, so a
+ * connector blind to group vCards still passed every test — the fixture
+ * gap that let the `contact_groups` defect hide. */
+ groupMemberUids?: string[];
photo?: { base64: string; mediaType: string };
uid: string;
}): string {
@@ -336,6 +451,12 @@ export function buildVCard(fields: {
if (fields.categories?.length) {
lines.push(`CATEGORIES:${fields.categories.join(",")}`);
}
+ if (fields.groupMemberUids) {
+ lines.push("X-ADDRESSBOOKSERVER-KIND:group");
+ for (const member of fields.groupMemberUids) {
+ lines.push(`X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:${member}`);
+ }
+ }
if (fields.photo) {
lines.push(`PHOTO;ENCODING=b;TYPE=${fields.photo.mediaType.toUpperCase()}:${fields.photo.base64}`);
}
diff --git a/packages/polyfill-connectors/connectors/chase/cursor-staleness.test.ts b/packages/polyfill-connectors/connectors/chase/cursor-staleness.test.ts
new file mode 100644
index 000000000..bb847849d
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/chase/cursor-staleness.test.ts
@@ -0,0 +1,137 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Tests for the two cursor defects that let Chase collection stall silently.
+ *
+ * 1. `chooseActivity` picked "Since last statement" whenever a cursor
+ * existed, with no check that the cursor was recent enough for that
+ * option's window to reach it. Chase's "Since last statement" covers the
+ * current cycle only (~30 days), so once a cursor fell behind, every
+ * scheduled run downloaded a QFX, saw nothing older, and reported
+ * success — no run could ever reach back across the gap.
+ *
+ * 2. `per_account` cursors were carried forward wholesale, so an account
+ * that disappeared from `discoverAccounts()` kept its cursor forever and
+ * was skipped silently with no gap emitted.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { chooseActivity, cursorAgeInDays, prunePerAccountCursors } from "./parsers.ts";
+import type { TransactionCursor, TransactionsStateShape } from "./types.ts";
+
+const ACCOUNT = "acct-1";
+const RUN_DATE = "2026-08-20";
+
+function stateWithCursor(maxSeenDate: string): TransactionsStateShape {
+ return { per_account: { [ACCOUNT]: { max_seen_date: maxSeenDate } as TransactionCursor } };
+}
+
+// ─── cursorAgeInDays ─────────────────────────────────────────────────────
+
+test("cursorAgeInDays measures whole days between cursor and run date", () => {
+ assert.equal(cursorAgeInDays("2026-08-10", RUN_DATE), 10);
+ assert.equal(cursorAgeInDays("2026-08-20", RUN_DATE), 0);
+});
+
+test("cursorAgeInDays tolerates a full ISO timestamp", () => {
+ assert.equal(cursorAgeInDays("2026-08-10T12:34:56Z", RUN_DATE), 10);
+});
+
+test("cursorAgeInDays returns null for an unparseable date", () => {
+ // Must NOT read as fresh — the caller takes the safe path on null.
+ assert.equal(cursorAgeInDays("not-a-date", RUN_DATE), null);
+ assert.equal(cursorAgeInDays("", RUN_DATE), null);
+});
+
+// ─── chooseActivity: an explicit scope still wins ────────────────────────
+
+test("chooseActivity honours an explicit time_range over any cursor", () => {
+ const requested = new Map([["transactions", { time_range: { since: "2026-05-01", until: "2026-07-31" } }]]);
+ const choice = chooseActivity(requested, stateWithCursor("2026-08-19"), "transactions", ACCOUNT, RUN_DATE);
+ assert.equal(choice.activity, "date_range");
+ assert.deepEqual(choice.dateRange, { from: "2026-05-01", to: "2026-07-31" });
+});
+
+test("chooseActivity bootstraps with 'all' when there is no cursor", () => {
+ const choice = chooseActivity(new Map(), {}, "transactions", ACCOUNT, RUN_DATE);
+ assert.equal(choice.activity, "all");
+});
+
+// ─── chooseActivity: the staleness fix ───────────────────────────────────
+
+test("chooseActivity uses 'since_last_statement' for a FRESH cursor", () => {
+ // The incremental fast path must still work — this is the common case.
+ const choice = chooseActivity(new Map(), stateWithCursor("2026-08-19"), "transactions", ACCOUNT, RUN_DATE);
+ assert.equal(choice.activity, "since_last_statement");
+});
+
+test("chooseActivity still uses 'since_last_statement' at the freshness edge", () => {
+ // 25 days: the last age that is still considered fresh.
+ const choice = chooseActivity(new Map(), stateWithCursor("2026-07-26"), "transactions", ACCOUNT, RUN_DATE);
+ assert.equal(choice.activity, "since_last_statement");
+});
+
+test("chooseActivity falls back to a bounded date_range for a STALE cursor", () => {
+ // This is the defect: a 90-day-old cursor previously still produced
+ // "since_last_statement", whose window could not reach it, so the gap
+ // between them was never re-downloaded by any scheduled run.
+ const choice = chooseActivity(new Map(), stateWithCursor("2026-05-22"), "transactions", ACCOUNT, RUN_DATE);
+ assert.equal(choice.activity, "date_range");
+ assert.deepEqual(choice.dateRange, { from: "2026-05-22", to: RUN_DATE });
+});
+
+test("chooseActivity date_range starts AT the cursor, keeping a safe overlap", () => {
+ const choice = chooseActivity(new Map(), stateWithCursor("2026-01-15"), "transactions", ACCOUNT, RUN_DATE);
+ // Starting at (not after) the cursor re-sees one day; stable transaction
+ // ids make that a suppression, not a duplicate.
+ assert.equal(choice.dateRange?.from, "2026-01-15");
+});
+
+test("chooseActivity treats an unparseable cursor as unprovable, not fresh", () => {
+ const choice = chooseActivity(new Map(), stateWithCursor("garbage"), "transactions", ACCOUNT, RUN_DATE);
+ assert.equal(choice.activity, "date_range");
+});
+
+test("chooseActivity treats a future-dated cursor as unprovable", () => {
+ // Clock skew must not be read as maximally fresh.
+ const choice = chooseActivity(new Map(), stateWithCursor("2027-01-01"), "transactions", ACCOUNT, RUN_DATE);
+ assert.equal(choice.activity, "date_range");
+});
+
+// ─── prunePerAccountCursors ──────────────────────────────────────────────
+
+const CURSORS: Record = {
+ "acct-1": { max_seen_date: "2026-08-19" } as TransactionCursor,
+ "acct-2": { max_seen_date: "2026-08-18" } as TransactionCursor,
+ "acct-gone": { max_seen_date: "2025-01-01" } as TransactionCursor,
+};
+
+test("prunePerAccountCursors drops cursors for undiscovered accounts", () => {
+ const { kept, dropped } = prunePerAccountCursors(CURSORS, new Set(["acct-1", "acct-2"]));
+ assert.deepEqual(Object.keys(kept).sort(), ["acct-1", "acct-2"]);
+ assert.deepEqual(dropped, ["acct-gone"]);
+});
+
+test("prunePerAccountCursors keeps every cursor when all accounts are present", () => {
+ const { kept, dropped } = prunePerAccountCursors(CURSORS, new Set(["acct-1", "acct-2", "acct-gone"]));
+ assert.deepEqual(Object.keys(kept).sort(), ["acct-1", "acct-2", "acct-gone"]);
+ assert.deepEqual(dropped, []);
+});
+
+test("prunePerAccountCursors reports the drop rather than swallowing it", () => {
+ // The orphaned cursor must be VISIBLE: a vanished account may be a closed
+ // account or a discovery regression, and silently dropping it hides both.
+ const { dropped } = prunePerAccountCursors(CURSORS, new Set(["acct-1"]));
+ assert.deepEqual(
+ [...dropped].sort((a, b) => a.localeCompare(b)),
+ ["acct-2", "acct-gone"]
+ );
+});
+
+test("prunePerAccountCursors handles an empty cursor map", () => {
+ const { kept, dropped } = prunePerAccountCursors({}, new Set(["acct-1"]));
+ assert.deepEqual(kept, {});
+ assert.deepEqual(dropped, []);
+});
diff --git a/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts b/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts
index 6987562a6..67a394708 100644
--- a/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts
+++ b/packages/polyfill-connectors/connectors/chase/detail-gap-recovery.test.ts
@@ -2,31 +2,43 @@
// SPDX-License-Identifier: Apache-2.0
/**
- * Chase served-gap recovery: DETAIL_GAP_RECOVERED for reached account gaps.
+ * Chase served-gap recovery: DETAIL_GAP_RECOVERED for reached account gaps
+ * and for statement gaps whose PDF is hydrated again.
*
* When the runtime serves the Chase connector a pending per-account
* `DETAIL_GAP` at START (`ctx.detailGaps`), Chase re-enumerates and re-downloads
* every in-scope account anyway, so the served account is hydrated by the normal
- * QFX pass. The missing step this suite pins is the acknowledgement: on a
- * successful (or source-limited no-activity) outcome for a served account, the
- * connector emits `DETAIL_GAP_RECOVERED` with the served `gap_id`, so the
- * durable `connector_detail_gaps` row moves to `recovered` instead of being
- * reset to `pending` by runtime cleanup.
+ * QFX pass. On a successful (or source-limited no-activity) outcome for a
+ * served account, the connector emits `DETAIL_GAP_RECOVERED` with the served
+ * `gap_id`, so the durable `connector_detail_gaps` row moves to `recovered`
+ * instead of being reset to `pending` by runtime cleanup.
+ *
+ * Chase `statements` has the identical shape: a failed PDF download opens a
+ * retryable `DETAIL_GAP` (see `processStatementRow`), and only a later run
+ * that hydrates that statement's PDF again can close it — via
+ * `recoverServedStatementGaps`, mirroring the account-gap path above.
*
* These exercise the exported recovery helpers directly through the recording
* harness (the same pattern detail-coverage.test.ts uses) so they validate the
- * emitted protocol messages without driving Playwright. The account is treated
+ * emitted protocol messages without driving Playwright. An account is treated
* as reached exactly when `emitTransactionsDetailCoverage` would count it as a
- * `hydrated_key`, keeping recovery and coverage in lockstep.
+ * `hydrated_key`; a statement is treated as recoverable exactly when
+ * `emitStatementDetailCoverage` counts it in `hydratedKeys` — keeping recovery
+ * and coverage in lockstep for both streams.
*
* Safety pinned here (lose-no-data):
* - only a served gap whose account was reached is recovered;
+ * - only a served statement gap whose PDF is hydrated again this run is
+ * recovered — a considered-but-still-missing statement never is, even
+ * though it appears in the same run's `requiredKeys`;
* - a served gap whose account still FAILS this run is never recovered
* (it stays on the DETAIL_GAP re-emit path → runtime resets to pending);
* - a served gap whose account is not enumerated this run is never recovered;
* - the recovery gap_id is always the served gap_id, never synthesized;
- * - only account-level chase.account transaction gaps are recovered — a
- * foreign or malformed served gap is ignored.
+ * - only account-level chase.account transaction gaps are recovered by
+ * `buildServedAccountGapLookup`, and only chase.statement statements gaps
+ * by `buildServedStatementGapLookup` — a foreign, wrong-stream, or
+ * malformed served gap is ignored by both.
*/
import assert from "node:assert/strict";
import { test } from "node:test";
@@ -35,10 +47,14 @@ import { makeRecordingEmit } from "../../src/test-harness.ts";
import {
type AccountDetailOutcome,
buildServedAccountGapLookup,
+ buildServedStatementGapLookup,
classifyNoActivityOutcome,
type EmitDeps,
emitNoActivityProgress,
+ emitStatementDetailCoverage,
recoverServedAccountGaps,
+ recoverServedStatementGaps,
+ type StatementDetailOutcome,
} from "./index.ts";
import { validateRecord } from "./schemas.ts";
import type { TransactionCursor, TransactionsStateShape } from "./types.ts";
@@ -50,8 +66,10 @@ interface HarnessOverrides {
requestedStreams?: readonly StreamScope[];
resFilters?: Map | null>;
servedAccountGaps?: ReadonlyMap;
+ servedStatementGaps?: ReadonlyMap;
txState?: TransactionsStateShape;
wantsAccounts?: boolean;
+ wantsStatements?: boolean;
wantsTransactions?: boolean;
}
@@ -78,12 +96,13 @@ function makeHarness(overrides: HarnessOverrides = {}): Harness {
requested,
resFilters: overrides.resFilters ?? new Map(),
servedAccountGaps: overrides.servedAccountGaps,
+ servedStatementGaps: overrides.servedStatementGaps,
tmpDir: "/tmp/pdpp-chase-test-noop",
txState: overrides.txState ?? {},
wantsAccounts: overrides.wantsAccounts ?? true,
wantsBalances: true,
wantsCurrentActivity: false,
- wantsStatements: false,
+ wantsStatements: overrides.wantsStatements ?? false,
wantsTransactions: overrides.wantsTransactions ?? true,
};
return { deps, messages: harness.protocolMessages };
@@ -146,7 +165,8 @@ test("buildServedAccountGapLookup: ignores foreign, non-transactions, or malform
status: "pending",
detail_locator: { kind: "amazon.order_detail", order_id: "O1" },
},
- // wrong stream
+ // wrong stream for THIS lookup — a statements gap must go through
+ // buildServedStatementGapLookup instead, never this one
{
gap_id: "g2",
stream: "statements",
@@ -248,3 +268,171 @@ test("recoverServedAccountGaps: undefined servedAccountGaps is a no-op (legacy/o
await recoverServedAccountGaps(deps, outcomes);
assert.deepEqual(recoveriesOf(messages), []);
});
+
+// ─── buildServedStatementGapLookup: only chase.statement gaps ────────────
+
+function servedStatementGap(statementId: string, gapId: string): DetailGapStartEntry {
+ return {
+ gap_id: gapId,
+ stream: "statements",
+ status: "pending",
+ reference_only: true,
+ record_key: statementId,
+ detail_locator: { kind: "chase.statement", statement_id: statementId },
+ };
+}
+
+test("buildServedStatementGapLookup: maps served chase.statement gaps by statement id", () => {
+ const lookup = buildServedStatementGapLookup([
+ servedStatementGap("stmt-1", "gap-1"),
+ servedStatementGap("stmt-2", "gap-2"),
+ ]);
+ assert.equal(lookup.get("stmt-1"), "gap-1");
+ assert.equal(lookup.get("stmt-2"), "gap-2");
+ assert.equal(lookup.size, 2);
+});
+
+test("buildServedStatementGapLookup: ignores foreign, non-statements, or malformed served gaps", () => {
+ const lookup = buildServedStatementGapLookup([
+ // foreign connector locator kind
+ {
+ gap_id: "g1",
+ stream: "statements",
+ status: "pending",
+ detail_locator: { kind: "amazon.order_detail", order_id: "O1" },
+ },
+ // wrong stream (a transactions gap, never recoverable by the statements path)
+ {
+ gap_id: "g2",
+ stream: "transactions",
+ status: "pending",
+ detail_locator: { kind: "chase.statement", statement_id: "stmt-9" },
+ },
+ // missing statement_id
+ { gap_id: "g3", stream: "statements", status: "pending", detail_locator: { kind: "chase.statement" } },
+ // null locator
+ { gap_id: "g4", stream: "statements", status: "pending", detail_locator: null },
+ // not pending (already recovered)
+ { ...servedStatementGap("stmt-done", "g6"), status: "recovered" as never },
+ // a valid one survives the filter
+ servedStatementGap("stmt-ok", "g5"),
+ ] as readonly DetailGapStartEntry[]);
+ assert.deepEqual([...lookup.entries()], [["stmt-ok", "g5"]]);
+});
+
+// ─── recoverServedStatementGaps / emitStatementDetailCoverage: statements close on hydration ───
+
+test("recoverServedStatementGaps: a served statement gap whose PDF is hydrated again this run is recovered", async () => {
+ const { deps, messages } = makeHarness({
+ servedStatementGaps: new Map([["stmt-1", "gap-1"]]),
+ });
+ await recoverServedStatementGaps(deps, ["stmt-1"]);
+
+ assert.deepEqual(recoveriesOf(messages), [
+ {
+ type: "DETAIL_GAP_RECOVERED",
+ reference_only: true,
+ gap_id: "gap-1",
+ stream: "statements",
+ record_key: "stmt-1",
+ },
+ ]);
+});
+
+test("recoverServedStatementGaps: a served gap whose statement is not in hydratedKeys is never recovered", async () => {
+ // Mirrors USAA's mutation-killing case: feeding requiredKeys (every
+ // considered id) instead of hydratedKeys (only proven-hydrated ids) would
+ // recover a still-missing statement's gap. hydratedKeys must be the only
+ // acceptable input.
+ const { deps, messages } = makeHarness({
+ servedStatementGaps: new Map([
+ ["stmt-have", "gap-have"],
+ ["stmt-missing", "gap-missing"],
+ ]),
+ });
+ // Only stmt-have is passed as hydrated; stmt-missing was considered
+ // (it would appear in requiredKeys) but never proven hydrated.
+ await recoverServedStatementGaps(deps, ["stmt-have"]);
+
+ assert.deepEqual(
+ recoveriesOf(messages).map((r) => r.gap_id),
+ ["gap-have"],
+ "recovery must follow hydration proof, never merely a considered/required key"
+ );
+});
+
+test("recoverServedStatementGaps: an unmatched or unserved statement id is never recovered", async () => {
+ const { deps, messages } = makeHarness({ servedStatementGaps: new Map() });
+ await recoverServedStatementGaps(deps, ["stmt-unserved"]);
+ assert.deepEqual(
+ recoveriesOf(messages),
+ [],
+ "a gap id the runtime did not serve must never be synthesized — that could close unrelated work"
+ );
+});
+
+test("recoverServedStatementGaps: undefined servedStatementGaps is a no-op (legacy/ordinary run)", async () => {
+ const { deps, messages } = makeHarness({});
+ await recoverServedStatementGaps(deps, ["stmt-1"]);
+ assert.deepEqual(recoveriesOf(messages), []);
+});
+
+const STATEMENT_OUTCOME_HYDRATED = (id: string): StatementDetailOutcome => ({ kind: "hydrated", id });
+const STATEMENT_OUTCOME_GAP = (id: string): StatementDetailOutcome => ({
+ kind: "gap",
+ id,
+ reason: "temporary_unavailable",
+ errorClass: "pdf_download_failed",
+});
+
+test("emitStatementDetailCoverage: recovers a served statement gap once its PDF is hydrated again", async () => {
+ // The exact defect this suite pins: a statement PDF download failure opens
+ // a DETAIL_GAP (see processStatementRow), and until this recovery path
+ // existed, NOTHING ever closed it — buildServedAccountGapLookup only ever
+ // recovers `stream: "transactions"`, so a served `statements` gap was
+ // filtered out as wrong-stream and stayed pending forever, even after the
+ // statement was collected on a later run.
+ const { deps, messages } = makeHarness({
+ wantsStatements: true,
+ servedStatementGaps: new Map([["stmt-recovered", "gap-recovered"]]),
+ });
+ await emitStatementDetailCoverage(deps, [STATEMENT_OUTCOME_HYDRATED("stmt-recovered")]);
+
+ assert.deepEqual(
+ recoveriesOf(messages),
+ [
+ {
+ type: "DETAIL_GAP_RECOVERED",
+ reference_only: true,
+ gap_id: "gap-recovered",
+ stream: "statements",
+ record_key: "stmt-recovered",
+ },
+ ],
+ "a statement whose PDF is hydrated again must close its served gap, or the gap stays pending forever"
+ );
+});
+
+test("emitStatementDetailCoverage: never closes a gap for a statement still missing its PDF", async () => {
+ const { deps, messages } = makeHarness({
+ wantsStatements: true,
+ servedStatementGaps: new Map([
+ ["stmt-have", "gap-have"],
+ ["stmt-missing", "gap-missing"],
+ ]),
+ });
+ await emitStatementDetailCoverage(deps, [
+ STATEMENT_OUTCOME_HYDRATED("stmt-have"),
+ STATEMENT_OUTCOME_GAP("stmt-missing"),
+ ]);
+
+ assert.deepEqual(
+ recoveriesOf(messages).map((r) => r.gap_id),
+ ["gap-have"],
+ "recovery must follow the hydration proof, never merely the fact that a gap was served"
+ );
+ assert.ok(
+ messages.some((m) => m.type === "DETAIL_GAP" && m.record_key === "stmt-missing"),
+ "the still-missing statement must keep a pending gap"
+ );
+});
diff --git a/packages/polyfill-connectors/connectors/chase/index.ts b/packages/polyfill-connectors/connectors/chase/index.ts
index 55eb096eb..7b01ff717 100644
--- a/packages/polyfill-connectors/connectors/chase/index.ts
+++ b/packages/polyfill-connectors/connectors/chase/index.ts
@@ -90,6 +90,7 @@ import {
parseDashboardAccountsDom,
parseDateDelivered,
parseStatementsListDom,
+ prunePerAccountCursors,
resolveAccountIdForRow,
sha256Hex,
shortHash,
@@ -1313,6 +1314,13 @@ export interface EmitDeps {
* durable gap moves to `recovered` instead of being reset to `pending` by
* runtime cleanup. Empty on an ordinary run with no served gaps. */
servedAccountGaps?: ReadonlyMap | undefined;
+ /** Pending `statements` PDF gaps the runtime served this run at START,
+ * keyed by statement `id` → served `gap_id`. When a served statement's
+ * PDF is hydrated again this run, the connector emits `DETAIL_GAP_RECOVERED`
+ * with the served `gap_id` so the durable gap moves to `recovered` instead
+ * of staying `pending` forever. Empty on an ordinary run with no served
+ * statement gaps. */
+ servedStatementGaps?: ReadonlyMap | undefined;
tmpDir: string;
/** Per-transaction fingerprint cursor (excludes the run-clock
* `fetched_at`). Shared across all accounts for the whole transactions
@@ -1760,6 +1768,8 @@ export async function emitStatementDetailCoverage(
}
}
+ await recoverServedStatementGaps(deps, hydratedKeys);
+
await deps.emit(
buildDetailCoverageMessage({
stream: "statements",
@@ -1773,6 +1783,82 @@ export async function emitStatementDetailCoverage(
);
}
+/**
+ * Build the `statement_id → served gap_id` lookup from the pending detail
+ * gaps the runtime served this run at START (`ctx.detailGaps`). Filtered to
+ * Chase statement-PDF gaps — the exact shape `emitStatementDetailCoverage`
+ * writes via `buildDetailGap`: `stream === "statements"`,
+ * `detail_locator.kind === "chase.statement"`, and a non-empty
+ * `statement_id`. Any other served gap (a different connector's locator, a
+ * non-statements stream, a malformed locator) is ignored so the connector can
+ * only ever recover a gap it actually understands. Sourcing the `gap_id` from
+ * the served gap — never synthesizing one — is what guarantees the connector
+ * cannot mark an unrelated or unserved gap recovered.
+ */
+export function buildServedStatementGapLookup(
+ detailGaps: readonly BrowserCollectContext["detailGaps"][number][]
+): Map {
+ const lookup = new Map();
+ for (const gap of detailGaps) {
+ if (gap.stream !== "statements" || gap.status !== "pending") {
+ continue;
+ }
+ const locator = gap.detail_locator;
+ if (locator?.kind !== "chase.statement") {
+ continue;
+ }
+ const statementId = locator.statement_id;
+ if (
+ typeof statementId !== "string" ||
+ statementId.length === 0 ||
+ typeof gap.gap_id !== "string" ||
+ !gap.gap_id
+ ) {
+ continue;
+ }
+ // First served gap per statement wins; the runtime serves at most one
+ // pending gap per (instance, stream, record_key) so a duplicate would be
+ // a store anomaly, not an expected state.
+ if (!lookup.has(statementId)) {
+ lookup.set(statementId, gap.gap_id);
+ }
+ }
+ return lookup;
+}
+
+/**
+ * Emit `DETAIL_GAP_RECOVERED` for each served statement gap whose PDF is
+ * hydrated again this run. `hydratedKeys` is the honest input: it is exactly
+ * the set `emitStatementDetailCoverage` already proved hydrated
+ * (`outcome.kind === "hydrated"`, driven by `isHydrated` at the download
+ * site), the same predicate feeding the coverage numerator, so recovery and
+ * coverage cannot drift apart. A statement whose PDF is still missing this
+ * run is left on the `DETAIL_GAP` re-emit path and is never recovered here —
+ * lose-no-data preserved.
+ */
+export async function recoverServedStatementGaps(
+ deps: EmitDeps,
+ hydratedKeys: readonly string[]
+): Promise {
+ const served = deps.servedStatementGaps;
+ if (!served || served.size === 0) {
+ return;
+ }
+ for (const statementId of hydratedKeys) {
+ const gapId = served.get(statementId);
+ if (!gapId) {
+ continue;
+ }
+ await deps.emit({
+ type: "DETAIL_GAP_RECOVERED",
+ reference_only: true,
+ gap_id: gapId,
+ stream: "statements",
+ record_key: statementId,
+ });
+ }
+}
+
/**
* Emit the transactions STATE cursor iff we actually emitted
* transactions this run. Skipping the emit on empty runs keeps
@@ -2606,9 +2692,10 @@ if (isMainModule(import.meta.url)) {
// `design-notes/chase-anti-bot.md`. Isolated-per-connector profile works.
browser: { profileName: "chase" },
timeRangeField: chaseTimeRangeField,
- async ensureSession({ context, onCredentialSubmit, page, sendInteraction }): Promise {
+ async ensureSession({ context, credentials, onCredentialSubmit, page, sendInteraction }): Promise {
await ensureChaseSession({
context,
+ credentials,
onCredentialSubmit,
page,
sendInteraction,
@@ -2687,6 +2774,7 @@ if (isMainModule(import.meta.url)) {
requested,
resFilters,
servedAccountGaps: buildServedAccountGapLookup(ctx.detailGaps),
+ servedStatementGaps: buildServedStatementGapLookup(ctx.detailGaps),
tmpDir,
txState,
transactionsFingerprintCursor,
@@ -2702,9 +2790,38 @@ if (isMainModule(import.meta.url)) {
const accounts = await collectChaseAccountInventory({ capture, emit, page });
if (accounts.length === 0) {
+ // Discovery found nothing. That is how a failed or blocked
+ // dashboard scrape presents, so the per-account cursors are
+ // deliberately left untouched — pruning here would erase every
+ // saved position and force a full re-download of every account.
return; // runtime emits DONE succeeded
}
+ // Prune per-account cursors for accounts discovery no longer
+ // returns. An orphaned cursor is indistinguishable from a live one,
+ // so without this an account that vanished from discovery is
+ // skipped silently, forever, with no gap emitted. Safe here because
+ // discovery is known non-empty.
+ const { dropped: droppedAccountCursors } = prunePerAccountCursors(
+ deps.maxSeenByAccount,
+ new Set(accounts.map((a) => a.internal_id))
+ );
+ if (droppedAccountCursors.length > 0) {
+ // Surfaced, not swallowed: this may be a closed account (benign)
+ // or a discovery regression (not benign), and the connector does
+ // not guess which.
+ await emit({
+ type: "SKIP_RESULT",
+ stream: "transactions",
+ reason: "account_no_longer_discovered",
+ message: "An account with a saved position is no longer listed on the dashboard",
+ diagnostics: { dropped_account_count: droppedAccountCursors.length },
+ });
+ for (const accountId of droppedAccountCursors) {
+ delete deps.maxSeenByAccount[accountId];
+ }
+ }
+
// Snapshot the dashboard overview DOM now while the page is still on
// it — the MDS recent-activity table (tr.mds-activity-table__row
// [data-values]) is only present here, NOT on the QFX download form
@@ -2796,8 +2913,10 @@ if (isMainModule(import.meta.url)) {
}
// Emit STATE for incremental resumption. The per_account cursor drives
- // the next run's chooseActivity() — when max_seen_date is present we'll
- // use "since_last_statement" instead of re-downloading all transactions.
+ // the next run's chooseActivity() — when max_seen_date is present and
+ // still FRESH we'll use "since_last_statement" instead of
+ // re-downloading all transactions; a stale cursor falls back to a
+ // bounded date_range export that actually reaches back to it.
await emitTransactionsStateIfAny(deps);
},
});
diff --git a/packages/polyfill-connectors/connectors/chase/parsers.test.ts b/packages/polyfill-connectors/connectors/chase/parsers.test.ts
index 814300a1a..0924b28e7 100644
--- a/packages/polyfill-connectors/connectors/chase/parsers.test.ts
+++ b/packages/polyfill-connectors/connectors/chase/parsers.test.ts
@@ -668,14 +668,27 @@ test("chooseActivity: a since-only scope closes at the deterministic run date",
assert.deepEqual(choice.dateRange, { from: "2026-05-01", to: "2026-08-13" });
});
-test("chooseActivity: cursor max_seen_date → since_last_statement", () => {
+test("chooseActivity: FRESH cursor max_seen_date → since_last_statement", () => {
const state: TransactionsStateShape = {
- per_account: { ID: { max_seen_date: "2026-03-01" } },
+ per_account: { ID: { max_seen_date: "2026-08-01" } },
};
const choice = chooseActivity(new Map(), state, "transactions", "ID", "2026-08-13T12:00:00Z");
assert.equal(choice.activity, "since_last_statement");
});
+test("chooseActivity: STALE cursor max_seen_date → bounded date_range", () => {
+ // This case previously asserted `since_last_statement`, which was the
+ // defect: a cursor 165 days behind the run date cannot be reached by
+ // Chase's "Since last statement" window (~30 days), so every scheduled
+ // run skipped the intervening period and still reported success.
+ const state: TransactionsStateShape = {
+ per_account: { ID: { max_seen_date: "2026-03-01" } },
+ };
+ const choice = chooseActivity(new Map(), state, "transactions", "ID", "2026-08-13T12:00:00Z");
+ assert.equal(choice.activity, "date_range");
+ assert.deepEqual(choice.dateRange, { from: "2026-03-01", to: "2026-08-13" });
+});
+
test("chooseActivity: no hints → all (bootstrap)", () => {
assert.equal(chooseActivity(new Map(), {}, "transactions", "ID", "2026-08-13T12:00:00Z").activity, "all");
});
diff --git a/packages/polyfill-connectors/connectors/chase/parsers.ts b/packages/polyfill-connectors/connectors/chase/parsers.ts
index dfab0bcb7..121437591 100644
--- a/packages/polyfill-connectors/connectors/chase/parsers.ts
+++ b/packages/polyfill-connectors/connectors/chase/parsers.ts
@@ -21,6 +21,7 @@ import type {
QfxExtracted,
QfxTransaction,
StatementRow,
+ TransactionCursor,
TransactionsStateShape,
} from "./types.ts";
@@ -786,10 +787,54 @@ export interface StreamScopeLike {
time_range?: TimeRange;
}
+/**
+ * How stale a cursor may be before "Since last statement" stops being a safe
+ * incremental choice.
+ *
+ * Chase's "Since last statement" option covers the current statement cycle
+ * only — roughly 30 days. If a cursor is older than that, the option's window
+ * no longer reaches back to where collection left off, and every scheduled
+ * run silently skips the gap between them: the run succeeds, downloads a
+ * QFX, and never sees the missing period. Nothing in the run reports a
+ * problem, because from the connector's point of view it asked for
+ * everything new and got it.
+ *
+ * 25 days is deliberately shorter than the ~30-day cycle so a cursor that is
+ * merely at the edge of the window still falls back to a bounded
+ * `date_range` export rather than gambling on the cycle boundary. Being
+ * early costs one wider export; being late costs a silent hole.
+ */
+const SINCE_LAST_STATEMENT_MAX_CURSOR_AGE_DAYS = 25;
+
+const MS_PER_DAY = 86_400_000;
+
+/**
+ * Whole days between two YYYY-MM-DD dates, or null when either is not a
+ * usable date. Returning null (rather than 0 or Infinity) keeps an
+ * unparseable cursor from being read as "fresh" — the caller treats null as
+ * "cannot prove freshness" and takes the safe path.
+ */
+export function cursorAgeInDays(maxSeenDate: string, runDate: string): number | null {
+ const seen = Date.parse(`${maxSeenDate.slice(0, 10)}T00:00:00Z`);
+ const now = Date.parse(`${runDate.slice(0, 10)}T00:00:00Z`);
+ if (!(Number.isFinite(seen) && Number.isFinite(now))) {
+ return null;
+ }
+ return Math.floor((now - seen) / MS_PER_DAY);
+}
+
/**
* Pick the QFX download "activity" option for a given account based on
* (1) an explicit scope time_range, (2) an existing cursor's
- * max_seen_date, or (3) "all" as the bootstrap default.
+ * max_seen_date — but only while that cursor is still fresh enough for
+ * "Since last statement" to actually reach it — or (3) "all" as the
+ * bootstrap default.
+ *
+ * The staleness check is the load-bearing part. Without it, the mere
+ * EXISTENCE of a cursor pinned every future run to "Since last statement",
+ * so once a gap opened no scheduled run could ever reach back across it.
+ * A stale cursor now produces a bounded `date_range` export that starts
+ * where collection actually left off.
*/
export function chooseActivity(
requested: Map,
@@ -811,11 +856,60 @@ export function chooseActivity(
}
const cursor = state.per_account?.[accountId];
if (cursor?.max_seen_date) {
- return { activity: "since_last_statement" };
+ const age = cursorAgeInDays(cursor.max_seen_date, runDate);
+ if (age !== null && age >= 0 && age <= SINCE_LAST_STATEMENT_MAX_CURSOR_AGE_DAYS) {
+ return { activity: "since_last_statement" };
+ }
+ // The cursor is stale (or unparseable, or in the future — a clock skew
+ // that makes freshness unprovable). "Since last statement" would not
+ // reach back to it, so export a bounded date range that starts AT the
+ // cursor instead. Starting at the cursor rather than after it keeps a
+ // one-day overlap, which is harmless: transaction ids are stable, so a
+ // re-seen transaction is suppressed rather than duplicated.
+ return {
+ activity: "date_range",
+ dateRange: { from: cursor.max_seen_date.slice(0, 10), to: runDate.slice(0, 10) },
+ };
}
return { activity: "all" };
}
+/**
+ * Drop `per_account` cursors for accounts the current run did not discover,
+ * and report which ones were dropped.
+ *
+ * Why this matters: the cursor map was carried forward wholesale, so an
+ * account that disappeared from `discoverAccounts()` kept its cursor
+ * forever. That is not merely untidy — the stale entry is indistinguishable
+ * from a live one, so the account is skipped silently with no gap emitted
+ * and nothing ever reports that it stopped being collected.
+ *
+ * `dropped` is returned rather than logged here so the caller can surface it
+ * as a real finding. An account vanishing from discovery is exactly the kind
+ * of event that should be visible: it may be a closed account (fine) or a
+ * discovery regression (not fine), and this function deliberately does not
+ * guess which.
+ *
+ * `known` is the set of accounts discovered THIS run. When discovery itself
+ * failed the caller must not call this with an empty set — pruning against a
+ * failed discovery would erase every cursor. See the caller's guard.
+ */
+export function prunePerAccountCursors(
+ perAccount: Record,
+ known: ReadonlySet
+): { kept: Record; dropped: string[] } {
+ const kept: Record = {};
+ const dropped: string[] = [];
+ for (const [accountId, cursor] of Object.entries(perAccount)) {
+ if (known.has(accountId)) {
+ kept[accountId] = cursor;
+ } else {
+ dropped.push(accountId);
+ }
+ }
+ return { kept, dropped };
+}
+
// ─── Public labels used by the click-path (exported for entry point) ──────
export const ACTIVITY_LABELS: Record = {
diff --git a/packages/polyfill-connectors/connectors/chatgpt/branch-reconciliation.test.ts b/packages/polyfill-connectors/connectors/chatgpt/branch-reconciliation.test.ts
new file mode 100644
index 000000000..2aac95d48
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/chatgpt/branch-reconciliation.test.ts
@@ -0,0 +1,231 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * ChatGPT conversations declare `message_count_on_current_branch`, and the
+ * obvious reconciliation — compare it against the messages we emitted — proves
+ * nothing. That field is computed by OUR OWN `countBranchMessages` from the
+ * same `mapping` object, inside the same call that emits those messages. Both
+ * sides of that comparison read one in-memory graph, so it is a tautology.
+ *
+ * Worse, it is a tautology that HIDES the real defect. `flattenTreeCurrentBranch`
+ * walks parent pointers and stops silently when a parent is missing from the
+ * mapping. A truncated payload therefore produces a short branch AND a
+ * correspondingly short declared count — the denominator shrinks to match the
+ * loss and the conversation reads complete.
+ *
+ * The provider assertion worth reconciling against is structural: `current_node`
+ * and the `parent` chain declare a branch that must be present and must
+ * terminate at a real root. These tests pin that contract.
+ *
+ * Grounding: reconciled against 5,821 live conversations. On-branch message
+ * counts matched the declared count exactly for 5,815 and never exceeded it,
+ * so equality is the right contract. Three conversations in the historical
+ * archive were short by exactly one message, each with a dangling non-system
+ * parent — the shape `truncatedBranch` reproduces below.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { makeRecordingEmit } from "../../src/test-harness.ts";
+import { processConversationDetail, type StreamDeps } from "./index.ts";
+import { buildConversationRecord, type ConversationDetail } from "./parsers.ts";
+import { validateRecord } from "./schemas.ts";
+import type { ChatGptFetchResult, ChatGptNode, ConversationListItem } from "./types.ts";
+
+function makeHarness(requested: readonly string[] = ["conversations", "messages"]) {
+ const harness = makeRecordingEmit(validateRecord);
+ const deps: StreamDeps = {
+ api: {
+ auth: (): Promise => Promise.reject(new Error("unused")),
+ fetch: (): Promise => Promise.resolve({ status: 200, json: null }),
+ },
+ emit: harness.emit,
+ emitRecord: harness.emitRecord,
+ progress: (): Promise => Promise.resolve(),
+ requested: new Map(requested.map((name) => [name, { name }])),
+ };
+ // SKIP_RESULT is a protocol message, not a schema rejection, so the gaps this
+ // contract emits live in protocolMessages.
+ const skips = (): Record[] =>
+ harness.protocolMessages.filter((m) => (m as { type?: string }).type === "SKIP_RESULT") as unknown as Record<
+ string,
+ unknown
+ >[];
+ return { deps, emitted: harness.emitted, messages: harness.protocolMessages, skips };
+}
+
+function makeConvo(currentNode: string): ConversationListItem {
+ return {
+ id: "convo-abc",
+ title: "Hello world",
+ create_time: 1_700_000_000,
+ update_time: 1_700_000_100,
+ current_node: currentNode,
+ };
+}
+
+function emitConversation(deps: StreamDeps) {
+ return async (c: ConversationListItem, detail: ConversationDetail | null): Promise => {
+ await deps.emitRecord("conversations", buildConversationRecord(c, detail));
+ };
+}
+
+function userNode(parent: string | null, children: string[] = []): ChatGptNode {
+ return {
+ parent,
+ children,
+ message: {
+ author: { role: "user" },
+ create_time: 1_700_000_000,
+ content: { content_type: "text", parts: ["hello"] },
+ },
+ };
+}
+
+function assistantNode(parent: string, children: string[] = []): ChatGptNode {
+ return {
+ parent,
+ children,
+ message: {
+ author: { role: "assistant" },
+ create_time: 1_700_000_001,
+ end_turn: true,
+ content: { content_type: "text", parts: ["hi there"] },
+ },
+ };
+}
+
+/** A whole conversation: root → u1 → a1, every parent present. */
+function wholeBranch(): Record {
+ return {
+ root: { parent: null, children: ["u1"] },
+ u1: userNode("root", ["a1"]),
+ a1: assistantNode("u1"),
+ };
+}
+
+/**
+ * The real defect shape found in the live archive: the branch tip is present
+ * and walkable, but the opening user turn it descends from was never
+ * delivered. `flattenTreeCurrentBranch` stops at `a1` and reports a 1-message
+ * branch, so the declared count agrees with the truncated data and nothing
+ * looks wrong.
+ */
+function truncatedBranch(): Record {
+ return {
+ a1: assistantNode("aaa2c1fa-missing-user-turn"),
+ };
+}
+
+async function run(mapping: Record, currentNode: string) {
+ const harness = makeHarness();
+ const detail: ChatGptFetchResult = {
+ status: 200,
+ json: {
+ title: "Hello world",
+ create_time: 1_700_000_000,
+ update_time: 1_700_000_100,
+ mapping,
+ current_node: currentNode,
+ },
+ };
+ await processConversationDetail(harness.deps, makeConvo(currentNode), detail, emitConversation(harness.deps));
+ return harness;
+}
+
+test("chatgpt branch: a whole conversation reconciles clean and reports no gap", async () => {
+ const { skips, emitted } = await run(wholeBranch(), "a1");
+
+ assert.equal(emitted.filter((r) => r.stream === "messages").length, 2, "u1 + a1 emit; root is synthetic");
+ assert.equal(skips().length, 0, "an intact parent chain must not manufacture a gap");
+});
+
+test("chatgpt branch: a truncated branch is surfaced as a gap, not a silent pass", async () => {
+ // This is the defect the declared-count comparison cannot see: the count and
+ // the data agree with each other, and both are short.
+ const { skips } = await run(truncatedBranch(), "a1");
+
+ const gap = skips().find((s) => s.reason === "branch_truncated");
+ assert.ok(gap, "a branch whose parent chain dangles must report a gap");
+ assert.equal(
+ (gap.diagnostics as { missing_parent_id: string }).missing_parent_id,
+ "aaa2c1fa-missing-user-turn",
+ "the gap names the message that was not delivered, so it is actionable"
+ );
+});
+
+test("chatgpt branch: the tautological count check would have passed this truncated payload", async () => {
+ // Proves the point of the whole contract. The conversation record's declared
+ // count is derived from the same truncated mapping, so declared == emitted
+ // and a count-based reconciliation reads complete. The gap is the only signal.
+ const { emitted, skips } = await run(truncatedBranch(), "a1");
+
+ const convo = emitted.find((r) => r.stream === "conversations");
+ const declared = convo?.data.message_count_on_current_branch;
+ const emittedOnBranch = emitted.filter((r) => r.stream === "messages" && r.data.on_current_branch === true).length;
+
+ assert.equal(declared, 1, "the declared count shrank to match the truncation");
+ assert.equal(emittedOnBranch, 1, "so declared == emitted and the counts agree");
+ assert.equal(
+ skips().some((s) => s.reason === "branch_truncated"),
+ true,
+ "only the structural check catches it"
+ );
+});
+
+test("chatgpt branch: a current_node absent from the mapping is surfaced", async () => {
+ // The conversation says it is on a tip the payload does not contain, so the
+ // branch we walked is not the branch it claims to be on.
+ const { skips } = await run(wholeBranch(), "tip-we-never-received");
+
+ const gap = skips().find((s) => s.reason === "branch_tip_missing");
+ assert.ok(gap, "an unreachable declared tip must report a gap");
+ assert.equal((gap.diagnostics as { conversation_id: string }).conversation_id, "convo-abc");
+});
+
+test("chatgpt branch: an off-branch alternative does not trigger a false gap", async () => {
+ // Branching is normal: holding MORE than the current branch is legitimate and
+ // must stay silent. Live data showed 28% of conversations hold off-branch
+ // messages, so a check that fired on these would be useless.
+ const mapping: Record = {
+ root: { parent: null, children: ["u1"] },
+ u1: userNode("root", ["a1", "a2"]),
+ a1: assistantNode("u1"),
+ a2: assistantNode("u1"),
+ };
+ const { skips, emitted } = await run(mapping, "a1");
+
+ assert.equal(emitted.filter((r) => r.stream === "messages").length, 3, "both branches are held");
+ assert.equal(skips().length, 0, "an extra branch is data we have, not data we lost");
+});
+
+test("chatgpt branch: a cyclic parent chain terminates instead of hanging", async () => {
+ // Defensive: a malformed graph must not spin the walk forever.
+ const mapping: Record = {
+ a1: assistantNode("a2"),
+ a2: assistantNode("a1"),
+ };
+ const { skips } = await run(mapping, "a1");
+
+ assert.equal(
+ skips().some((s) => s.reason === "branch_truncated"),
+ false,
+ "a cycle is fully present in the mapping; it is not a truncation"
+ );
+});
+
+test("chatgpt branch: a conversation with no current_node is not reconciled", async () => {
+ // Nothing was declared, so there is nothing to hold the payload to. Inventing
+ // a gap here would be a false positive on a legitimate shape.
+ const harness = makeHarness();
+ const mapping = wholeBranch();
+ const detail: ChatGptFetchResult = {
+ status: 200,
+ json: { title: "t", create_time: 1, update_time: 2, mapping, current_node: null },
+ };
+ const convo: ConversationListItem = { ...makeConvo("a1"), current_node: null };
+ await processConversationDetail(harness.deps, convo, detail, emitConversation(harness.deps));
+
+ assert.equal(harness.skips().length, 0, "no declared tip means no claim to reconcile against");
+});
diff --git a/packages/polyfill-connectors/connectors/chatgpt/index.ts b/packages/polyfill-connectors/connectors/chatgpt/index.ts
index cc2a0eeab..67bd466fc 100644
--- a/packages/polyfill-connectors/connectors/chatgpt/index.ts
+++ b/packages/polyfill-connectors/connectors/chatgpt/index.ts
@@ -85,6 +85,7 @@ import type {
ChatGptAuth,
ChatGptFetchResult,
ChatGptJson,
+ ChatGptNode,
ConversationListItem,
RawCustomInstructionsBody,
RawMemoryEntry,
@@ -2025,6 +2026,114 @@ export async function runCustomInstructionsStream(
deps.emit(buildFullScanCoverageMessage("custom_instructions", 1));
}
+/**
+ * Reconcile the current branch we walked against the branch the conversation
+ * actually declares, and surface a shortfall as a gap rather than a silent pass.
+ *
+ * The subtlety that shapes this whole check: ChatGPT does NOT hand us an
+ * independent provider total. `message_count_on_current_branch` is computed by
+ * our own `countBranchMessages` from the SAME `mapping` object, in the same
+ * call that emits these messages. Comparing emitted-vs-declared there is a
+ * tautology — both sides read one in-memory graph — so it would prove nothing.
+ *
+ * The real provider assertion is structural: `current_node` plus each node's
+ * `parent` pointer declare a chain, and `flattenTreeCurrentBranch` walks it by
+ * following parents until one is absent from the mapping. That walk STOPS
+ * SILENTLY on a missing parent. A truncated payload therefore yields a short
+ * branch AND a correspondingly short `message_count_on_current_branch` — the
+ * declared count shrinks to match the loss, so the conversation reads complete.
+ * That is the silent-truncation class this check closes.
+ *
+ * So the reconciliation is against the graph's own claims:
+ * - `current_node` must be present in the mapping. If the tip is absent, the
+ * branch we walked is not the branch the conversation says it is on.
+ * - The chain must terminate at a real root (a node with no parent), not at a
+ * dangling pointer into a node the payload did not include.
+ *
+ * Both are provider-asserted facts measured at the payload boundary, before any
+ * emit decision. Live reconciliation over 5,821 collected conversations:
+ * counting on-branch messages matched the declared count for 5,815 exactly and
+ * NEVER exceeded it, so equality is the right contract and a shortfall is real.
+ *
+ * Emitted as a SKIP_RESULT rather than a DETAIL_GAP for the same reason
+ * `empty_detail` is: the fetch SUCCEEDED and the conversation is genuinely
+ * hydrated. Re-fetching returns the same truncated graph, so a retryable gap
+ * would spin forever. The honest report is "we reached it, and what it gave us
+ * is internally inconsistent".
+ */
+async function emitBranchReconciliation(
+ deps: StreamDeps,
+ conversationId: string,
+ mapping: Record,
+ currentNode: string | null | undefined,
+ emittedBranchCount: number
+): Promise {
+ if (!currentNode) {
+ return;
+ }
+ // Measured at the payload boundary, independently of what was emitted: does
+ // the graph contain the tip it claims to be on?
+ if (!mapping[currentNode]) {
+ await deps.emit({
+ type: "SKIP_RESULT",
+ stream: "messages",
+ reason: "branch_tip_missing",
+ message: `conversation ${conversationId} declares a current_node its mapping does not contain; the branch is truncated`,
+ diagnostics: {
+ conversation_id: conversationId,
+ emitted_on_branch: emittedBranchCount,
+ node_count: Object.keys(mapping).length,
+ },
+ });
+ return;
+ }
+ // Walk the declared parent chain to its end. A chain that ends on a node
+ // whose `parent` names an id the mapping lacks was cut short upstream — the
+ // branch continues in the provider's data but not in ours.
+ const danglingParent = findDanglingBranchParent(mapping, currentNode);
+ if (danglingParent !== null) {
+ await deps.emit({
+ type: "SKIP_RESULT",
+ stream: "messages",
+ reason: "branch_truncated",
+ message: `conversation ${conversationId} has a current-branch node whose parent is absent from the mapping; earlier messages on this branch were not delivered`,
+ diagnostics: {
+ conversation_id: conversationId,
+ emitted_on_branch: emittedBranchCount,
+ // A ChatGPT node UUID, not user content — safe to disclose, and the
+ // only handle that makes the gap actionable.
+ missing_parent_id: danglingParent,
+ node_count: Object.keys(mapping).length,
+ },
+ });
+ }
+}
+
+/**
+ * Follow the current branch root-ward and return the first parent id the
+ * mapping does not contain, or null when the chain terminates cleanly. Mirrors
+ * `flattenTreeCurrentBranch`'s traversal — including its cycle guard — so the
+ * two agree on what "the current branch" means.
+ *
+ * A parentless root needs no special case: `id = parent` sets `id` falsy and
+ * the loop condition ends the walk, returning null. An explicit root check was
+ * tried and removed as dead code — it changed the result on none of the 1,029
+ * enumerated parent graphs over three nodes.
+ */
+function findDanglingBranchParent(mapping: Record, currentNode: string): string | null {
+ const visited = new Set();
+ let id: string | null | undefined = currentNode;
+ while (id && !visited.has(id) && mapping[id]) {
+ visited.add(id);
+ const parent: string | null | undefined = mapping[id]?.parent;
+ if (parent && !mapping[parent]) {
+ return parent;
+ }
+ id = parent;
+ }
+ return null;
+}
+
/**
* Process a single fetched conversation detail payload: emit the merged
* conversation record first, then emit messages along the current branch
@@ -2064,15 +2173,21 @@ export async function processConversationDetail(
const currentNode = detail.json.current_node || c.current_node;
const currentBranchIds = new Set(flattenTreeCurrentBranch(mapping, currentNode).map((x) => x.nodeId));
let emittedMessageCount = 0;
+ let emittedBranchCount = 0;
for (const [nodeId, node] of Object.entries(mapping)) {
- const msg = extractMessage(nodeId, node, c.id, currentBranchIds.has(nodeId));
+ const onBranch = currentBranchIds.has(nodeId);
+ const msg = extractMessage(nodeId, node, c.id, onBranch);
if (!msg?.role) {
// synthetic root — skip
continue;
}
emittedMessageCount += 1;
+ if (onBranch) {
+ emittedBranchCount += 1;
+ }
await deps.emitRecord("messages", msg);
}
+ await emitBranchReconciliation(deps, c.id, mapping, currentNode, emittedBranchCount);
if (emittedMessageCount === 0) {
// Completeness guard. A 200-with-mapping detail whose graph contains NO
// message-bearing node leaves a bare conversation row with zero messages
diff --git a/packages/polyfill-connectors/connectors/claude_code/index.ts b/packages/polyfill-connectors/connectors/claude_code/index.ts
index 8fa7e8cc9..e8852540a 100755
--- a/packages/polyfill-connectors/connectors/claude_code/index.ts
+++ b/packages/polyfill-connectors/connectors/claude_code/index.ts
@@ -26,6 +26,7 @@
* Skills/commands live under ~/.claude (overridable via CLAUDE_CODE_HOME).
*/
+import { createHash } from "node:crypto";
import { createReadStream, type Dirent, type Stats, statSync } from "node:fs";
import { readdir, stat } from "node:fs/promises";
import { homedir } from "node:os";
@@ -519,10 +520,11 @@ async function readFilesRecursively(
): Promise> {
const out: Array<{ fullPath: string; relPath: string }> = [];
const walk = async (dir: string, prefix: string): Promise => {
- let items: Dirent[];
- try {
- items = await readdir(dir, { withFileTypes: true });
- } catch {
+ // Fail closed on an unreadable directory — see `readLocalDirOrFailClosed`.
+ // A missing directory (ENOENT) is honestly empty; an unreadable one is a
+ // source-boundary failure and must never be reported as "no files".
+ const items = await readLocalDirOrFailClosed(dir);
+ if (items === null) {
return;
}
for (const ent of items.sort((a, b) => a.name.localeCompare(b.name))) {
@@ -622,14 +624,77 @@ interface EmitSkillsArgs {
requested: Map;
}
-function markFileMtimeAndShouldSkip(
+/**
+ * Read a local directory, distinguishing legitimate absence from failure.
+ *
+ * `null` means the directory genuinely does not exist (ENOENT) — an owner who
+ * has no `~/.claude/skills` truly has no skills, and an empty enumeration is
+ * the honest answer.
+ *
+ * Any OTHER error (EACCES, EPERM, ENOTDIR, EIO) means the enumeration did not
+ * happen. The filesystem is this connector's entire source of truth, so an
+ * unreadable directory is a source-boundary failure, not evidence of emptiness.
+ * It THROWS rather than returning empty.
+ *
+ * WHY: reproduced before this guard — a `chmod 000` on a skills directory
+ * holding a real skill produced 0 records, ZERO skips, no error, and a STATE
+ * checkpoint carrying an empty cursor. The run silently recorded "this owner
+ * has no skills" as fact. Never treat an unreadable directory as "zero files,
+ * complete". This mirrors codex's `listIfExists`, which already fails closed.
+ */
+async function readLocalDirOrFailClosed(dir: string): Promise {
+ try {
+ return await readdir(dir, { withFileTypes: true });
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
+ return null;
+ }
+ throw error;
+ }
+}
+
+/**
+ * Content-hash gate for the markdown streams (skills, slash_commands,
+ * memory_notes).
+ *
+ * These files are small and already read in full to build their record, so
+ * hashing the bytes we just read costs nothing extra and is strictly stronger
+ * than the mtime equality this replaces.
+ *
+ * WHY mtime alone was wrong: mtime is owner-controlled metadata, not a content
+ * fact. `git checkout` and `rsync --times` both restore a previous mtime onto
+ * genuinely different bytes, and the old gate then skipped the file forever —
+ * reproduced directly: a SKILL.md whose body changed completely, with its mtime
+ * restored, emitted zero records on the next run. Silent data loss.
+ *
+ * The stored value stays a `number` so the existing `file_mtimes` cursor shape
+ * is untouched: a 52-bit prefix of the SHA-256 is an exact integer in a double
+ * and never collides in practice at these file counts. A cursor written by the
+ * old mtime-based build simply mismatches the new hash once and the file is
+ * re-emitted — a one-time re-read, never a miss.
+ */
+function contentGateValue(text: string): number {
+ const digest = createHash("sha256").update(text, "utf8").digest();
+ // Take 52 bits — the largest integer width a JS double holds exactly — by
+ // dividing rather than shifting (Biome bans bitwise operators here).
+ // `2n ** 12n` drops the low 12 bits of the leading 64.
+ return Number(digest.readBigUInt64BE(0) / 4096n);
+}
+
+/**
+ * Record this file's content gate and report whether it is unchanged since the
+ * cursor. Callers MUST have the file's text already — the gate is a fact about
+ * bytes, never about metadata.
+ */
+function markFileContentAndShouldSkip(
fileMtimes: Record,
newMtimes: Record,
path: string,
- mtime: number
+ text: string
): boolean {
- newMtimes[path] = mtime;
- return fileMtimes[path] === mtime;
+ const gate = contentGateValue(text);
+ newMtimes[path] = gate;
+ return fileMtimes[path] === gate;
}
async function readBoundedUtf8(path: string): Promise {
@@ -642,10 +707,8 @@ async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMt
return;
}
const skillsDir = join(claudeHome, "skills");
- let entries: Dirent[];
- try {
- entries = await readdir(skillsDir, { withFileTypes: true });
- } catch {
+ const entries = await readLocalDirOrFailClosed(skillsDir);
+ if (entries === null) {
return;
}
for (const ent of entries) {
@@ -663,9 +726,9 @@ async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMt
} catch {
continue;
}
- if (markFileMtimeAndShouldSkip(fileMtimes, newMtimes, skillPath, st.mtimeMs)) {
- continue;
- }
+ // Read BEFORE gating: the gate is a fact about content, so the bytes must
+ // be in hand to compute it. These files are small and were already read to
+ // build the record, so this reorder costs no extra I/O.
try {
raw = await readBoundedUtf8(skillPath);
} catch {
@@ -674,6 +737,9 @@ async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMt
if (raw === null) {
continue;
}
+ if (markFileContentAndShouldSkip(fileMtimes, newMtimes, skillPath, raw)) {
+ continue;
+ }
const { frontmatter, body } = parseFrontmatter(raw);
await emitRecord(
"skills",
@@ -702,9 +768,7 @@ async function processSlashCommandFile(args: ProcessSlashCommandArgs): Promise => {
- let items: Dirent[];
- try {
- items = await readdir(dir, { withFileTypes: true });
- } catch {
+ // Fail closed on an unreadable directory — see `readLocalDirOrFailClosed`.
+ // A missing directory (ENOENT) is honestly empty; an unreadable one is a
+ // source-boundary failure and must never be reported as "no files".
+ const items = await readLocalDirOrFailClosed(dir);
+ if (items === null) {
return;
}
for (const ent of items) {
@@ -793,9 +861,7 @@ async function emitProjectMemoryNotes({
} catch {
continue;
}
- if (markFileMtimeAndShouldSkip(fileMtimes, newMtimes, fullPath, st.mtimeMs)) {
- continue;
- }
+ // Content gate, not mtime — see `markFileContentAndShouldSkip`. Read first.
try {
raw = await readBoundedUtf8(fullPath);
} catch {
@@ -804,6 +870,9 @@ async function emitProjectMemoryNotes({
if (raw === null) {
continue;
}
+ if (markFileContentAndShouldSkip(fileMtimes, newMtimes, fullPath, raw)) {
+ continue;
+ }
const { frontmatter, body } = parseFrontmatter(raw);
await emitRecord(
"memory_notes",
@@ -1665,6 +1734,12 @@ async function runSkillsAndCommands(
newSlashCommandMtimes: Record;
}
): Promise {
+ // A scan that FAILED must not checkpoint. The cursor is a claim about what
+ // the source contained; writing one after a failed enumeration persists
+ // "this owner has no skills" as fact and suppresses the files forever on
+ // subsequent runs. Track each scan's outcome and gate its STATE on success.
+ let skillsScanned = true;
+ let slashCommandsScanned = true;
try {
await emitSkills({
claudeHome,
@@ -1674,6 +1749,7 @@ async function runSkillsAndCommands(
newMtimes: state.newSkillsMtimes,
});
} catch {
+ skillsScanned = false;
await emit({ type: "PROGRESS", message: "Claude Code phase=index pass=index stream=skills scan_skipped=true" });
}
try {
@@ -1685,24 +1761,43 @@ async function runSkillsAndCommands(
newMtimes: state.newSlashCommandMtimes,
});
} catch {
+ slashCommandsScanned = false;
await emit({
type: "PROGRESS",
message: "Claude Code phase=index pass=index stream=slash_commands scan_skipped=true",
});
}
if (requested.has("skills")) {
- await emit({
- type: "STATE",
- stream: "skills",
- cursor: { file_mtimes: state.newSkillsMtimes, fetched_at: nowIso() },
- });
+ if (skillsScanned) {
+ await emit({
+ type: "STATE",
+ stream: "skills",
+ cursor: { file_mtimes: state.newSkillsMtimes, fetched_at: nowIso() },
+ });
+ } else {
+ await emit({
+ type: "SKIP_RESULT",
+ stream: "skills",
+ reason: "source_unreadable",
+ message: "The Claude Code skills directory could not be enumerated, so its contents are unknown for this run",
+ });
+ }
}
if (requested.has("slash_commands")) {
- await emit({
- type: "STATE",
- stream: "slash_commands",
- cursor: { file_mtimes: state.newSlashCommandMtimes, fetched_at: nowIso() },
- });
+ if (slashCommandsScanned) {
+ await emit({
+ type: "STATE",
+ stream: "slash_commands",
+ cursor: { file_mtimes: state.newSlashCommandMtimes, fetched_at: nowIso() },
+ });
+ } else {
+ await emit({
+ type: "SKIP_RESULT",
+ stream: "slash_commands",
+ reason: "source_unreadable",
+ message: "The Claude Code commands directory could not be enumerated, so its contents are unknown for this run",
+ });
+ }
}
}
diff --git a/packages/polyfill-connectors/connectors/claude_code/markdown-content-gate.test.ts b/packages/polyfill-connectors/connectors/claude_code/markdown-content-gate.test.ts
new file mode 100644
index 000000000..6e9420f91
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/claude_code/markdown-content-gate.test.ts
@@ -0,0 +1,136 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * The markdown streams (skills, slash_commands, memory_notes) gate re-emission
+ * on FILE CONTENT, not on mtime.
+ *
+ * mtime is owner-controlled metadata, not a content fact. `git checkout` and
+ * `rsync --times` both restore a prior mtime onto genuinely different bytes.
+ * Under the old mtime-equality gate that file was skipped forever — reproduced
+ * directly before the fix: a SKILL.md whose body changed completely, with its
+ * mtime restored to the cursor's value, emitted ZERO records on the next run.
+ *
+ * These tests pin both directions, because a gate is only honest if it holds
+ * both:
+ * - changed content + identical mtime => MUST re-emit (the data-loss bug)
+ * - identical content + changed mtime => MUST stay suppressed (no churn)
+ */
+
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+import type { EmittedMessage } from "../../src/connector-runtime.ts";
+import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts";
+
+const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "claude_code", "index.ts");
+
+/** A fixed mtime, reapplied after every write so mtime is never the signal. */
+const PINNED_MTIME = new Date("2026-01-01T00:00:00Z");
+
+async function writePinned(path: string, body: string): Promise {
+ await writeFile(path, body);
+ await utimes(path, PINNED_MTIME, PINNED_MTIME);
+}
+
+async function runClaudeCode(
+ home: string,
+ stream: string,
+ state: Record
+): Promise<{ carried: Record; records: EmittedMessage[] }> {
+ const result = await runConnectorProtocolSubprocess({
+ cwd: PACKAGE_ROOT,
+ entrypoint: ENTRYPOINT,
+ env: {
+ CLAUDE_CODE_HOME: home,
+ HOME: home,
+ PDPP_OWNER_TOKEN: "",
+ PDPP_RS_URL: "",
+ RS_URL: "",
+ },
+ start: { scope: { streams: [{ name: stream }] }, state, type: "START" },
+ });
+ const carried: Record = {};
+ for (const message of result.messages) {
+ if (message.type === "STATE" && typeof (message as { stream?: string }).stream === "string") {
+ carried[(message as { stream: string }).stream] = (message as { cursor?: unknown }).cursor;
+ }
+ }
+ return {
+ carried,
+ records: result.messages.filter((m) => m.type === "RECORD" && m.stream === stream),
+ };
+}
+
+test("a skill whose content changes under a restored mtime is re-emitted", async () => {
+ const home = await mkdtemp(join(tmpdir(), "pdpp-cc-content-gate-"));
+ try {
+ const skillDir = join(home, "skills", "demo");
+ await mkdir(skillDir, { recursive: true });
+ const skillPath = join(skillDir, "SKILL.md");
+ await writePinned(skillPath, "---\nname: demo\n---\nORIGINAL BODY\n");
+
+ const first = await runClaudeCode(home, "skills", {});
+ assert.equal(first.records.length, 1, "the skill should be emitted on a fresh run");
+
+ // The exact `git checkout` / `rsync --times` shape: content replaced,
+ // mtime restored to the value already in the cursor.
+ await writePinned(skillPath, "---\nname: demo\n---\nCOMPLETELY DIFFERENT BODY\n");
+
+ const second = await runClaudeCode(home, "skills", first.carried);
+ // The load-bearing assertion. Under the mtime gate this was 0.
+ assert.equal(second.records.length, 1, "changed content under an identical mtime must re-emit, not be skipped");
+ } finally {
+ await rm(home, { force: true, recursive: true });
+ }
+});
+
+test("an unchanged skill stays suppressed even when its mtime moves", async () => {
+ const home = await mkdtemp(join(tmpdir(), "pdpp-cc-content-gate-stable-"));
+ try {
+ const skillDir = join(home, "skills", "demo");
+ await mkdir(skillDir, { recursive: true });
+ const skillPath = join(skillDir, "SKILL.md");
+ const body = "---\nname: demo\n---\nSTABLE BODY\n";
+ await writePinned(skillPath, body);
+
+ const first = await runClaudeCode(home, "skills", {});
+ assert.equal(first.records.length, 1);
+
+ // Byte-identical rewrite, but the mtime jumps forward — a touch, or any
+ // tool that rewrites in place. Content is the signal, so this must NOT
+ // churn. Guards the opposite failure from the test above.
+ const later = new Date("2026-06-01T00:00:00Z");
+ await writeFile(skillPath, body);
+ await utimes(skillPath, later, later);
+
+ const second = await runClaudeCode(home, "skills", first.carried);
+ assert.equal(second.records.length, 0, "identical content must stay suppressed regardless of mtime");
+ } finally {
+ await rm(home, { force: true, recursive: true });
+ }
+});
+
+test("a slash command whose content changes under a restored mtime is re-emitted", async () => {
+ const home = await mkdtemp(join(tmpdir(), "pdpp-cc-content-gate-cmd-"));
+ try {
+ const commandsDir = join(home, "commands");
+ await mkdir(commandsDir, { recursive: true });
+ const cmdPath = join(commandsDir, "deploy.md");
+ await writePinned(cmdPath, "---\ndescription: original\n---\nrun the original steps\n");
+
+ const first = await runClaudeCode(home, "slash_commands", {});
+ assert.equal(first.records.length, 1);
+
+ await writePinned(cmdPath, "---\ndescription: rewritten\n---\nrun completely different steps\n");
+
+ const second = await runClaudeCode(home, "slash_commands", first.carried);
+ assert.equal(second.records.length, 1, "slash_commands must use the same content gate as skills");
+ } finally {
+ await rm(home, { force: true, recursive: true });
+ }
+});
diff --git a/packages/polyfill-connectors/connectors/claude_code/unreadable-source-fails-closed.test.ts b/packages/polyfill-connectors/connectors/claude_code/unreadable-source-fails-closed.test.ts
new file mode 100644
index 000000000..0e6d1f5fe
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/claude_code/unreadable-source-fails-closed.test.ts
@@ -0,0 +1,116 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * An unreadable source directory must never be reported as "zero files".
+ *
+ * The filesystem is this connector's whole source of truth, so a directory it
+ * cannot enumerate is a source-boundary failure — NOT evidence of emptiness.
+ * A missing directory (ENOENT) is different: an owner with no
+ * `~/.claude/skills` genuinely has no skills, and an empty answer is correct.
+ *
+ * Reproduced before the guard: `chmod 000` on a skills directory holding a real
+ * skill produced 0 records, ZERO skips, no error, and a STATE checkpoint with
+ * an empty cursor — the run silently recorded "this owner has no skills" and
+ * persisted it. That is the fail-open shape these tests forbid.
+ *
+ * These tests are skipped when running as root, which bypasses permission bits.
+ */
+
+import assert from "node:assert/strict";
+import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts";
+
+const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "claude_code", "index.ts");
+// chmod bits do not restrain root, so the unreadable-dir premise cannot hold.
+const RUNNING_AS_ROOT = typeof process.getuid === "function" && process.getuid() === 0;
+
+async function runSkills(home: string): Promise<{
+ records: number;
+ states: number;
+ unreadable: number;
+}> {
+ const result = await runConnectorProtocolSubprocess({
+ // Matches every existing claude_code protocol test: this connector can exit
+ // non-zero after DONE for reasons unrelated to the streams under test.
+ allowFailedDone: true,
+ cwd: PACKAGE_ROOT,
+ entrypoint: ENTRYPOINT,
+ env: {
+ CLAUDE_CODE_HOME: home,
+ CLAUDE_CODE_PROJECTS_DIR: join(home, "projects"),
+ HOME: home,
+ PDPP_OWNER_TOKEN: "",
+ PDPP_RS_URL: "",
+ RS_URL: "",
+ },
+ start: { scope: { streams: [{ name: "skills" }] }, state: {}, type: "START" },
+ });
+ return {
+ records: result.messages.filter((m) => m.type === "RECORD" && m.stream === "skills").length,
+ states: result.messages.filter((m) => m.type === "STATE" && m.stream === "skills").length,
+ unreadable: result.messages.filter(
+ (m) => m.type === "SKIP_RESULT" && m.stream === "skills" && m.reason === "source_unreadable"
+ ).length,
+ };
+}
+
+test("an unreadable skills directory does not checkpoint an empty cursor", { skip: RUNNING_AS_ROOT }, async () => {
+ const home = await mkdtemp(join(tmpdir(), "pdpp-cc-unreadable-"));
+ const skillsDir = join(home, "skills");
+ try {
+ await mkdir(join(home, "projects"), { recursive: true });
+ const skillDir = join(skillsDir, "demo");
+ await mkdir(skillDir, { recursive: true });
+ await writeFile(join(skillDir, "SKILL.md"), "---\nname: demo\n---\nREAL BODY\n");
+ await chmod(skillsDir, 0o000);
+
+ const outcome = await runSkills(home);
+
+ // The load-bearing assertion. Before the guard this was `states: 1` with an
+ // empty `file_mtimes` map — a persisted claim that the owner has no skills.
+ assert.equal(
+ outcome.states,
+ 0,
+ "an unreadable source directory must not produce a STATE checkpoint claiming an empty enumeration"
+ );
+ assert.equal(outcome.records, 0, "no records can be read from an unreadable directory");
+ assert.equal(outcome.unreadable, 1, "the failed enumeration must surface as a skip, not silence");
+ } finally {
+ await chmod(skillsDir, 0o755).catch(() => undefined);
+ await rm(home, { force: true, recursive: true });
+ }
+});
+
+test("a genuinely missing skills directory is still an honest empty enumeration", async () => {
+ const home = await mkdtemp(join(tmpdir(), "pdpp-cc-absent-"));
+ try {
+ // A readable home with a `projects` dir but NO `skills` dir: the skills
+ // enumeration hits ENOENT. This owner truly has no skills, so the run must
+ // succeed and checkpoint normally. Guards the opposite failure — a
+ // fail-closed guard that also tripped on legitimate absence would break
+ // every owner who has never created a skill.
+ //
+ // (`projects` is created because the connector requires the surrounding
+ // home layout to run at all; a bare temp dir exits non-zero on the
+ // unmodified connector too, so its absence would not test this guard.)
+ await mkdir(join(home, "projects"), { recursive: true });
+
+ const outcome = await runSkills(home);
+
+ // The distinction that matters: absence is NOT reported as a source
+ // failure. (Verified against the unmodified connector, an absent skills
+ // directory also produces no STATE — the stream is simply not exercised.
+ // This guard must not change that; it must only stop an UNREADABLE
+ // directory from checkpointing.)
+ assert.equal(outcome.unreadable, 0, "an absent directory is legitimate, not a source failure");
+ assert.equal(outcome.records, 0);
+ } finally {
+ await rm(home, { force: true, recursive: true });
+ }
+});
diff --git a/packages/polyfill-connectors/connectors/codex/index.ts b/packages/polyfill-connectors/connectors/codex/index.ts
index aada3090b..5b71f9991 100644
--- a/packages/polyfill-connectors/connectors/codex/index.ts
+++ b/packages/polyfill-connectors/connectors/codex/index.ts
@@ -568,11 +568,22 @@ async function emitSkillsStream(
): Promise {
// Each skill is a subdirectory with SKILL.md at its root. Follows symlinks
// (skills are often symlinked from dotfiles). Skips hidden dirs (.system).
+ //
+ // Fail closed on an unreadable directory, matching `emitRulesStream` and
+ // `emitPromptsStream`, which already route through `listIfExists`. This
+ // function previously swallowed EVERY error, so a permission failure on the
+ // skills directory was indistinguishable from "this owner has no skills" —
+ // the filesystem is the whole source of truth here, so an unreadable
+ // directory is a source-boundary failure, not evidence of emptiness. ENOENT
+ // (genuinely absent) remains a legitimate empty enumeration.
let entries: Dirent[];
try {
entries = await readdir(skillsDir, { withFileTypes: true });
- } catch {
- return;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
+ return;
+ }
+ throw error;
}
for (const ent of entries) {
if (shouldSkipSkillEntry(ent)) {
diff --git a/packages/polyfill-connectors/connectors/codex/skills-source-fails-closed.test.ts b/packages/polyfill-connectors/connectors/codex/skills-source-fails-closed.test.ts
new file mode 100644
index 000000000..30bb5384b
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/codex/skills-source-fails-closed.test.ts
@@ -0,0 +1,85 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * `emitSkillsStream` must fail closed on an unreadable skills directory.
+ *
+ * codex's other two markdown streams (`rules`, `prompts`) already route through
+ * `listIfExists`, which distinguishes ENOENT (legitimate absence) from a real
+ * read failure and rethrows the latter. `emitSkillsStream` was the odd one out:
+ * it swallowed EVERY error, so a permission failure on the skills directory was
+ * indistinguishable from "this owner has no skills".
+ *
+ * The filesystem is this connector's entire source of truth, so an unreadable
+ * directory is a source-boundary failure, never evidence of emptiness.
+ *
+ * The unreadable case is skipped as root, which bypasses permission bits.
+ */
+
+import assert from "node:assert/strict";
+import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts";
+
+const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "codex", "index.ts");
+const RUNNING_AS_ROOT = typeof process.getuid === "function" && process.getuid() === 0;
+
+async function runSkills(codexHome: string): Promise<{ records: number; states: number }> {
+ const result = await runConnectorProtocolSubprocess({
+ allowFailedDone: true,
+ cwd: PACKAGE_ROOT,
+ entrypoint: ENTRYPOINT,
+ env: { CODEX_HOME: codexHome, HOME: codexHome },
+ start: { scope: { streams: [{ name: "skills" }] }, state: {}, type: "START" },
+ });
+ return {
+ records: result.messages.filter((m) => m.type === "RECORD" && m.stream === "skills").length,
+ states: result.messages.filter((m) => m.type === "STATE" && m.stream === "skills").length,
+ };
+}
+
+test("an unreadable codex skills directory does not report an empty enumeration", {
+ skip: RUNNING_AS_ROOT,
+}, async () => {
+ const home = await mkdtemp(join(tmpdir(), "pdpp-codex-unreadable-"));
+ const skillsDir = join(home, "skills");
+ try {
+ await mkdir(join(skillsDir, "demo"), { recursive: true });
+ await writeFile(join(skillsDir, "demo", "SKILL.md"), "---\nname: demo\n---\nREAL BODY\n");
+ await chmod(skillsDir, 0o000);
+
+ const outcome = await runSkills(home);
+
+ // The load-bearing assertion: the unreadable directory must not be
+ // silently reported as a clean, empty, checkpointed enumeration.
+ assert.equal(outcome.records, 0, "no records can be read from an unreadable directory");
+ assert.equal(
+ outcome.states,
+ 0,
+ "an unreadable source directory must not checkpoint a cursor claiming an empty enumeration"
+ );
+ } finally {
+ await chmod(skillsDir, 0o755).catch(() => undefined);
+ await rm(home, { force: true, recursive: true });
+ }
+});
+
+test("a readable codex skills directory still emits its skills", async () => {
+ const home = await mkdtemp(join(tmpdir(), "pdpp-codex-readable-"));
+ try {
+ await mkdir(join(home, "skills", "demo"), { recursive: true });
+ await writeFile(join(home, "skills", "demo", "SKILL.md"), "---\nname: demo\n---\nREAL BODY\n");
+
+ const outcome = await runSkills(home);
+
+ // Guards the opposite failure: a fail-closed guard that also tripped on a
+ // perfectly readable directory would break every owner who has skills.
+ assert.equal(outcome.records, 1, "a readable skills directory must still be enumerated");
+ } finally {
+ await rm(home, { force: true, recursive: true });
+ }
+});
diff --git a/packages/polyfill-connectors/connectors/github/index.ts b/packages/polyfill-connectors/connectors/github/index.ts
index d0ade7e38..031889795 100755
--- a/packages/polyfill-connectors/connectors/github/index.ts
+++ b/packages/polyfill-connectors/connectors/github/index.ts
@@ -304,6 +304,43 @@ async function guardGithubPagination(
* cannot know its full inventory for the run (e.g. a search-API cap truncation)
* MUST NOT call this — it leaves `considered` unknown and relies on its
* terminal-gap evidence instead.
+ *
+ * ── Why no provider-reported total is bound here ──────────────────────────
+ *
+ * GitHub exposes several tempting scalars. Each was measured against this
+ * instance's live holdings and REJECTED. Do not bind them:
+ *
+ * `public_repos` / `public_gists` (from `/user`, already stored on the
+ * `user_stats` record) measure a strict SUBSET, not this stream's boundary.
+ * Live: `public_repos: 94` against 575 held repositories — of which 355 are
+ * private and 465 belong to orgs, neither of which `public_repos` counts.
+ * `public_gists: 8` matched the 8 public gists exactly while 43 secret gists
+ * sat outside it. Binding either would assert a permanent ~6x false gap.
+ *
+ * `Link: rel="last"` yields a PAGE count, so an item total only under the
+ * assumption that every page is full — which the last page never is. It also
+ * cannot survive the deletion semantics below.
+ *
+ * `total_count` on `/search/issues` IS authoritative for its query, and is
+ * already consumed for cap-detection (see `PR_SEARCH_RESULT_CAP`). It is not
+ * promoted to the denominator because a search index is eventually
+ * consistent with the REST list this stream walks, so a benign index lag
+ * would read as coverage loss.
+ *
+ * The deeper constraint applies to ALL of them: PDPP deliberately RETAINS
+ * records after the provider deletes them, and GitHub genuinely deletes repos,
+ * issues and gists. A provider total therefore describes the surviving account
+ * and is legitimately SMALLER than what we hold. Any two-way
+ * `provider_total === held_count` check flags successful preservation as a
+ * defect. A sound anchor here would have to be the three-way relation
+ * `provider_total === live_holdings - known_tombstoned`, and this connector
+ * declares no tombstones at all (no `isTombstone`), so the third term is
+ * unavailable and the relation cannot be closed.
+ *
+ * A scalar also cannot distinguish missing from surplus from duplicated. If a
+ * real anchor is wanted later, compare the provider's ID SET against the held
+ * ID set — GitHub returns stable numeric ids on every one of these streams —
+ * and tombstone the upstream-absent ids rather than counting them as loss.
*/
async function declareListConsidered(
ctx: StreamCtx,
diff --git a/packages/polyfill-connectors/connectors/github/provider-total-not-an-anchor.test.ts b/packages/polyfill-connectors/connectors/github/provider-total-not-an-anchor.test.ts
new file mode 100644
index 000000000..f8598a61f
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/github/provider-total-not-an-anchor.test.ts
@@ -0,0 +1,80 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * `public_repos` / `public_gists` are NOT coverage anchors.
+ *
+ * They are the most tempting numbers GitHub hands us — provider-reported,
+ * already fetched, already stored on the `user_stats` record. They are also
+ * wrong for the job, and this test exists so that stays discovered.
+ *
+ * They measure a strict SUBSET of what the collected streams walk:
+ * - `/user/repos` returns private and org repositories; `public_repos`
+ * counts only the user's own PUBLIC ones.
+ * - `/gists` returns secret gists; `public_gists` counts only public ones.
+ *
+ * Measured against this instance's live holdings when the anchor work was done:
+ * `public_repos: 94` against 575 held repositories (355 private, 465 org-owned),
+ * and `public_gists: 8` against 51 held gists — where the 8 matched the public
+ * gists EXACTLY and 43 secret gists sat outside the number entirely. Binding
+ * either as a denominator would assert a permanent false gap on a correct run.
+ *
+ * This test pins the structural fact behind those numbers: the stream walks
+ * repositories the provider scalar does not count. It is deliberately about the
+ * RELATIONSHIP, not a specific count.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { repoRecord, userStatsRecord } from "./parsers.ts";
+
+test("a private or org repository is collected but is not counted by public_repos", () => {
+ // One public user-owned repo — the only kind `public_repos` counts.
+ const publicOwn = repoRecord({
+ id: 1,
+ name: "public-own",
+ full_name: "octocat/public-own",
+ private: false,
+ owner: { login: "octocat", id: 99 },
+ pushed_at: "2026-01-01T00:00:00Z",
+ } as never);
+ // A private repo and an org repo — both returned by `/user/repos`, and both
+ // invisible to `public_repos`.
+ const privateOwn = repoRecord({
+ id: 2,
+ name: "private-own",
+ full_name: "octocat/private-own",
+ private: true,
+ owner: { login: "octocat", id: 99 },
+ pushed_at: "2026-01-02T00:00:00Z",
+ } as never);
+ const orgRepo = repoRecord({
+ id: 3,
+ name: "org-repo",
+ full_name: "acme/org-repo",
+ private: false,
+ owner: { login: "acme", id: 1234 },
+ pushed_at: "2026-01-03T00:00:00Z",
+ } as never);
+
+ const collected = [publicOwn, privateOwn, orgRepo];
+
+ // The provider scalar, reported by `/user`, sees only the single public
+ // user-owned repository.
+ const stats = userStatsRecord(
+ { id: 99, login: "octocat", public_repos: 1, public_gists: 0, followers: 0, following: 0 } as never,
+ "2026-01-04"
+ );
+
+ assert.equal(stats.public_repos, 1, "the provider scalar counts only public user-owned repositories");
+ assert.equal(collected.length, 3, "the stream collects private and org repositories too");
+
+ // The load-bearing assertion: using the scalar as this stream's denominator
+ // would claim 2 of 3 collected repositories are missing, on a fully correct
+ // run. It is a different set, not a smaller measurement of the same set.
+ assert.notEqual(
+ stats.public_repos,
+ collected.length,
+ "public_repos must never be used as the repositories denominator — it measures a different set"
+ );
+});
diff --git a/packages/polyfill-connectors/connectors/gmail/all-mail-inventory.test.ts b/packages/polyfill-connectors/connectors/gmail/all-mail-inventory.test.ts
new file mode 100644
index 000000000..5e710d713
--- /dev/null
+++ b/packages/polyfill-connectors/connectors/gmail/all-mail-inventory.test.ts
@@ -0,0 +1,270 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Gmail receives an authoritative mailbox-wide message count on every run and
+ * used to throw it away: the SELECT that opens All Mail returns IMAP `EXISTS`,
+ * and the connector read `uidValidity`/`uidNext` off the same `MailboxObject`
+ * while ignoring the count sitting next to them. The `labels` record body still
+ * carries the fossil of that decision — `message_count: null`.
+ *
+ * These tests drive the real `runAllMailPasses` with a stubbed IMAP client and
+ * assert on the emitted protocol, so they fail if the total stops being bound,
+ * stops being validated, or stops being carried across runs.
+ *
+ * The contract deliberately does NOT make `EXISTS` the `messages`
+ * DETAIL_COVERAGE denominator. That fact is per-page by design — the runtime
+ * admits a bounded continuation only when a page reports
+ * `considered === covered` on same-page facts — so substituting a 140k-message
+ * mailbox total there would make every run read `partial` forever and break
+ * continuation outright. A wrong denominator is worse than none. The mailbox
+ * total is disclosed as its own fact instead, and the per-page denominator is
+ * left alone; `messagesPageCoverageIsStillPerPage` below pins that boundary.
+ */
+
+import assert from "node:assert/strict";
+import { mock, test } from "node:test";
+import type { FetchMessageObject, ImapFlow, ListResponse, MessageEnvelopeObject } from "imapflow";
+import { runAllMailPasses } from "./index.ts";
+import type { StreamRequest } from "./types.ts";
+
+const FROZEN_NOW = "2026-04-21T00:00:00.000Z";
+
+function makeAllMailMailbox(): ListResponse {
+ return {
+ delimiter: "/",
+ flags: new Set(["\\All"]),
+ listed: true,
+ name: "All Mail",
+ path: "[Gmail]/All Mail",
+ pathAsListed: "[Gmail]/All Mail",
+ parent: ["[Gmail]"],
+ parentPath: "[Gmail]",
+ specialUse: "\\All",
+ subscribed: true,
+ };
+}
+
+function makeRequested(streams: readonly string[]): Map {
+ return new Map(streams.map((name) => [name, { name }]));
+}
+
+function makeMsg(uid: number): FetchMessageObject {
+ const envelope: MessageEnvelopeObject = {
+ date: new Date("2026-04-20T10:00:00.000Z"),
+ subject: "Test subject",
+ from: [{ name: "Alice", address: "alice@example.com" }],
+ to: [{ name: "Bob", address: "bob@example.com" }],
+ cc: [],
+ bcc: [],
+ messageId: ``,
+ };
+ return {
+ seq: uid,
+ uid,
+ emailId: `gmmsgid-${uid}`,
+ threadId: "gmthrid-2222",
+ flags: new Set(["\\Seen"]),
+ labels: new Set(["\\Inbox"]),
+ envelope,
+ internalDate: new Date("2026-04-20T10:00:05.000Z"),
+ size: 1024,
+ } as FetchMessageObject;
+}
+
+interface RunOutcome {
+ inventory: Record | undefined;
+ messages: Record[];
+ state: Record | undefined;
+}
+
+/**
+ * Run one All Mail pass against a stubbed mailbox and return the protocol it
+ * emitted. `mailboxOverrides` is how each test injects the EXISTS value under
+ * examination — including the malformed values a real server should never send
+ * but which must fail closed rather than read as a complete mailbox.
+ */
+async function runPass(
+ mailboxOverrides: Record,
+ state: Record = {}
+): Promise {
+ const originalWrite = globalThis.process.stdout.write;
+ const protocolMessages: Record[] = [];
+ globalThis.process.stdout.write = ((data: string): boolean => {
+ if (typeof data === "string") {
+ try {
+ protocolMessages.push(JSON.parse(data) as Record);
+ } catch {
+ // Ignore non-protocol output.
+ }
+ }
+ return true;
+ }) as typeof process.stdout.write;
+
+ try {
+ const client: Pick = {
+ close: mock.fn(),
+ download: () => {
+ throw new Error("download must not be called without attachments");
+ },
+ fetchOne: () => {
+ throw new Error("fetchOne must not be called without bodies");
+ },
+ search: mock.fn(async () => []),
+ mailbox: {
+ delimiter: "/",
+ exists: 1200,
+ flags: new Set(),
+ path: "[Gmail]/All Mail",
+ uidNext: 1201,
+ uidValidity: 123n,
+ ...mailboxOverrides,
+ } as ImapFlow["mailbox"],
+ // biome-ignore lint/suspicious/useAwait: async generator is required by the ImapFlow fetch shape.
+ async *fetch() {
+ for (const uid of [1, 2]) {
+ yield makeMsg(uid);
+ }
+ },
+ };
+
+ await runAllMailPasses(client, makeAllMailMailbox(), state, {
+ emitRecord: async () => true,
+ emittedAt: FROZEN_NOW,
+ requested: makeRequested(["messages"]),
+ });
+ } finally {
+ globalThis.process.stdout.write = originalWrite;
+ }
+
+ return {
+ messages: protocolMessages,
+ inventory: protocolMessages.find((m) => m.type === "PROGRESS" && m.all_mail_inventory !== undefined),
+ state: protocolMessages.find((m) => m.type === "STATE" && m.stream === "messages"),
+ };
+}
+
+test("gmail all mail: the server-declared EXISTS is bound and disclosed, not discarded", async () => {
+ const { inventory } = await runPass({ exists: 1200 });
+
+ assert.ok(inventory, "a run must disclose the mailbox total the server handed it");
+ assert.deepEqual(inventory.all_mail_inventory, {
+ all_mail_exists: 1200,
+ backfilled_through_uid: 0,
+ forward_floor_uid: 1200,
+ historical_backfill_complete: false,
+ uidvalidity: 123,
+ });
+});
+
+test("gmail all mail: EXISTS is measured at the provider boundary, not derived from what was emitted", async () => {
+ // The pass emits exactly 2 message records but the mailbox holds 1200. If the
+ // total were ever derived from the emitted/collected count instead of read off
+ // the SELECT, this would report 2 and a 1198-message mailbox would read as
+ // fully accounted for — the exact defect this contract exists to prevent.
+ const { inventory } = await runPass({ exists: 1200 });
+ const disclosed = inventory?.all_mail_inventory as { all_mail_exists: number };
+
+ assert.equal(disclosed.all_mail_exists, 1200, "the total is the server's count, independent of the 2 emitted");
+ assert.notEqual(disclosed.all_mail_exists, 2, "the total must never collapse to the emitted count");
+});
+
+test("gmail all mail: the EXISTS total is carried on STATE so the next run can compare it", async () => {
+ const { state } = await runPass({ exists: 1200 });
+ const allMail = (state?.cursor as Record> | undefined)?.all_mail;
+
+ assert.ok(allMail, "the messages STATE carries an all_mail cursor");
+ assert.equal(allMail.exists, 1200, "the cursor persists the epoch's inventory size");
+ assert.equal(allMail.uidvalidity, 123, "the count is only meaningful alongside the epoch it was measured in");
+});
+
+test("gmail all mail: a missing EXISTS fails closed instead of reading as a complete mailbox", async () => {
+ // A silently absent total that defaults to success reproduces the bug being
+ // fixed, so absence must be louder than a wrong number, not quieter.
+ await assert.rejects(
+ () => runPass({ exists: undefined }),
+ /gmail_all_mail_exists_missing/,
+ "no EXISTS means no proof of inventory; the run must fail rather than assume"
+ );
+});
+
+test("gmail all mail: a non-numeric EXISTS fails closed", async () => {
+ await assert.rejects(() => runPass({ exists: "1200" }), /gmail_all_mail_exists_not_number/);
+});
+
+test("gmail all mail: a non-finite EXISTS fails closed", async () => {
+ await assert.rejects(() => runPass({ exists: Number.POSITIVE_INFINITY }), /gmail_all_mail_exists_not_finite/);
+});
+
+test("gmail all mail: a fractional EXISTS fails closed", async () => {
+ await assert.rejects(() => runPass({ exists: 12.5 }), /gmail_all_mail_exists_not_integer/);
+});
+
+test("gmail all mail: a negative EXISTS fails closed", async () => {
+ await assert.rejects(() => runPass({ exists: -1 }), /gmail_all_mail_exists_negative/);
+});
+
+test("gmail all mail: a shrinking mailbox within one UID epoch throws", async () => {
+ // Mirrors Jellyfin's decreasing-total guard. Inside a single UIDVALIDITY the
+ // UID space is stable, so a smaller count is deletion or a server bug — either
+ // way it is a fact about the data that must not pass silently.
+ const priorState = {
+ messages: { all_mail: { uidvalidity: 123, uidnext: 1201, forward_uidnext: 1201, exists: 1200 } },
+ };
+
+ await assert.rejects(
+ () => runPass({ exists: 900 }, priorState),
+ /gmail_all_mail_exists_decreased: 900 < 1200/,
+ "a mailbox that lost 300 messages must be surfaced, not absorbed"
+ );
+});
+
+test("gmail all mail: a growing mailbox within one UID epoch is normal and does not throw", async () => {
+ const priorState = {
+ messages: { all_mail: { uidvalidity: 123, uidnext: 1201, forward_uidnext: 1201, exists: 1000 } },
+ };
+ const { inventory } = await runPass({ exists: 1200 }, priorState);
+
+ assert.ok(inventory, "a run must disclose the mailbox total the server handed it");
+ assert.equal(
+ (inventory.all_mail_inventory as { all_mail_exists: number }).all_mail_exists,
+ 1200,
+ "new mail is the expected case and must not be mistaken for corruption"
+ );
+});
+
+test("gmail all mail: a UIDVALIDITY re-key does not read a lower count as loss", async () => {
+ // A UIDVALIDITY change means the server rebuilt the UID space. The old count
+ // describes a different space, so comparing across the boundary would turn
+ // every legitimate re-key into a spurious failure. The guard must scope its
+ // comparison to one epoch — this is the case that distinguishes a real
+ // decreasing-total check from a naive one.
+ const priorState = {
+ messages: { all_mail: { uidvalidity: 999, uidnext: 5000, forward_uidnext: 5000, exists: 4000 } },
+ };
+ const { inventory } = await runPass({ exists: 1200 }, priorState);
+
+ assert.ok(inventory, "a run must disclose the mailbox total the server handed it");
+ assert.equal(
+ (inventory.all_mail_inventory as { all_mail_exists: number }).all_mail_exists,
+ 1200,
+ "a re-key must read as a new epoch, never as a 2800-message loss"
+ );
+ assert.equal((inventory.all_mail_inventory as { uidvalidity: number }).uidvalidity, 123);
+});
+
+test("gmail all mail: the messages page coverage denominator stays per-page", async () => {
+ // Guards the boundary this design turns on. The runtime's bounded-continuation
+ // check requires same-page `considered === covered`; if a future change routed
+ // the mailbox-wide EXISTS into this fact, a 1200-message mailbox walked 2
+ // messages at a time would report 2/1200, read `partial` on every run forever,
+ // and lose the continuation proof. The mailbox total belongs beside this fact,
+ // not inside it.
+ const { messages } = await runPass({ exists: 1200 });
+ const coverage = messages.find((m) => m.type === "DETAIL_COVERAGE" && m.stream === "messages");
+
+ assert.ok(coverage, "the bounded page still proves its own coverage");
+ assert.equal(coverage.considered, 2, "the page denominator is the page, not the mailbox");
+ assert.equal(coverage.covered, 2);
+ assert.notEqual(coverage.considered, 1200, "the mailbox total must not be substituted here");
+});
diff --git a/packages/polyfill-connectors/connectors/gmail/index.ts b/packages/polyfill-connectors/connectors/gmail/index.ts
index bccc57f3e..af5bf6c83 100644
--- a/packages/polyfill-connectors/connectors/gmail/index.ts
+++ b/packages/polyfill-connectors/connectors/gmail/index.ts
@@ -56,7 +56,6 @@ import {
type BodyPartSelection,
bigintToCursor,
bigintToNumber,
- buildDeltaMessageRecord,
buildMessageBodyRecord,
buildMessageRecord,
buildThreadRecord,
@@ -172,7 +171,40 @@ function getReadline(): ReadlineInterface {
// control chars out of body text before encoding. The JSONL encoding
// itself — BigInt coercion + U+2028/U+2029 escaping — lives in
// `stringifyForJsonl`.
+// DONE is terminal on the wire, and nothing below this line may write after
+// it. The runtime's `handleMsg` latches the first DONE of any status and
+// throws `Connector emitted after DONE` on the next message, failing
+// the whole run as a protocol violation.
+//
+// Enforcing that here rather than at each terminal path is deliberate: the
+// terminal paths do not stop this process. `flushAndExit` only registers
+// listeners and returns (see `connector-exit.ts` — it waits for the runtime
+// to close stdin, up to 30 minutes), so after `fail()` or a rejection
+// handler emits DONE, the interrupted work is still on the stack. An
+// in-flight `for await` over an IMAP FETCH keeps iterating and keeps calling
+// `emitRecord`, and each of those writes lands after the DONE.
+//
+// That is what production hit: gmail runs failed `connector_protocol_violation
+// "Connector emitted RECORD after DONE"` while reporting more records emitted
+// than the runtime ever flushed — the surplus was written into a channel that
+// had already closed. Making every terminal path unwind perfectly would be a
+// standing obligation on code that mostly cannot see it coming; making the
+// write refuse is a property of the channel.
+//
+// Suppressed messages go to stderr, so a swallowed record is diagnosable
+// rather than silent. Emitting them would fail the run outright, which is
+// strictly worse: the records ingested before DONE are durable and the run's
+// own terminal status is already decided.
+let doneEmitted = false;
+
function emit(msg: EmittedMessage): Promise {
+ if (doneEmitted) {
+ process.stderr.write(`[gmail] suppressed ${String(msg.type)} after DONE\n`);
+ return Promise.resolve();
+ }
+ if (msg.type === "DONE") {
+ doneEmitted = true;
+ }
const line = stringifyForJsonl(sanitizeForJsonl(msg));
const ok = process.stdout.write(line);
if (ok) {
@@ -185,6 +217,17 @@ function emit(msg: EmittedMessage): Promise {
});
}
+/** Test-only reset of the terminal latch; the real process never reuses it. */
+export function __resetDoneLatchForTests(): void {
+ doneEmitted = false;
+}
+
+/** Test-only handle on the real stdout emitter, so the latch is exercised
+ * through the same function every production path writes through. */
+export function gmailEmitForTests(msg: EmittedMessage): Promise {
+ return emit(msg);
+}
+
function flushAndExit(code: number): void {
flushAndExitAfterRuntimeAck(code);
}
@@ -305,6 +348,18 @@ interface AttachmentHydrationFailure {
readonly stage: AttachmentHydrationFailureStage;
}
+/**
+ * A `failed` hydration's stage, if known, else the same `unclassified_failed`
+ * bucket already used by `attachment_hydration_failure_outcome` telemetry.
+ * This is the ONLY hydration-failure detail that ever reaches a DETAIL_GAP:
+ * a bounded category, never `AttachmentRecord.hydration_error` (raw IMAP/blob
+ * text that can embed hostnames, tokens, or URLs — see
+ * `buildAttachmentDetailGap`'s doc comment on why that string never crosses).
+ */
+function attachmentHydrationFailureClass(failure: AttachmentHydrationFailure | null): string {
+ return failure?.stage ?? "unclassified_failed";
+}
+
export interface AttachmentHydrationResult {
readonly failure: AttachmentHydrationFailure | null;
readonly record: AttachmentRecord;
@@ -389,12 +444,28 @@ export function formatAttachmentBackfillSummary(summary: AttachmentBackfillSumma
* - `failed` → `gapKeys` (a retryable detail gap to re-attempt next run).
* - `too_large` or `deferred` → unaccounted (required denominator only).
*/
+/** A failed attachment hydration, paired with why it failed (bounded, non-secret). */
+export interface FailedAttachmentRecord {
+ readonly failureClass: string;
+ readonly record: AttachmentRecord;
+}
+
export interface AttachmentDetailCoverage {
/**
* Failed attachment records, retained so the run can emit one matching
- * DETAIL_GAP per `gapKeys` entry.
+ * DETAIL_GAP per `gapKeys` entry. The host commit-gate credits a missing
+ * required key only when it is hydrated, optional-skipped, or backed by a
+ * durable pending DETAIL_GAP — `gap_keys` alone do not satisfy it. Each
+ * record's `id` is exactly the value that landed in `gapKeys`, keeping the
+ * gap's `record_key` and the coverage key a single source of truth.
+ *
+ * `failureClass` is the bounded, non-secret classification (see
+ * `attachmentHydrationFailureClass`) — the only hydration-failure detail
+ * that reaches the emitted DETAIL_GAP. Without it, a gap that keeps
+ * failing on every retry never records why: `attempt_count` climbs but
+ * `last_error_json` stays null forever (the 2026-08 Gmail 5-row defect).
*/
- failedRecords: AttachmentRecord[];
+ failedRecords: FailedAttachmentRecord[];
gapKeys: string[];
hydratedKeys: string[];
requiredKeys: string[];
@@ -410,8 +481,28 @@ export function makeAttachmentDetailCoverage(): AttachmentDetailCoverage {
* its terminal `hydration_status`. A `deferred` status (never hydrated this
* run) is still a considered key but has no terminal outcome bucket, so it
* counts only toward the denominator. Pure: mutates the passed accumulator.
+ *
+ * `failure` is the same `AttachmentHydrationFailure | null` the hydrator
+ * already computes for aggregate telemetry — passed through here so a
+ * `failed` outcome's DETAIL_GAP can carry a bounded, non-secret `error.class`
+ * instead of recording an attempt with no evidence of why it failed.
*/
-export function recordAttachmentCoverage(coverage: AttachmentDetailCoverage, record: AttachmentRecord): void {
+export function recordAttachmentCoverage(
+ coverage: AttachmentDetailCoverage,
+ record: AttachmentRecord,
+ failure: AttachmentHydrationFailure | null = null
+): void {
+ // The runtime rejects a DETAIL_COVERAGE whose required_keys repeats a key.
+ // The same attachment id can legitimately reach this accumulator twice in a
+ // run -- a message re-observed across pages, or a retry re-walking a
+ // partially hydrated thread -- so pushing unconditionally turns an ordinary
+ // duplicate observation into a hard run failure ("invalid
+ // DETAIL_COVERAGE.required_keys: duplicate key"), which is what took Gmail
+ // out on 2026-08-17. Record each key once; the terminal-outcome buckets
+ // below are already keyed by the same id and stay consistent with it.
+ if (coverage.requiredKeys.includes(record.id)) {
+ return;
+ }
coverage.requiredKeys.push(record.id);
switch (record.hydration_status) {
case "hydrated":
@@ -419,7 +510,10 @@ export function recordAttachmentCoverage(coverage: AttachmentDetailCoverage, rec
return;
case "failed":
coverage.gapKeys.push(record.id);
- coverage.failedRecords.push(record);
+ // Retain the record so a matching DETAIL_GAP is emitted for this key.
+ // `gap_keys` on DETAIL_COVERAGE are not enough on their own: the host
+ // commit-gate requires a durable pending DETAIL_GAP to credit the key.
+ coverage.failedRecords.push({ failureClass: attachmentHydrationFailureClass(failure), record });
return;
case "too_large":
case "deferred":
@@ -436,10 +530,8 @@ export function recordAttachmentCoverage(coverage: AttachmentDetailCoverage, rec
/**
* Build the per-run attachments DETAIL_COVERAGE after the detail lane settles.
- * A requested attachments pass that scans the parent `messages` boundary and
- * finds zero attachment parts has a real empty denominator: `required_keys: []`
- * means "nothing owed", not "unknown". The list cursor that anchors this
- * detail pass lives on `messages`, so that is the `state_stream`.
+ * The list cursor that anchors this detail pass lives on `messages`, so that is
+ * the `state_stream`.
* Reference-only: this reuses DETAIL_COVERAGE without promoting it to portable
* protocol.
*
@@ -459,15 +551,69 @@ export function buildAttachmentDetailCoverageMessage(coverage: AttachmentDetailC
}
/**
- * Emit the per-run attachments DETAIL_COVERAGE after the detail lane settles.
+ * Whether this run walked a boundary that can support an attachments coverage
+ * claim at all.
+ *
+ * `attachments` has no enumeration of its own: an attachment is only ever
+ * discovered by decoding the BODYSTRUCTURE of a message the run already
+ * fetched. Its denominator is therefore inherited from whatever slice of the
+ * mailbox the run walked — and on an ordinary scheduled run that slice is the
+ * incremental forward window plus the MODSEQ delta, i.e. a CHANGE FEED, not an
+ * inventory.
+ *
+ * That is exactly the apple_contacts `contactsBoundaryEstablished` situation
+ * (`connectors/apple_contacts/index.ts`), and it needs the same answer. A quiet
+ * 16-minute run that observes no new mail decodes no attachment parts and would
+ * otherwise emit `considered: 0, covered: 0` — numbers that are trivially
+ * self-consistent and read downstream as "this stream is proven complete",
+ * when the honest statement is "this run proved nothing about the mailbox".
+ * Worse, the two are indistinguishable to the gate: a stream that has walked
+ * almost nothing and a stream that is genuinely fully collected emit the
+ * identical fact. Withholding is what keeps those two apart.
+ *
+ * The one boundary that does justify a whole-stream claim is a COMPLETED
+ * historical `messages` walk: the historical pass fetches every UID below the
+ * forward floor and `processMessage` decodes attachments for each one, so once
+ * `messages.backfill.completed_at` is set, every message in All Mail has had
+ * its attachment parts enumerated. Until then the mailbox below the resume
+ * point is simply unwalked, and no count taken above it describes it.
+ *
+ * Emitting nothing leaves the stream honestly unproven (the coherence
+ * contract's `checkpoint_only`/`no_proof_strategy` -> axis `unknown`) rather
+ * than falsely complete. `unknown` is a worse-looking verdict than a green
+ * `0/0` and that is the point: it is the true one, and it is the verdict an
+ * owner can act on.
+ */
+export function attachmentsCoverageBoundaryEstablished(session: { messagesBackfill: MessagesBackfillCursor }): boolean {
+ return typeof session.messagesBackfill.completed_at === "string" && session.messagesBackfill.completed_at !== "";
+}
+
+/**
+ * Emit the per-run attachments DETAIL_COVERAGE after the detail lane settles —
+ * but only when the run established a boundary that can support the claim
+ * (see `attachmentsCoverageBoundaryEstablished`).
*
* Extracted from `runAllMailPasses` to keep that orchestrator under the
* cognitive-complexity ceiling (authoring guide §"Rules the tooling enforces").
*/
-async function emitAttachmentDetailCoverage(coverage: AttachmentDetailCoverage | undefined): Promise {
+async function emitAttachmentDetailCoverage(
+ coverage: AttachmentDetailCoverage | undefined,
+ boundaryEstablished: boolean
+): Promise {
if (!coverage) {
return;
}
+ if (!boundaryEstablished) {
+ // Deliberately not a SKIP_RESULT: nothing was skipped by policy. The run
+ // simply has no denominator worth reporting, and says so by staying quiet.
+ await emit({
+ type: "PROGRESS",
+ stream: "attachments",
+ message:
+ "Withholding attachments coverage: the historical messages walk has not completed, so this run's attachment counts describe an incremental window, not the mailbox",
+ });
+ return;
+ }
await emit(buildAttachmentDetailCoverageMessage(coverage));
}
@@ -488,11 +634,23 @@ async function emitAttachmentDetailCoverage(coverage: AttachmentDetailCoverage |
* order-detail gap; retrying next run is the honest, non-destructive default.
*
* Reference-only and bounded: only opaque message and part identifiers cross
- * (X-GM-MSGID, the BODYSTRUCTURE part index, and the attachment id). No
- * filename, content, blob bytes, raw error text, tokens, cookies, URLs, request
- * bodies, or payload snippets are carried.
+ * (X-GM-MSGID, the BODYSTRUCTURE part index, and the attachment id), plus —
+ * when `failureClass` is supplied — a bounded, non-secret failure category
+ * (e.g. `imap_download_failed`, `blob_upload_http_5xx`, `unclassified_failed`;
+ * see `attachmentHydrationFailureClass`). No filename, content, blob bytes,
+ * raw error text, tokens, cookies, URLs, request bodies, or payload snippets
+ * are ever carried — `AttachmentRecord.hydration_error` (which CAN embed
+ * that raw text) never crosses this boundary, by construction: only the
+ * caller-supplied category string can reach `error.class`.
+ *
+ * Without a failure class, a repeatedly-retried gap accrues `attempt_count`
+ * on every run but never records why it keeps failing — `last_error_json`
+ * stays null forever, which is indistinguishable from "never attempted" and
+ * blocks the stream from ever proving its remaining gaps are impossible
+ * (2026-08 Gmail: 5 `attachments` gaps reached terminal status with 37-117
+ * attempts and a permanently-null `last_error_json`).
*/
-export function buildAttachmentDetailGap(attachment: AttachmentRecord): DetailGapMessage {
+export function buildAttachmentDetailGap(attachment: AttachmentRecord, failureClass?: string): DetailGapMessage {
return buildDetailGap({
stream: "attachments",
parentStream: "messages",
@@ -504,6 +662,7 @@ export function buildAttachmentDetailGap(attachment: AttachmentRecord): DetailGa
part_index: attachment.part_index,
attachment_id: attachment.id,
},
+ ...(failureClass ? { error: { class: failureClass } } : {}),
});
}
@@ -586,7 +745,7 @@ async function emitAttachmentRecords(
// Record the outcome BEFORE emitting so the coverage denominator counts
// every attempt even if the emit is scope-filtered downstream.
if (deps.attachmentCoverage) {
- recordAttachmentCoverage(deps.attachmentCoverage, hydrated);
+ recordAttachmentCoverage(deps.attachmentCoverage, hydrated, hydration.failure);
}
const emitted = await deps.emitRecord("attachments", { ...hydrated });
// Only `hydrated` may acknowledge a served gap as recovered. Unaccounted
@@ -619,8 +778,8 @@ async function emitAttachmentDetailGaps(coverage: AttachmentDetailCoverage | und
if (!coverage) {
return;
}
- for (const attachment of coverage.failedRecords) {
- await emit(buildAttachmentDetailGap(attachment));
+ for (const failed of coverage.failedRecords) {
+ await emit(buildAttachmentDetailGap(failed.record, failed.failureClass));
}
}
@@ -802,6 +961,45 @@ export async function emitMessagesPass(
return { considered: metas.length, covered: count };
}
+/**
+ * Disclose the mailbox-wide inventory total against this run's walk boundary.
+ *
+ * Gmail's `messages` DETAIL_COVERAGE is deliberately PER-PAGE: the runtime's
+ * `isHealthyBoundedContinuation` admits a bounded page only when
+ * `considered === covered` on same-page facts, so the page denominator must
+ * stay the page. Overwriting it with the mailbox-wide `EXISTS` would make every
+ * run of a 140k-message mailbox read `partial` forever and would break the
+ * continuation contract outright — a wrong denominator, not a better one.
+ *
+ * The honest place for the provider total is therefore a SEPARATE fact: the
+ * server-declared inventory size, plus how much of the UID space this
+ * connector's two cursors have actually claimed. `EXISTS` is validated
+ * fail-closed in `validateExistsTotal` before this point, so a missing or
+ * malformed total has already thrown and can never reach here as a silent
+ * "complete". What this adds is visibility: the number the provider asserts,
+ * next to the boundary we have walked to.
+ */
+async function emitAllMailInventoryDisclosure(
+ emitFn: (msg: EmittedMessage) => Promise,
+ session: AllMailSession,
+ historicalCursor: MessagesBackfillCursor,
+ forwardFloorUid: number
+): Promise {
+ const backfilledThroughUid = historicalCursor.backfilled_through_uid ?? 0;
+ await emitFn({
+ type: "PROGRESS",
+ stream: "messages",
+ message: `All Mail reports ${session.existsTotal} messages; historical walk is through UID ${backfilledThroughUid} of ${forwardFloorUid}`,
+ all_mail_inventory: {
+ all_mail_exists: session.existsTotal,
+ backfilled_through_uid: backfilledThroughUid,
+ forward_floor_uid: forwardFloorUid,
+ historical_backfill_complete: historicalCursor.completed_at !== null,
+ uidvalidity: session.uidvalidityNum,
+ },
+ });
+}
+
/** Record a bounded terminal outcome so a poison UID cannot replay forever. */
async function emitHistoricalMessageSkip(deps: PerMessageDeps, uid: number | undefined): Promise {
await deps.emitProtocol({
@@ -1049,15 +1247,57 @@ async function findAllMailbox(client: ImapFlow): Promise {
interface AllMailSession {
attachmentBackfill: AttachmentAllMailCursor;
+ /**
+ * IMAP `EXISTS` for All Mail: the server's own count of the messages in the
+ * mailbox, taken from the SELECT this run already performed. This is the
+ * only mailbox-wide inventory total Gmail hands us, and it is measured at
+ * the provider boundary — before a single UID is walked and independently
+ * of what any pass emits.
+ */
+ existsTotal: number;
fullResync: boolean;
highestModseqCursor: number | string | null;
messagesBackfill: MessagesBackfillCursor;
+ priorExistsTotal: number | undefined;
priorModseq: number | string | null | undefined;
priorUidnext: number;
uidnext: number | undefined;
uidvalidityNum: number;
}
+/**
+ * Validate the IMAP `EXISTS` count for All Mail and bind it as the mailbox's
+ * inventory total. Fail closed on missing/malformed, mirroring Jellyfin's
+ * `validateTotalRecordCount`: a silently absent total that reads as success is
+ * precisely the bug this contract exists to prevent.
+ *
+ * `priorTotal` is compared only WITHIN a UIDVALIDITY epoch (the caller passes
+ * `undefined` across a re-key). A decrease inside one epoch is real deletion or
+ * a server bug and must throw rather than pass quietly. Across a re-key the UID
+ * space was rebuilt, so a different count is expected and is not loss.
+ */
+function validateExistsTotal(value: unknown, priorTotal: number | undefined): number {
+ if (value === undefined || value === null) {
+ throw new Error("gmail_all_mail_exists_missing: SELECT reported no EXISTS for All Mail");
+ }
+ if (typeof value !== "number") {
+ throw new Error("gmail_all_mail_exists_not_number");
+ }
+ if (!Number.isFinite(value)) {
+ throw new Error("gmail_all_mail_exists_not_finite");
+ }
+ if (!Number.isInteger(value)) {
+ throw new Error("gmail_all_mail_exists_not_integer");
+ }
+ if (value < 0) {
+ throw new Error("gmail_all_mail_exists_negative");
+ }
+ if (priorTotal !== undefined && value < priorTotal) {
+ throw new Error(`gmail_all_mail_exists_decreased: ${value} < ${priorTotal}`);
+ }
+ return value;
+}
+
/**
* Narrow the MailboxObject + prior state into the (UIDVALIDITY, cursor,
* resync-flag) triple the fetch loop needs. Returns null when UIDVALIDITY
@@ -1084,6 +1324,15 @@ function deriveAllMailSession(mailbox: MailboxObject, state: Record {
const priorEnd = args.prior.backfilled_through_uid ?? 0;
@@ -1878,7 +2185,7 @@ async function settleServedAttachmentRecoveryAttempt(
): Promise {
const hydrated = hydration.record;
if (deps.attachmentCoverage) {
- recordAttachmentCoverage(deps.attachmentCoverage, hydrated);
+ recordAttachmentCoverage(deps.attachmentCoverage, hydrated, hydration.failure);
}
const emitted = await deps.emitRecord("attachments", { ...hydrated });
if (emitted && hydrated.hydration_status === "hydrated") {
@@ -2837,6 +3144,42 @@ interface ImapDownloadResponse {
meta?: ImapDownloadMeta;
}
+/**
+ * Download one attachment PART's bytes.
+ *
+ * `expectedSize` is deliberately taken from BODYSTRUCTURE (`attachment.size_bytes`)
+ * and NOT from the download response's `meta.expectedSize`.
+ *
+ * imapflow populates `meta.expectedSize` from the FETCH `RFC822.SIZE` item
+ * (`lib/imap-flow.js`: `meta = { expectedSize: response.size }`, where `size`
+ * is requested as the `RFC822.SIZE` atom in `lib/commands/fetch.js`).
+ * RFC822.SIZE is the size of the ENTIRE MESSAGE — every part, every MIME
+ * header, every boundary — not of the part being downloaded. It is the same
+ * number for every part of a multipart message.
+ *
+ * Feeding a message-scoped size into a per-attachment cap made a message
+ * reject ALL of its attachments whenever their SUM crossed the cap, even
+ * though no single one came close. Observed live on the owner's mailbox:
+ * 68 attachments marked `too_large` across 18 messages, of which only 2 were
+ * genuinely over the 25 MiB cap; one 3,080-byte attachment was rejected as
+ * "29830196 > 26214400 bytes". Two parts of one message recorded the byte-for-byte
+ * identical "observed" size — impossible for real per-part sizes, and the tell
+ * that the number was never the part's.
+ *
+ * That number is not just a wrong skip: `isProvenUnfillableGap`
+ * (`server/connector-gap-classification.ts`) parses this exact
+ * "exceeds max size: > " text as DURABLE PROOF that an item
+ * is permanently uncollectable. A message-scoped size therefore manufactured
+ * per-item impossibility proofs for items that are collectible, which is the
+ * fabricated-evidence failure that predicate exists to refuse.
+ *
+ * BODYSTRUCTURE's per-part `size` is the only per-part size IMAP gives us
+ * before transfer, so it is the only honest pre-flight number. When it is
+ * absent we report `null` (unknown) rather than substituting a
+ * message-scoped stand-in: `enforceMaxBytes` still counts real bytes
+ * mid-stream, so an under-reported or missing size is caught by observation
+ * instead of by a guess.
+ */
export async function fetchAttachmentPart(
client: Pick,
msg: FetchMessageObject,
@@ -2850,7 +3193,7 @@ export async function fetchAttachmentPart(
})) as ImapDownloadResponse;
return {
content: response.content,
- expectedSize: typeof response.meta?.expectedSize === "number" ? response.meta.expectedSize : attachment.size_bytes,
+ expectedSize: attachment.size_bytes,
mimeType: response.meta?.contentType || attachment.content_type || DEFAULT_ATTACHMENT_MIME_TYPE,
};
}
@@ -2893,12 +3236,35 @@ export function validateAttachmentHydrationPreflight(args: {
// ─── Delta pass (flag/label changes since priorModseq) ──────────────────
-async function runDeltaPass(
+/**
+ * Emit flag/label changes for messages modified since `priorModseq`.
+ *
+ * The envelope is fetched alongside the flags. That is not an optimization —
+ * it is what makes the pass safe. PDPP records are whole-document upserts
+ * (`records` is keyed `UNIQUE(connector_instance_id, stream, record_key)` and
+ * ingest replaces `record_json`), so emitting a partial `messages` record
+ * overwrites the stored row rather than merging into it. An envelope-free
+ * delta record therefore nulls `subject`, `from_email`, `date`, `size_bytes`,
+ * and `snippet` on a message that had them, and sets `received_at` to the run
+ * clock — silently destroying already-collected history every time a label or
+ * a `\Seen` flag changes.
+ *
+ * A message whose envelope the server does not return is skipped rather than
+ * emitted partially: losing a flag update is recoverable on the next pass,
+ * losing the envelope is not.
+ *
+ * `fetchBodiesFn` is the same seam `processMessage` uses. Called with
+ * `wantBodies: false` it fetches at most `SNIPPET_FETCH_MAX_BYTES` of the
+ * plain part — enough to rebuild `snippet`, without touching the
+ * externally-throttled full-body/attachment path.
+ */
+export async function runDeltaPass(
client: Pick,
session: AllMailSession,
requested: Map,
emitRecord: EmitRecordFn,
- receivedAtFallback: string
+ receivedAtFallback: string,
+ fetchBodiesFn: FetchBodiesFn
): Promise {
if (session.fullResync || session.priorModseq === undefined || session.priorModseq === null) {
return;
@@ -2909,41 +3275,81 @@ async function runDeltaPass(
type: "PROGRESS",
message: `Fetching flag/label deltas since modseq=${String(priorModseq)}`,
});
+ // `envelope`/`internalDate`/`size`/`bodyStructure` ride along with the flags
+ // so the emitted record is whole. See this function's doc comment: a partial
+ // record is a destructive upsert, not a cheap one. BODYSTRUCTURE is metadata
+ // only — it is what `has_attachments` is derived from, and fetching it does
+ // not pull attachment or body content.
const deltaQuery: ExtendedFetchQuery = {
uid: true,
flags: true,
labels: true,
threadId: true,
emailId: true,
- envelope: false,
+ envelope: true,
+ internalDate: true,
+ size: true,
+ bodyStructure: true,
};
+ // Phase A: drain the delta FETCH completely before issuing any other IMAP
+ // command. Phase B: fetch snippets and emit.
+ //
+ // The split is required, not stylistic — it is the same rule the message
+ // pass states at its own Phase A/B boundary: imapflow multiplexes one
+ // command at a time over a single connection, so a nested command issued
+ // while the outer iterator is still open hangs that iterator. Calling
+ // `fetchBodiesFn` (a `fetchOne`) inside the `for await` did exactly that.
+ //
+ // `fetchBodies` swallows its own errors, so the nested call surfaced not as
+ // a body-fetch failure but as a wedged connection: the run stopped making
+ // progress after "Fetching flag/label deltas", sat until a timeout, and
+ // then tripped the runtime's post-DONE guard while the abandoned iterator
+ // drained. Draining first keeps the envelope guarantee below intact and
+ // costs only the metadata already held in memory.
+ const deltaMetas: FetchMessageObject[] = [];
for await (const msg of client.fetch("1:*", deltaQuery, {
uid: true,
changedSince: priorModseqBig,
})) {
- const gmMsgid = String(msg.emailId ?? "");
- if (!gmMsgid) {
+ if (!requested.has("messages")) {
continue;
}
- // Flag/label delta update: emit a tombstone-free upsert of the message
- // envelope (minimal fields since envelope not re-fetched). For now, we
- // emit a RECORD with the same id so the RS upserts flag/label state.
- // Note: PDPP records are "whole-document" upserts in the current RS,
- // so this delta path is effectively a full re-fetch. Simpler: mark
- // this path as "only flags" by emitting the fields we have plus nulls.
- // For robustness, let's actually re-fetch envelope in v2. For v1, emit
- // flags only.
- if (!requested.has("messages")) {
+ if (!String(msg.emailId ?? "")) {
+ continue;
+ }
+ // No envelope means no safe record to write. Skipping preserves the
+ // stored row; emitting would blank it.
+ if (!msg.envelope) {
+ continue;
+ }
+ deltaMetas.push(msg);
+ }
+
+ for (const msg of deltaMetas) {
+ const gmMsgid = String(msg.emailId ?? "");
+ const env = msg.envelope;
+ if (!env) {
continue;
}
+ const receivedAt = perMessageInternalDateToIso(msg.internalDate, () => receivedAtFallback);
+ // Bounded snippet-only body read, exactly as the forward pass does for a
+ // messages-without-bodies scope. Sequential by necessity: this is one IMAP
+ // command at a time on a connection that is not concurrent.
+ const { snippet } = await fetchBodiesFn(msg, selectBodyParts(msg.bodyStructure, false), false, true);
await emitRecord(
"messages",
- buildDeltaMessageRecord({
+ buildMessageRecord({
+ attachmentsCount: decodeBodystructureForAttachments(msg.bodyStructure, gmMsgid, receivedAt).length,
+ dateHeader: env.date ? new Date(env.date).toISOString() : null,
+ envelope: env,
flagsArr: toFlagsArray(msg.flags),
gmMsgid,
gmThrid: String(msg.threadId ?? ""),
labels: toLabelsArray(msg.labels),
- receivedAtFallback,
+ rawHeaders: msg.headers,
+ receivedAt,
+ sizeBytes: typeof msg.size === "number" ? msg.size : null,
+ snippet,
})
);
}
@@ -3101,8 +3507,21 @@ export async function runAllMailPasses(
deps.requested.has("attachments") ||
deps.requested.has("threads");
const forwardFetchRange = selectAllMailFetchRange(session, deps.requested);
- const historicalTargetUid =
- session.messagesBackfill.target_uid ?? Math.max(0, (session.uidnext ?? session.priorUidnext) - 1);
+ // Where the forward walk resumes on the NEXT run, and therefore the last UID
+ // it will not fetch. The historical walk must own everything up to here or
+ // the interval between the two belongs to neither (see
+ // `resolveMessagesBackfillTargetUid`).
+ //
+ // On a full resync there is no forward range at all: the forward watermark
+ // below is written from the live `uidnext` without a single UID having been
+ // walked, so the ENTIRE mailbox below it is historical work and the ceiling
+ // must say so. Reading the same `session.uidnext` the watermark is written
+ // from is what keeps the two pointers describing one space.
+ const forwardFloorUid = Math.max(0, (session.uidnext ?? session.priorUidnext) - 1);
+ const historicalTargetUid = resolveMessagesBackfillTargetUid({
+ forwardFloorUid,
+ prior: session.messagesBackfill,
+ });
const historicalCursor: MessagesBackfillCursor = {
...session.messagesBackfill,
backfilled_through_uid: session.messagesBackfill.backfilled_through_uid ?? 0,
@@ -3186,10 +3605,12 @@ export async function runAllMailPasses(
recoveryOnly: true,
session,
});
- // Recovery-only mode still reports attachment evidence for the served
- // gaps it actually touched, but it returns before the ordinary Gmail
- // walk and cursor advancement.
- await emitAttachmentDetailCoverage(attachmentCoverage);
+ // Recovery-only mode touches exactly the served gaps it was handed and
+ // returns before the ordinary Gmail walk, so it never establishes a
+ // mailbox boundary — its counts describe a retry list, not an inventory.
+ // It still emits its DETAIL_GAPs below; only the coverage CLAIM is
+ // withheld (see `attachmentsCoverageBoundaryEstablished`).
+ await emitAttachmentDetailCoverage(attachmentCoverage, false);
await emitAttachmentDetailGaps(attachmentCoverage);
}
return;
@@ -3287,6 +3708,7 @@ export async function runAllMailPasses(
stream: "messages",
message: `Collected ${metas.length} headers; beginning body pass`,
});
+ await emitAllMailInventoryDisclosure(emit, session, historicalCursor, forwardFloorUid);
const perMessageDeps: PerMessageDeps = {
...(attachmentCoverage ? { attachmentCoverage } : {}),
emitProtocol: emit,
@@ -3308,18 +3730,22 @@ export async function runAllMailPasses(
historicalMetas
);
const forwardMessageCoverage = await emitMessagesPass(perMessageDeps, forwardMetas);
- if (deps.requested.has("message_bodies")) {
- await emit(
- buildDetailCoverageMessage({
- considered: historicalMessageCoverage.considered + forwardMessageCoverage.considered,
- covered: historicalMessageCoverage.covered + forwardMessageCoverage.covered,
- hydratedKeys: [],
- requiredKeys: [],
- stateStream: "messages",
- stream: "message_bodies",
- })
- );
- }
+ // `message_bodies` deliberately emits NO DETAIL_COVERAGE. The manifest
+ // declares it `state_stream: messages`, i.e. a static single-parent detail
+ // stream, and such a stream's checkpoint status is projected from the
+ // declared parent's own commit outcome — the runtime rejects the run
+ // outright if it emits coverage of its own
+ // (`validateDetailCoverageAgainstManifest`).
+ //
+ // It could not honestly emit one anyway. The numbers available here are the
+ // *parent* pass's considered/covered — how many MESSAGES were walked, not
+ // how many bodies were hydrated against a per-key denominator. Re-reporting
+ // the parent's counts under the body stream's name is the `covered ==
+ // considered` fabrication this codebase has worked to eliminate: it would
+ // claim every walked message proved a body, including the ones whose body
+ // fetch was skipped or failed. `attachments` is the contrast — it earns its
+ // coverage from a real attempt-per-key tally (see
+ // `emitAttachmentDetailCoverage`), which is why it may emit at all.
await runAttachmentBackfillAndRecoveryPass({
allMail,
@@ -3339,7 +3765,7 @@ export async function runAllMailPasses(
// record (primary pass + historical backfill) has settled and before the
// messages STATE cursor commits — the ordering the progress-evidence
// contract expects (records, then DETAIL_COVERAGE, then STATE).
- await emitAttachmentDetailCoverage(attachmentCoverage);
+ await emitAttachmentDetailCoverage(attachmentCoverage, attachmentsCoverageBoundaryEstablished(session));
// Then one matching DETAIL_GAP per failed attachment, so the commit-gate can
// credit each gap_keys entry against a durable pending gap. Without this the
// gate aborts an otherwise-successful run and the messages cursor never
@@ -3347,14 +3773,35 @@ export async function runAllMailPasses(
await emitAttachmentDetailGaps(attachmentCoverage);
// Pass 2: detect flag/label changes on already-seen messages (incremental only)
- await runDeltaPass(client, session, deps.requested, deps.emitRecord, deps.emittedAt);
+ await runDeltaPass(client, session, deps.requested, deps.emitRecord, deps.emittedAt, fetchBodiesBound);
if (messageHistoryRequested && historicalFetchRange) {
const historicalPageEndUid = Number(historicalFetchRange.split(":")[1]);
+ // Sums BOTH passes, like the `message_bodies` DETAIL_COVERAGE above: the
+ // forward pass runs in the same call to `runAllMailPasses` and emits its
+ // own `messages` records via the same shared `emitRecord`, so the raw
+ // collected-record count already includes them. Reporting only
+ // `historicalMessageCoverage` undercounted the denominator against that
+ // total every scheduled run with new mail waiting alongside a pending
+ // historical backfill.
+ //
+ // Both emissions below MUST read these same two numbers. The runtime's
+ // `isHealthyBoundedContinuation` accepts a bounded page only when the
+ // continuation's considered/covered are identical to the DETAIL_COVERAGE
+ // fact's — it binds a continuation to complete *same-page* facts, and the
+ // summing above is what defines "the page" here. Feeding the continuation
+ // historical-only counts desyncs the pair by exactly the forward-pass
+ // count, the identity check fails, and the stream degrades to a
+ // retryable_gap instead of deriving complete. The sibling `threads`
+ // emission never desyncs precisely because it feeds one variable to both.
+ const messagesCoverage = {
+ considered: historicalMessageCoverage.considered + forwardMessageCoverage.considered,
+ covered: historicalMessageCoverage.covered + forwardMessageCoverage.covered,
+ };
await emit(
buildDetailCoverageMessage({
- considered: historicalMessageCoverage.considered,
- covered: historicalMessageCoverage.covered,
+ considered: messagesCoverage.considered,
+ covered: messagesCoverage.covered,
hydratedKeys: [],
requiredKeys: [],
stateStream: "messages",
@@ -3364,8 +3811,8 @@ export async function runAllMailPasses(
if (historicalPageEndUid < historicalTargetUid) {
await emitHistoricalContinuationSkip(emit, "messages", {
boundary: String(historicalCursor.uidvalidity),
- considered: historicalMessageCoverage.considered,
- covered: historicalMessageCoverage.covered,
+ considered: messagesCoverage.considered,
+ covered: messagesCoverage.covered,
slice_start: Number(historicalFetchRange.split(":")[0]),
slice_end: historicalPageEndUid,
});
@@ -3388,6 +3835,11 @@ export async function runAllMailPasses(
prior: historicalCursor,
});
} else {
+ // A completed walk with no fetch range left. This is reachable only when
+ // the ceiling did NOT move — `historicalTargetUid` is raised before the
+ // range is selected, so any reopened band yields a non-null range and is
+ // handled by the branch above. With an unchanged ceiling this cursor is
+ // already at its target, so carrying it forward is exact.
nextMessagesBackfill = historicalCursor;
}
}
@@ -3405,6 +3857,9 @@ export async function runAllMailPasses(
uidnext: nextUidnext,
forward_uidnext: nextForwardUidnext,
highest_modseq: session.highestModseqCursor ?? null,
+ // Carry the mailbox's own EXISTS forward so the next run in this epoch
+ // can prove the inventory did not shrink underneath us.
+ exists: session.existsTotal,
},
...(nextMessagesBackfill ? { backfill: nextMessagesBackfill } : {}),
},
diff --git a/packages/polyfill-connectors/connectors/gmail/integration.test.ts b/packages/polyfill-connectors/connectors/gmail/integration.test.ts
index c6c744db7..ef9186f32 100644
--- a/packages/polyfill-connectors/connectors/gmail/integration.test.ts
+++ b/packages/polyfill-connectors/connectors/gmail/integration.test.ts
@@ -32,6 +32,7 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
import { Readable } from "node:stream";
import { mock, test } from "node:test";
import type {
@@ -46,6 +47,7 @@ import { buildFullScanCoverageMessage } from "../../src/connector-runtime.ts";
import { ReferenceBlobUploadFailure, runtimeBlobUploadAvailable } from "../../src/reference-blob-uploader.ts";
import { type EmittedRecord, makeRecordingEmit, type RecordedEvent } from "../../src/test-harness.ts";
import {
+ __resetDoneLatchForTests,
ATTACHMENT_BACKFILL_PAGE_DEFAULT_BYTES,
ATTACHMENT_BACKFILL_PAGE_MAX_BYTES,
ATTACHMENT_BACKFILL_PAGE_MIN_BYTES,
@@ -58,6 +60,7 @@ import {
addAttachmentBackfillRecordToSummary,
advanceMessagesBackfillCursor,
attachmentBackfillPageByteBudget,
+ attachmentsCoverageBoundaryEstablished,
buildAttachmentDetailCoverageMessage,
buildAttachmentDetailGap,
buildAttachmentTransferProgressMessage,
@@ -71,7 +74,9 @@ import {
emitMessagesPass,
type FetchBodiesFn,
type FetchedBodies,
+ fetchAttachmentPart,
formatAttachmentBackfillSummary,
+ gmailEmitForTests,
type HydrateAttachmentFn,
isoToImapDate,
makeAttachmentDetailCoverage,
@@ -90,8 +95,10 @@ import {
resolveGmailAddressFromEnv,
resolveGmailPasswordFromEnv,
resolveMaxAttachmentBytes,
+ resolveMessagesBackfillTargetUid,
runAllMailPasses,
runAttachmentBackfillAndRecoveryPass,
+ runDeltaPass,
selectAllMailFetchRange,
selectAttachmentBackfillFetchRange,
selectMessagesBackfillFetchRange,
@@ -542,7 +549,7 @@ test("processMessage: a served gap whose attachment fails hydration AGAIN is nev
"the re-failed attachment must be a retryable gap key"
);
assert.deepEqual(
- attachmentCoverage.failedRecords.map((r) => r.id),
+ attachmentCoverage.failedRecords.map((r) => r.record.id),
["gmmsgid-1111:2"],
"the re-failed attachment must be retained so a fresh DETAIL_GAP is emitted for it"
);
@@ -1491,6 +1498,150 @@ test("selectMessagesBackfillFetchRange: first and later pages are bounded UID ra
);
});
+/**
+ * The reopening-band guards.
+ *
+ * `backfill.target_uid` and `all_mail.forward_uidnext` split ONE UID space, so
+ * they must meet: `target_uid + 1 >= forward_uidnext`. The forward watermark
+ * climbs on every run that sees new mail; when the ceiling merely copied itself
+ * forward the interval between them reopened continuously and grew without
+ * bound. Live evidence: a 297-UID band swallowed two days of mail, was repaired
+ * to 0, and measured 2 then 3 within minutes as new mail arrived.
+ *
+ * Each behavior is pinned separately below so a mutation to one guard reddens
+ * on its own rather than being masked by a sibling.
+ */
+test("resolveMessagesBackfillTargetUid: the ceiling rises to meet the forward watermark", () => {
+ // The mechanism defect: a frozen ceiling under a climbing watermark.
+ assert.equal(
+ resolveMessagesBackfillTargetUid({
+ forwardFloorUid: 324_022,
+ prior: { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 },
+ }),
+ 324_022,
+ "a watermark that moved past the ceiling must pull the ceiling up, or the gap reopens every run"
+ );
+});
+
+test("resolveMessagesBackfillTargetUid: the ceiling never falls, so backfill progress is never rewound", () => {
+ // A ceiling that could fall would strand `backfilled_through_uid` above its
+ // own target and re-open a finished walk. On the live instance the walk is
+ // ~150k UIDs deep; rewinding would re-fetch every one of them.
+ assert.equal(
+ resolveMessagesBackfillTargetUid({
+ forwardFloorUid: 900,
+ prior: { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 },
+ }),
+ 324_020,
+ "a lower forward floor must never lower the ceiling"
+ );
+ assert.equal(
+ resolveMessagesBackfillTargetUid({
+ forwardFloorUid: 0,
+ prior: { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 },
+ }),
+ 324_020,
+ "a zero/absent forward floor must not collapse the ceiling"
+ );
+});
+
+test("resolveMessagesBackfillTargetUid: a first run with no stored ceiling adopts the forward floor", () => {
+ // On a full resync the forward pass fetches NOTHING, yet the watermark is
+ // written from the live uidnext. Everything below it is therefore historical
+ // work and the ceiling must say so.
+ assert.equal(
+ resolveMessagesBackfillTargetUid({ forwardFloorUid: 1200, prior: {} }),
+ 1200,
+ "an unstarted walk takes the forward floor as its ceiling"
+ );
+});
+
+test("resolveMessagesBackfillTargetUid: a quiet mailbox leaves the ceiling exactly where it was", () => {
+ // Termination guard: when no mail arrived, the ceiling is unchanged, so the
+ // walk converges instead of chasing an ever-rising target forever.
+ const prior = { backfilled_through_uid: 150_000, target_uid: 324_020, uidvalidity: 1 };
+ assert.equal(resolveMessagesBackfillTargetUid({ forwardFloorUid: 324_020, prior }), 324_020);
+ // Idempotent: re-resolving against its own output is a fixed point.
+ assert.equal(
+ resolveMessagesBackfillTargetUid({
+ forwardFloorUid: 324_020,
+ prior: { ...prior, target_uid: 324_020 },
+ }),
+ 324_020,
+ "re-resolving must be a fixed point, not a ratchet that keeps finding new work"
+ );
+});
+
+test("advanceMessagesBackfillCursor: an explicit raised ceiling reopens a completed walk without rewinding it", () => {
+ // A walk that finished at 1200 must reopen when the forward watermark has
+ // moved to 1301 — but `backfilled_through_uid` must hold at 1200, not rewind.
+ const completed = {
+ backfilled_through_uid: 1200,
+ completed_at: FROZEN_NOW,
+ target_uid: 1200,
+ uidvalidity: 123,
+ };
+ const reopened = advanceMessagesBackfillCursor({
+ now: FROZEN_NOW,
+ pageEndUid: 1200,
+ prior: { ...completed, target_uid: 1300 },
+ });
+ assert.equal(reopened.target_uid, 1300, "the raised ceiling must be persisted");
+ assert.equal(reopened.backfilled_through_uid, 1200, "progress must never rewind when the ceiling rises");
+ assert.equal(reopened.completed_at, null, "a walk with UIDs left to reach is not complete");
+});
+
+test("advanceMessagesBackfillCursor: a settled walk under an unchanged ceiling stays settled", () => {
+ // The other half of the reopen rule: without new mail the ceiling does not
+ // move, so a finished walk must stay finished rather than re-scanning.
+ const completed = {
+ backfilled_through_uid: 1200,
+ completed_at: FROZEN_NOW,
+ target_uid: 1200,
+ uidvalidity: 123,
+ };
+ const settled = advanceMessagesBackfillCursor({
+ now: FROZEN_NOW,
+ pageEndUid: 1200,
+ prior: completed,
+ });
+ assert.equal(settled.backfilled_through_uid, 1200);
+ assert.equal(typeof settled.completed_at, "string", "a walk that reached its ceiling stays complete");
+});
+
+test("advanceMessagesBackfillCursor: the commit honours the raised ceiling carried on the cursor", () => {
+ // The ceiling has exactly one source of truth: `prior.target_uid`, already
+ // raised by `resolveMessagesBackfillTargetUid`. A page that walked into the
+ // reopened band cannot be committed against a STALE ceiling — that is the
+ // half-fix where the repair looks applied but the next run re-reads the old
+ // value and the band reopens.
+ const stale = { backfilled_through_uid: 1200, completed_at: FROZEN_NOW, target_uid: 1200, uidvalidity: 123 };
+ assert.throws(
+ () => advanceMessagesBackfillCursor({ now: FROZEN_NOW, pageEndUid: 1300, prior: stale }),
+ /must not pass its target/,
+ "a page that walked the reopened band must not be committable against the stale ceiling"
+ );
+ const committed = advanceMessagesBackfillCursor({
+ now: FROZEN_NOW,
+ pageEndUid: 1300,
+ prior: { ...stale, target_uid: 1300 },
+ });
+ assert.equal(committed.target_uid, 1300, "the cursor must store the raised ceiling for the next run to read");
+});
+
+test("advanceMessagesBackfillCursor: a partial page under a reopened ceiling stays incomplete", () => {
+ // A reopened band larger than one page must leave the walk open, so the next
+ // run continues into it rather than declaring the mailbox finished.
+ const partial = advanceMessagesBackfillCursor({
+ now: FROZEN_NOW,
+ pageEndUid: 1700,
+ prior: { backfilled_through_uid: 1200, completed_at: FROZEN_NOW, target_uid: 2000, uidvalidity: 123 },
+ });
+ assert.equal(partial.target_uid, 2000);
+ assert.equal(partial.backfilled_through_uid, 1700);
+ assert.equal(partial.completed_at, null, "the walk has 1701..2000 left and must not report completion");
+});
+
test("advanceMessagesBackfillCursor: page completion is monotonic and partial pages stay incomplete", () => {
const partial = advanceMessagesBackfillCursor({
now: FROZEN_NOW,
@@ -1636,19 +1787,12 @@ test("runAllMailPasses: first historical page is bounded, durable only at page e
true,
"a bounded page proves its own detail coverage even while historical continuation remains"
);
- assert.deepEqual(
+ assert.equal(
protocolMessages.find((message) => message.type === "DETAIL_COVERAGE" && message.stream === "message_bodies"),
- {
- type: "DETAIL_COVERAGE",
- reference_only: true,
- stream: "message_bodies",
- state_stream: "messages",
- required_keys: [],
- hydrated_keys: [],
- considered: 2,
- covered: 2,
- },
- "the body stream reports the same bounded parent-message pass"
+ undefined,
+ "message_bodies is declared `state_stream: messages` in the manifest, so it must emit NO DETAIL_COVERAGE — " +
+ "the runtime rejects the whole run if it does, and the only counts available here are the parent " +
+ "message pass's, which would fabricate covered == considered for bodies never hydrated"
);
assert.deepEqual(
protocolMessages.find((message) => message.type === "SKIP_RESULT" && message.stream === "messages"),
@@ -1675,6 +1819,105 @@ test("runAllMailPasses: first historical page is bounded, durable only at page e
}
});
+/**
+ * The manifest — not this test — is the authority on which streams may prove
+ * their own coverage. A stream declared with a `state_stream` parent is a
+ * static single-parent detail stream: its checkpoint status is projected from
+ * that parent's commit outcome, so the runtime rejects the ENTIRE run if such
+ * a stream emits DETAIL_COVERAGE (see `validateDetailCoverageAgainstManifest`).
+ *
+ * This reads the real manifest rather than hard-coding `message_bodies`, so a
+ * future stream that gains a `state_stream` parent is covered the day the
+ * manifest says so.
+ *
+ * Regression: a `message_bodies` DETAIL_COVERAGE reporting the PARENT message
+ * pass's considered/covered shipped to production and failed every Gmail run
+ * with `runtime_error`, driving the scheduler into cooling_off. It was also
+ * dishonest on its own terms — it claimed covered == considered for bodies
+ * that were never hydrated.
+ */
+test("runAllMailPasses: no stream the manifest declares with a state_stream parent may emit DETAIL_COVERAGE", async () => {
+ const manifest = JSON.parse(await readFile(new URL("../../manifests/gmail.json", import.meta.url), "utf8")) as {
+ streams?: Array<{ name: string; state_stream?: string }>;
+ };
+ const stateStreamParented = new Set(
+ (manifest.streams || [])
+ .filter(
+ (stream) =>
+ typeof stream.state_stream === "string" && stream.state_stream && stream.state_stream !== stream.name
+ )
+ .map((stream) => stream.name)
+ );
+ assert.ok(
+ stateStreamParented.has("message_bodies"),
+ "guard precondition: the gmail manifest must still declare message_bodies with a state_stream parent"
+ );
+
+ const originalWrite = globalThis.process.stdout.write;
+ const protocolMessages: Record[] = [];
+ globalThis.process.stdout.write = ((data: string): boolean => {
+ if (typeof data === "string") {
+ try {
+ protocolMessages.push(JSON.parse(data) as Record);
+ } catch {
+ // Ignore non-protocol output.
+ }
+ }
+ return true;
+ }) as typeof process.stdout.write;
+
+ try {
+ const client: Pick = {
+ close: mock.fn(),
+ download: () => {
+ throw new Error("download must not be called without attachments");
+ },
+ fetchOne: () => {
+ throw new Error("fetchOne must not be called without bodies");
+ },
+ search: mock.fn(async () => []),
+ mailbox: {
+ delimiter: "/",
+ exists: 1200,
+ flags: new Set(),
+ path: "[Gmail]/All Mail",
+ uidNext: 1201,
+ uidValidity: 123n,
+ },
+ // biome-ignore lint/suspicious/useAwait: async generator is required by the ImapFlow fetch shape.
+ async *fetch() {
+ for (const uid of [1, 2]) {
+ yield makeMsg({ uid, emailId: `msg-${uid}` });
+ }
+ },
+ };
+
+ await runAllMailPasses(
+ client,
+ makeAllMailMailbox(),
+ {},
+ {
+ emitRecord: async () => true,
+ emittedAt: FROZEN_NOW,
+ requested: makeRequested(["messages", "message_bodies"]),
+ }
+ );
+
+ const illegal = protocolMessages
+ .filter((message) => message.type === "DETAIL_COVERAGE")
+ .map((message) => message.stream as string)
+ .filter((stream) => stateStreamParented.has(stream));
+ assert.deepEqual(
+ illegal,
+ [],
+ "these streams emitted DETAIL_COVERAGE despite a manifest-declared state_stream parent, which fails the " +
+ `whole run at runtime: ${illegal.join(", ")}`
+ );
+ } finally {
+ globalThis.process.stdout.write = originalWrite;
+ }
+});
+
test("runAllMailPasses: scheduled runs advance historical pages while forwarding new mail", async () => {
const originalWrite = globalThis.process.stdout.write;
const protocolMessages: Record[] = [];
@@ -1748,6 +1991,10 @@ test("runAllMailPasses: scheduled runs advance historical pages while forwarding
const first = await run({});
assert.deepEqual(fetchRanges, ["1:500"]);
assert.deepEqual(first.all_mail, {
+ // The mailbox's own EXISTS count rides along so the next run in this
+ // epoch can prove the inventory did not shrink (see
+ // all-mail-inventory.test.ts).
+ exists: 1200,
forward_uidnext: 1201,
highest_modseq: null,
uidnext: 501,
@@ -1763,11 +2010,203 @@ test("runAllMailPasses: scheduled runs advance historical pages while forwarding
emittedRecords.some((record) => record.stream === "messages" && record.data.id === "msg-1250"),
"new mail in the forward range is collected while historical backfill is pending"
);
+ const messagesCoverage = protocolMessages.find(
+ (message) => message.type === "DETAIL_COVERAGE" && message.stream === "messages"
+ );
+ assert.deepEqual(
+ messagesCoverage && { considered: messagesCoverage.considered, covered: messagesCoverage.covered },
+ { considered: 2, covered: 2 },
+ "the messages DETAIL_COVERAGE must sum BOTH the historical page (msg-1001) and the forward page " +
+ "(msg-1250) — reporting only the historical pass's considered/covered undercounts the denominator " +
+ "against the raw collected-record total, the same class of defect message_bodies's coverage " +
+ "(which does sum both passes) avoids"
+ );
+
+ // The continuation must describe the SAME page the DETAIL_COVERAGE fact
+ // describes. The runtime's isHealthyBoundedContinuation
+ // (reference-implementation/server/continuation-proof.ts) admits a bounded
+ // page only when continuation.considered === fact.considered AND
+ // continuation.covered === fact.covered. When the coverage fact summed both
+ // passes but the continuation reported historical-only counts, the pair
+ // desynced by exactly the forward-pass count on every run that carried new
+ // mail alongside a pending backfill (observed live: fact 52/52 vs
+ // continuation 51/51), the identity check failed, and the stream fell
+ // through to retryable_gap instead of deriving complete.
+ const messagesSkip = protocolMessages.find(
+ (message) => message.type === "SKIP_RESULT" && message.stream === "messages"
+ );
+ const skipContinuation = messagesSkip?.continuation as Record | undefined;
+ assert.equal(
+ messagesSkip?.reason,
+ "historical_backfill_pending",
+ "a page with historical work remaining still emits its bounded continuation"
+ );
+ assert.deepEqual(
+ skipContinuation && { considered: skipContinuation.considered, covered: skipContinuation.covered },
+ { considered: messagesCoverage?.considered, covered: messagesCoverage?.covered },
+ "the historical continuation skip must carry the SAME considered/covered as the messages " +
+ "DETAIL_COVERAGE fact — the runtime's isHealthyBoundedContinuation requires that identity, so any " +
+ "drift between the two emissions silently degrades a complete stream to a retryable_gap"
+ );
+
+ // End-to-end: the runtime predicate itself accepts the synced pair, and
+ // would reject the historical-only counts the desynced code emitted.
+ const isHealthyBoundedContinuation = (
+ fact: { considered: number; covered: number },
+ cont: { considered: number; covered: number }
+ ) => cont.considered === fact.considered && cont.covered === fact.covered && fact.considered === fact.covered;
+ assert.equal(
+ isHealthyBoundedContinuation(
+ { considered: messagesCoverage?.considered as number, covered: messagesCoverage?.covered as number },
+ { considered: skipContinuation?.considered as number, covered: skipContinuation?.covered as number }
+ ),
+ true,
+ "the emitted fact/continuation pair satisfies the runtime's bounded-continuation identity check"
+ );
+ assert.equal(
+ isHealthyBoundedContinuation({ considered: 2, covered: 2 }, { considered: 1, covered: 1 }),
+ false,
+ "control: the historical-only counts the regression emitted do NOT satisfy that check"
+ );
+
+ // Run 2 raised the forward watermark to 1301, so UIDs 1201..1300 are now
+ // below where the forward walk resumes. The historical ceiling must have
+ // risen with it (1200 -> 1300) or that interval belongs to neither walk.
+ // Before the ceiling tracked the watermark this read "1001:1200", leaving
+ // 1201..1300 orphaned — the live 297-UID band in miniature.
+ assert.equal(
+ (second.backfill as Record).target_uid,
+ 1300,
+ "the historical ceiling must rise to meet the forward watermark, not stay frozen at 1200"
+ );
const third = await run({ messages: second });
- assert.deepEqual(fetchRanges, ["1001:1200", "1301:*"]);
+ assert.deepEqual(fetchRanges, ["1001:1300", "1301:*"]);
assert.equal((third.all_mail as Record).uidnext, 1301);
assert.equal(typeof (third.backfill as Record).completed_at, "string");
+ assert.equal(
+ (third.backfill as Record).backfilled_through_uid,
+ 1300,
+ "the walk closes the whole space up to the forward resume point"
+ );
+ } finally {
+ globalThis.process.stdout.write = originalWrite;
+ }
+});
+
+/**
+ * The band must not reopen after the historical walk has FINISHED.
+ *
+ * This is the live shape: the backfill reaches its ceiling, `completed_at` is
+ * stamped, and then mail keeps arriving. A completed walk that copies its stale
+ * ceiling forward leaves every newly-arrived UID above the ceiling and below the
+ * forward watermark — belonging to neither walk. That is precisely how the
+ * repaired live cursor went from band=0 to band=2 to band=3 within minutes.
+ *
+ * The end-to-end path is what makes this test necessary: the pure resolver is
+ * correct in isolation, but the completed-walk branch in `runAllMailPasses`
+ * chooses whether to consult it at all.
+ */
+test("runAllMailPasses: a completed historical walk reopens for new mail instead of freezing its ceiling", async () => {
+ const originalWrite = globalThis.process.stdout.write;
+ const protocolMessages: Record[] = [];
+ const fetchRanges: string[] = [];
+ let uidNext = 1201;
+ globalThis.process.stdout.write = ((data: string): boolean => {
+ if (typeof data === "string") {
+ try {
+ protocolMessages.push(JSON.parse(data) as Record);
+ } catch {
+ // Ignore non-protocol output.
+ }
+ }
+ return true;
+ }) as typeof process.stdout.write;
+
+ try {
+ const client: Pick = {
+ close: mock.fn(),
+ download: () => {
+ throw new Error("download must not be called without attachments");
+ },
+ fetchOne: () => {
+ throw new Error("fetchOne must not be called without bodies");
+ },
+ search: mock.fn(async () => []),
+ mailbox: {
+ delimiter: "/",
+ exists: 1200,
+ flags: new Set